mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-08-28 05:33:24 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d706d2a9fe | ||
|
|
5caa3327fd | ||
|
|
c72c8d1574 | ||
|
|
b1f5445689 |
@@ -27,7 +27,7 @@ from documents.models import DocumentType
|
|||||||
from documents.models import PaperlessTask
|
from documents.models import PaperlessTask
|
||||||
from documents.models import StoragePath
|
from documents.models import StoragePath
|
||||||
from documents.models import Tag
|
from documents.models import Tag
|
||||||
from documents.permissions import set_permissions_for_object
|
from documents.permissions import set_permissions_for_objects
|
||||||
from documents.plugins.helpers import DocumentsStatusManager
|
from documents.plugins.helpers import DocumentsStatusManager
|
||||||
from documents.tasks import bulk_update_documents
|
from documents.tasks import bulk_update_documents
|
||||||
from documents.tasks import consume_file
|
from documents.tasks import consume_file
|
||||||
@@ -430,10 +430,13 @@ def set_permissions(
|
|||||||
else:
|
else:
|
||||||
qs.update(owner=owner)
|
qs.update(owner=owner)
|
||||||
|
|
||||||
for doc in qs:
|
|
||||||
set_permissions_for_object(permissions=set_permissions, object=doc, merge=merge)
|
|
||||||
|
|
||||||
affected_docs = list(qs.values_list("pk", flat=True))
|
affected_docs = list(qs.values_list("pk", flat=True))
|
||||||
|
set_permissions_for_objects(
|
||||||
|
permissions=set_permissions,
|
||||||
|
model=Document,
|
||||||
|
pks=affected_docs,
|
||||||
|
merge=merge,
|
||||||
|
)
|
||||||
|
|
||||||
bulk_update_documents.apply_async(
|
bulk_update_documents.apply_async(
|
||||||
kwargs={"document_ids": affected_docs},
|
kwargs={"document_ids": affected_docs},
|
||||||
|
|||||||
@@ -173,6 +173,184 @@ def set_permissions_for_object(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_permissions(codenames: set[str], ctype: ContentType) -> list[Permission]:
|
||||||
|
"""
|
||||||
|
Resolves `codenames` to Permission rows, raising like the single-object
|
||||||
|
assign_perm() this bulk path replaces does (via a `.get()` internally)
|
||||||
|
if any codename doesn't exist -- e.g. a client-supplied action name that
|
||||||
|
was never validated (BulkEditObjectsSerializer._validate_permissions
|
||||||
|
calls validate_set_permissions() only for its side-effecting id checks
|
||||||
|
and discards the filtered dict it returns, so an unrecognized action key
|
||||||
|
reaches this function as-is). A plain `.filter()` with no existence
|
||||||
|
check would otherwise silently build zero rows and no-op instead of
|
||||||
|
reporting the bad input.
|
||||||
|
"""
|
||||||
|
permission_objs = list(
|
||||||
|
Permission.objects.filter(content_type=ctype, codename__in=codenames),
|
||||||
|
)
|
||||||
|
missing = codenames - {p.codename for p in permission_objs}
|
||||||
|
if missing:
|
||||||
|
raise Permission.DoesNotExist(
|
||||||
|
f"Permission matching query does not exist for codename(s): "
|
||||||
|
f"{', '.join(sorted(missing))}",
|
||||||
|
)
|
||||||
|
return permission_objs
|
||||||
|
|
||||||
|
|
||||||
|
# Target number of permission rows to build in Python before handing them to
|
||||||
|
# bulk_create -- keeps peak memory bounded for a large "apply to all" call,
|
||||||
|
# independent of bulk_create's own batch_size (which only caps the size of
|
||||||
|
# each INSERT statement, not how many row objects exist in memory at once).
|
||||||
|
_PERMISSION_ROW_CHUNK_SIZE = 5000
|
||||||
|
|
||||||
|
|
||||||
|
def _apply_bulk_permission_entry(
|
||||||
|
*,
|
||||||
|
perm_model: type[UserObjectPermission] | type[GroupObjectPermission],
|
||||||
|
identity_model: type[User] | type[Group],
|
||||||
|
identity_field: str,
|
||||||
|
ids: list[int],
|
||||||
|
codename: str,
|
||||||
|
permission_objs: list[Permission],
|
||||||
|
ctype: ContentType,
|
||||||
|
object_pks: list[str],
|
||||||
|
merge: bool,
|
||||||
|
) -> None:
|
||||||
|
# Only the ids are needed to build permission rows (via `<field>_id=`),
|
||||||
|
# so avoid fetching full User/Group rows for identities that may not
|
||||||
|
# even end up being granted anything new.
|
||||||
|
add_ids = set(
|
||||||
|
identity_model.objects.filter(id__in=ids).values_list("id", flat=True),
|
||||||
|
)
|
||||||
|
|
||||||
|
if not merge:
|
||||||
|
existing_ids = set(
|
||||||
|
perm_model.objects.filter(
|
||||||
|
content_type=ctype,
|
||||||
|
object_pk__in=object_pks,
|
||||||
|
permission__codename=codename,
|
||||||
|
)
|
||||||
|
.values_list(f"{identity_field}_id", flat=True)
|
||||||
|
.distinct(),
|
||||||
|
)
|
||||||
|
remove_ids = existing_ids - add_ids
|
||||||
|
if remove_ids:
|
||||||
|
perm_model.objects.filter(
|
||||||
|
content_type=ctype,
|
||||||
|
object_pk__in=object_pks,
|
||||||
|
permission__codename=codename,
|
||||||
|
**{f"{identity_field}_id__in": remove_ids},
|
||||||
|
).delete()
|
||||||
|
|
||||||
|
if not add_ids:
|
||||||
|
return
|
||||||
|
|
||||||
|
rows_per_pk = len(permission_objs) * len(add_ids)
|
||||||
|
pks_per_chunk = max(1, _PERMISSION_ROW_CHUNK_SIZE // rows_per_pk)
|
||||||
|
for start in range(0, len(object_pks), pks_per_chunk):
|
||||||
|
pk_chunk = object_pks[start : start + pks_per_chunk]
|
||||||
|
rows = [
|
||||||
|
perm_model(
|
||||||
|
content_type=ctype,
|
||||||
|
object_pk=pk,
|
||||||
|
permission=permission_obj,
|
||||||
|
**{f"{identity_field}_id": identity_id},
|
||||||
|
)
|
||||||
|
for permission_obj in permission_objs
|
||||||
|
for pk in pk_chunk
|
||||||
|
for identity_id in add_ids
|
||||||
|
]
|
||||||
|
# ignore_conflicts skips only rows that already exist as an exact
|
||||||
|
# (identity, permission, object) match -- the same de-dup the
|
||||||
|
# underlying (user|group, permission, object_pk) unique constraint
|
||||||
|
# already enforces for the single-object assign_perm() this
|
||||||
|
# replaces, so it doesn't change what counts as "already granted".
|
||||||
|
# batch_size caps how many rows go into a single INSERT so a huge
|
||||||
|
# chunk doesn't build one enormous statement.
|
||||||
|
perm_model.objects.bulk_create(rows, ignore_conflicts=True, batch_size=1000)
|
||||||
|
|
||||||
|
|
||||||
|
def set_permissions_for_objects(
|
||||||
|
permissions: dict,
|
||||||
|
model: type[Model],
|
||||||
|
pks: QuerySet | list,
|
||||||
|
*,
|
||||||
|
merge: bool = False,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
Bulk equivalent of set_permissions_for_object: applies the same
|
||||||
|
permission changes to every object identified by `pks` at once.
|
||||||
|
|
||||||
|
Takes a model + pks (rather than model instances) deliberately -- the
|
||||||
|
permission rows built below only ever need `pk`, `content_type`, and
|
||||||
|
identity ids, so callers shouldn't have to fetch full rows (with every
|
||||||
|
other field) just to hand them to this function.
|
||||||
|
|
||||||
|
Deliberately does not use guardian's queryset/list-aware assign_perm:
|
||||||
|
passing a list as the object routes to bulk_assign_perm, which skips
|
||||||
|
creating a direct permission row for anyone who already has the
|
||||||
|
permission via ANY group membership (it checks
|
||||||
|
ObjectPermissionChecker.has_perm, which is group-inheritance-aware) --
|
||||||
|
unlike the single-object assign_perm this replaces, which always
|
||||||
|
ensures a direct row via get_or_create regardless of group-derived
|
||||||
|
access. Losing that guarantee would mean a later revocation of the
|
||||||
|
group's grant silently strips access an admin explicitly asked to be
|
||||||
|
direct. Bulk-creating rows straight against the permission models
|
||||||
|
instead (see _apply_bulk_permission_entry) preserves the original
|
||||||
|
always-create-a-direct-row semantics while still batching every object
|
||||||
|
and every identity into one query per action, rather than one query per
|
||||||
|
(object, user) pair.
|
||||||
|
"""
|
||||||
|
object_pks = [str(pk) for pk in pks]
|
||||||
|
if not object_pks: # pragma: no cover
|
||||||
|
return
|
||||||
|
|
||||||
|
model_name = model.__name__.lower()
|
||||||
|
ctype = ContentType.objects.get_for_model(model)
|
||||||
|
|
||||||
|
for action, entry in permissions.items():
|
||||||
|
codename = f"{action}_{model_name}"
|
||||||
|
implied_codenames = {codename}
|
||||||
|
if action == "change":
|
||||||
|
# change gives view too
|
||||||
|
implied_codenames.add(f"view_{model_name}")
|
||||||
|
|
||||||
|
# Resolved once per action (not once per users/groups branch) and
|
||||||
|
# shared between both below -- also where an unrecognized action
|
||||||
|
# name (see _resolve_permissions) is caught.
|
||||||
|
permission_objs = (
|
||||||
|
_resolve_permissions(implied_codenames, ctype)
|
||||||
|
if "users" in entry or "groups" in entry
|
||||||
|
else []
|
||||||
|
)
|
||||||
|
|
||||||
|
if "users" in entry:
|
||||||
|
_apply_bulk_permission_entry(
|
||||||
|
perm_model=UserObjectPermission,
|
||||||
|
identity_model=User,
|
||||||
|
identity_field="user",
|
||||||
|
ids=entry["users"],
|
||||||
|
codename=codename,
|
||||||
|
permission_objs=permission_objs,
|
||||||
|
ctype=ctype,
|
||||||
|
object_pks=object_pks,
|
||||||
|
merge=merge,
|
||||||
|
)
|
||||||
|
|
||||||
|
if "groups" in entry:
|
||||||
|
_apply_bulk_permission_entry(
|
||||||
|
perm_model=GroupObjectPermission,
|
||||||
|
identity_model=Group,
|
||||||
|
identity_field="group",
|
||||||
|
ids=entry["groups"],
|
||||||
|
codename=codename,
|
||||||
|
permission_objs=permission_objs,
|
||||||
|
ctype=ctype,
|
||||||
|
object_pks=object_pks,
|
||||||
|
merge=merge,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def permitted_object_ids(
|
def permitted_object_ids(
|
||||||
user: User | None,
|
user: User | None,
|
||||||
model: type[Model],
|
model: type[Model],
|
||||||
|
|||||||
@@ -2,10 +2,13 @@ import datetime
|
|||||||
import json
|
import json
|
||||||
from unittest import mock
|
from unittest import mock
|
||||||
|
|
||||||
|
from django.contrib.auth.models import Group
|
||||||
from django.contrib.auth.models import Permission
|
from django.contrib.auth.models import Permission
|
||||||
from django.contrib.auth.models import User
|
from django.contrib.auth.models import User
|
||||||
from django.test import override_settings
|
from django.test import override_settings
|
||||||
from guardian.shortcuts import assign_perm
|
from guardian.shortcuts import assign_perm
|
||||||
|
from guardian.shortcuts import get_groups_with_perms
|
||||||
|
from guardian.shortcuts import get_users_with_perms
|
||||||
from rest_framework import status
|
from rest_framework import status
|
||||||
from rest_framework.test import APITestCase
|
from rest_framework.test import APITestCase
|
||||||
|
|
||||||
@@ -815,6 +818,47 @@ class TestBulkEditObjects(APITestCase):
|
|||||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||||
self.assertEqual(StoragePath.objects.count(), 0)
|
self.assertEqual(StoragePath.objects.count(), 0)
|
||||||
|
|
||||||
|
def test_bulk_objects_set_permissions_batched_across_object_count(
|
||||||
|
self,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- Many tags are being bulk-edited to set permissions at once
|
||||||
|
WHEN:
|
||||||
|
- bulk_edit_objects API endpoint is called with set_permissions
|
||||||
|
operation over a small batch vs. a much larger one
|
||||||
|
THEN:
|
||||||
|
- Permissions are applied correctly at both scales
|
||||||
|
"""
|
||||||
|
group1 = Group.objects.create(name="perm-group")
|
||||||
|
permissions = {
|
||||||
|
"view": {"users": [self.user1.id, self.user2.id], "groups": [group1.id]},
|
||||||
|
"change": {"users": [self.user1.id], "groups": [group1.id]},
|
||||||
|
}
|
||||||
|
|
||||||
|
def run_with_n_tags(n: int) -> None:
|
||||||
|
tags = [Tag.objects.create(name=f"perm-tag-{n}-{i}") for i in range(n)]
|
||||||
|
response = self.client.post(
|
||||||
|
"/api/bulk_edit_objects/",
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"objects": [t.id for t in tags],
|
||||||
|
"object_type": "tags",
|
||||||
|
"operation": "set_permissions",
|
||||||
|
"permissions": permissions,
|
||||||
|
"merge": False,
|
||||||
|
},
|
||||||
|
),
|
||||||
|
content_type="application/json",
|
||||||
|
)
|
||||||
|
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||||
|
for tag in tags:
|
||||||
|
self.assertEqual(get_users_with_perms(tag).count(), 2)
|
||||||
|
self.assertEqual(get_groups_with_perms(tag).count(), 1)
|
||||||
|
|
||||||
|
run_with_n_tags(5)
|
||||||
|
run_with_n_tags(50)
|
||||||
|
|
||||||
def test_bulk_objects_delete_all_filtered(self) -> None:
|
def test_bulk_objects_delete_all_filtered(self) -> None:
|
||||||
"""
|
"""
|
||||||
GIVEN:
|
GIVEN:
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ from unittest import mock
|
|||||||
|
|
||||||
import pikepdf
|
import pikepdf
|
||||||
from django.contrib.auth.models import Group
|
from django.contrib.auth.models import Group
|
||||||
|
from django.contrib.auth.models import Permission
|
||||||
from django.contrib.auth.models import User
|
from django.contrib.auth.models import User
|
||||||
from django.test import TestCase
|
from django.test import TestCase
|
||||||
from guardian.shortcuts import assign_perm
|
from guardian.shortcuts import assign_perm
|
||||||
@@ -19,6 +20,7 @@ from documents.models import Document
|
|||||||
from documents.models import DocumentType
|
from documents.models import DocumentType
|
||||||
from documents.models import StoragePath
|
from documents.models import StoragePath
|
||||||
from documents.models import Tag
|
from documents.models import Tag
|
||||||
|
from documents.permissions import set_permissions_for_objects
|
||||||
from documents.tests.utils import DirectoriesMixin
|
from documents.tests.utils import DirectoriesMixin
|
||||||
|
|
||||||
|
|
||||||
@@ -510,6 +512,120 @@ class TestBulkEdit(DirectoriesMixin, TestCase):
|
|||||||
)
|
)
|
||||||
self.assertEqual(groups_with_perms.count(), 2)
|
self.assertEqual(groups_with_perms.count(), 2)
|
||||||
|
|
||||||
|
@mock.patch("documents.tasks.bulk_update_documents.apply_async")
|
||||||
|
def test_set_permissions_batched_across_document_count(
|
||||||
|
self,
|
||||||
|
m,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- Many documents are being bulk-edited to set permissions at once
|
||||||
|
WHEN:
|
||||||
|
- set_permissions runs over a small batch vs. a much larger one
|
||||||
|
THEN:
|
||||||
|
- Permissions are applied correctly at both scales
|
||||||
|
"""
|
||||||
|
permissions = {
|
||||||
|
"view": {
|
||||||
|
"users": [self.user1.id, self.user2.id],
|
||||||
|
"groups": [self.group2.id],
|
||||||
|
},
|
||||||
|
"change": {
|
||||||
|
"users": [self.user1.id],
|
||||||
|
"groups": [self.group2.id],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
def run_with_n_documents(n: int) -> None:
|
||||||
|
docs = [
|
||||||
|
Document.objects.create(checksum=f"perm-{n}-{i}", title=f"perm-{n}-{i}")
|
||||||
|
for i in range(n)
|
||||||
|
]
|
||||||
|
bulk_edit.set_permissions(
|
||||||
|
[doc.id for doc in docs],
|
||||||
|
set_permissions=permissions,
|
||||||
|
owner=self.owner,
|
||||||
|
merge=False,
|
||||||
|
)
|
||||||
|
for doc in docs:
|
||||||
|
self.assertEqual(get_users_with_perms(doc).count(), 2)
|
||||||
|
self.assertEqual(get_groups_with_perms(doc).count(), 1)
|
||||||
|
|
||||||
|
run_with_n_documents(5)
|
||||||
|
run_with_n_documents(50)
|
||||||
|
|
||||||
|
@mock.patch("documents.tasks.bulk_update_documents.apply_async")
|
||||||
|
def test_set_permissions_grants_direct_perm_even_if_already_granted_via_group(
|
||||||
|
self,
|
||||||
|
m,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A user already has view access to a document via group
|
||||||
|
membership, with no direct grant of their own
|
||||||
|
WHEN:
|
||||||
|
- set_permissions explicitly grants that same user direct view
|
||||||
|
access via bulk_edit
|
||||||
|
THEN:
|
||||||
|
- A direct permission grant is created for the user, not skipped
|
||||||
|
because they already have equivalent access via the group
|
||||||
|
|
||||||
|
Regression test: guardian's queryset-aware assign_perm() (routed to
|
||||||
|
when the target is a list/queryset) skips creating a direct row for
|
||||||
|
anyone whose ObjectPermissionChecker.has_perm() already returns True
|
||||||
|
-- which includes group-derived access. The single-object assign_perm
|
||||||
|
this bulk path replaces has no such check; it always ensures a
|
||||||
|
direct row via get_or_create. Losing that guarantee would mean
|
||||||
|
revoking the group's grant later silently strips access that was
|
||||||
|
supposed to be explicit.
|
||||||
|
"""
|
||||||
|
self.doc1.owner = self.user1
|
||||||
|
self.doc1.save()
|
||||||
|
self.user1.groups.add(self.group1)
|
||||||
|
assign_perm("view_document", self.group1, self.doc1)
|
||||||
|
|
||||||
|
bulk_edit.set_permissions(
|
||||||
|
[self.doc1.id],
|
||||||
|
set_permissions={
|
||||||
|
"view": {"users": [self.user1.id], "groups": []},
|
||||||
|
},
|
||||||
|
merge=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
direct_users = get_users_with_perms(
|
||||||
|
self.doc1,
|
||||||
|
only_with_perms_in=["view_document"],
|
||||||
|
with_group_users=False,
|
||||||
|
)
|
||||||
|
self.assertIn(self.user1, direct_users)
|
||||||
|
|
||||||
|
def test_set_permissions_for_objects_raises_for_unknown_action(self) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- An unrecognized permission action name with users to grant it
|
||||||
|
to
|
||||||
|
WHEN:
|
||||||
|
- set_permissions_for_objects is called
|
||||||
|
THEN:
|
||||||
|
- Permission.DoesNotExist is raised, not a silent no-op
|
||||||
|
|
||||||
|
Regression test: the endpoint that calls this
|
||||||
|
(BulkEditObjectPermissionsView) never actually validates action
|
||||||
|
names against the raw client-supplied permissions dict --
|
||||||
|
BulkEditObjectsSerializer._validate_permissions calls
|
||||||
|
validate_set_permissions() only for its side-effecting user/group id
|
||||||
|
checks and discards the filtered dict it returns -- so a bogus
|
||||||
|
action key reaches this function as-is. Resolving the Permission via
|
||||||
|
a bare `.filter()` (which returns empty instead of raising) would
|
||||||
|
silently drop the grant and report success.
|
||||||
|
"""
|
||||||
|
with self.assertRaises(Permission.DoesNotExist):
|
||||||
|
set_permissions_for_objects(
|
||||||
|
{"not_a_real_action": {"users": [self.user1.id], "groups": []}},
|
||||||
|
Document,
|
||||||
|
[self.doc1.pk],
|
||||||
|
)
|
||||||
|
|
||||||
@mock.patch("documents.models.Document.delete")
|
@mock.patch("documents.models.Document.delete")
|
||||||
def test_delete_documents_old_uuid_field(self, m) -> None:
|
def test_delete_documents_old_uuid_field(self, m) -> None:
|
||||||
m.side_effect = Exception("Data too long for column 'transaction_id' at row 1")
|
m.side_effect = Exception("Data too long for column 'transaction_id' at row 1")
|
||||||
|
|||||||
@@ -178,7 +178,7 @@ from documents.permissions import has_perms_owner_aware
|
|||||||
from documents.permissions import has_system_status_permission
|
from documents.permissions import has_system_status_permission
|
||||||
from documents.permissions import permitted_document_ids
|
from documents.permissions import permitted_document_ids
|
||||||
from documents.permissions import permitted_object_ids
|
from documents.permissions import permitted_object_ids
|
||||||
from documents.permissions import set_permissions_for_object
|
from documents.permissions import set_permissions_for_objects
|
||||||
from documents.plugins.date_parsing import get_date_parser
|
from documents.plugins.date_parsing import get_date_parser
|
||||||
from documents.schema import generate_object_with_permissions_schema
|
from documents.schema import generate_object_with_permissions_schema
|
||||||
from documents.search import SearchHit
|
from documents.search import SearchHit
|
||||||
@@ -4914,12 +4914,12 @@ class BulkEditObjectsView(PassUserMixin):
|
|||||||
qs_owner_update.update(owner=owner)
|
qs_owner_update.update(owner=owner)
|
||||||
|
|
||||||
if "permissions" in serializer.validated_data:
|
if "permissions" in serializer.validated_data:
|
||||||
for obj in qs:
|
set_permissions_for_objects(
|
||||||
set_permissions_for_object(
|
permissions=permissions,
|
||||||
permissions=permissions,
|
model=object_class,
|
||||||
object=obj,
|
pks=qs.values_list("pk", flat=True),
|
||||||
merge=merge,
|
merge=merge,
|
||||||
)
|
)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(
|
logger.warning(
|
||||||
|
|||||||
@@ -5,14 +5,13 @@ from django.conf import settings
|
|||||||
from django.contrib.auth.models import User
|
from django.contrib.auth.models import User
|
||||||
|
|
||||||
from documents.models import Document
|
from documents.models import Document
|
||||||
from documents.permissions import permitted_object_ids
|
from documents.permissions import get_objects_for_user_owner_aware
|
||||||
from documents.permissions import restrict_queryset_to_visible
|
|
||||||
from documents.permissions import user_is_unrestricted
|
|
||||||
from paperless.config import AIConfig
|
from paperless.config import AIConfig
|
||||||
from paperless_ai.base_model import ClassificationSuggestions
|
from paperless_ai.base_model import ClassificationSuggestions
|
||||||
from paperless_ai.base_model import TaxonomyChoiceDict
|
from paperless_ai.base_model import TaxonomyChoiceDict
|
||||||
from paperless_ai.client import AIClient
|
from paperless_ai.client import AIClient
|
||||||
from paperless_ai.db import db_connection_released
|
from paperless_ai.db import db_connection_released
|
||||||
|
from paperless_ai.indexing import _node_document_ids
|
||||||
from paperless_ai.indexing import retrieve_similar_nodes
|
from paperless_ai.indexing import retrieve_similar_nodes
|
||||||
from paperless_ai.indexing import truncate_content
|
from paperless_ai.indexing import truncate_content
|
||||||
from paperless_ai.prompts.context import ClassificationPromptContext
|
from paperless_ai.prompts.context import ClassificationPromptContext
|
||||||
@@ -20,9 +19,7 @@ from paperless_ai.prompts.context import LocalizationPromptContext
|
|||||||
from paperless_ai.prompts.context import RagContextPromptContext
|
from paperless_ai.prompts.context import RagContextPromptContext
|
||||||
from paperless_ai.prompts.render import render_prompt
|
from paperless_ai.prompts.render import render_prompt
|
||||||
from paperless_ai.taxonomy import AssignedMetadata
|
from paperless_ai.taxonomy import AssignedMetadata
|
||||||
from paperless_ai.taxonomy import SimilarDocument
|
|
||||||
from paperless_ai.taxonomy import TaxonomyCandidates
|
from paperless_ai.taxonomy import TaxonomyCandidates
|
||||||
from paperless_ai.taxonomy import _node_document_weights
|
|
||||||
from paperless_ai.taxonomy import build_taxonomy_candidates
|
from paperless_ai.taxonomy import build_taxonomy_candidates
|
||||||
from paperless_ai.taxonomy import empty_taxonomy_candidates
|
from paperless_ai.taxonomy import empty_taxonomy_candidates
|
||||||
from paperless_ai.taxonomy import format_taxonomy_for_prompt
|
from paperless_ai.taxonomy import format_taxonomy_for_prompt
|
||||||
@@ -42,48 +39,6 @@ logger = logging.getLogger("paperless_ai.rag_classifier")
|
|||||||
TAXONOMY_CANDIDATE_TOP_K = 15
|
TAXONOMY_CANDIDATE_TOP_K = 15
|
||||||
|
|
||||||
|
|
||||||
def _fulltext_similar_documents(
|
|
||||||
document: Document,
|
|
||||||
user: User | None,
|
|
||||||
top_k: int,
|
|
||||||
) -> list[SimilarDocument]:
|
|
||||||
"""Rank-based fallback when no embedding backend is configured. Uses
|
|
||||||
Tantivy's "More Like This" (term-overlap similarity) instead of vector
|
|
||||||
similarity - cruder, but far better than no candidates at all.
|
|
||||||
more_like_this_ids returns only a ranked ID list, no scores, so weight is
|
|
||||||
synthesized from rank (descending from top_k) rather than claiming a
|
|
||||||
similarity magnitude that doesn't exist. An unrestricted user (none, or an
|
|
||||||
active superuser - see user_is_unrestricted) is normalized to ``None``
|
|
||||||
before calling, since the backend's permission filter has no superuser
|
|
||||||
short-circuit of its own. Results are re-checked with
|
|
||||||
restrict_queryset_to_visible() since Tantivy's indexed permission fields
|
|
||||||
lag the DB via async reindexing.
|
|
||||||
"""
|
|
||||||
from documents.search import get_backend
|
|
||||||
|
|
||||||
unrestricted = user_is_unrestricted(user)
|
|
||||||
search_user = None if unrestricted else user
|
|
||||||
backend = get_backend()
|
|
||||||
similar_ids = backend.more_like_this_ids(
|
|
||||||
document.pk,
|
|
||||||
user=search_user,
|
|
||||||
limit=top_k,
|
|
||||||
)
|
|
||||||
if not unrestricted:
|
|
||||||
allowed_ids = set(
|
|
||||||
restrict_queryset_to_visible(
|
|
||||||
Document.objects.filter(pk__in=similar_ids),
|
|
||||||
user,
|
|
||||||
"view_document",
|
|
||||||
).values_list("pk", flat=True),
|
|
||||||
)
|
|
||||||
similar_ids = [doc_id for doc_id in similar_ids if doc_id in allowed_ids]
|
|
||||||
return [
|
|
||||||
SimilarDocument(document_id=doc_id, weight=float(top_k - rank))
|
|
||||||
for rank, doc_id in enumerate(similar_ids)
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def get_language_name(language_code: str) -> str:
|
def get_language_name(language_code: str) -> str:
|
||||||
normalized_language_code = language_code.lower()
|
normalized_language_code = language_code.lower()
|
||||||
for code, name in settings.LANGUAGES:
|
for code, name in settings.LANGUAGES:
|
||||||
@@ -192,54 +147,45 @@ def get_taxonomy_context(
|
|||||||
user: User | None = None,
|
user: User | None = None,
|
||||||
max_docs: int = 5,
|
max_docs: int = 5,
|
||||||
) -> tuple[TaxonomyCandidates, AssignedMetadata, str]:
|
) -> tuple[TaxonomyCandidates, AssignedMetadata, str]:
|
||||||
"""One retrieval feeds both taxonomy candidates and RAG text context. Uses
|
"""One retrieval feeds both taxonomy candidates and RAG text context.
|
||||||
vector similarity when an embedding backend is configured, otherwise
|
On any retrieval failure, degrades to empty candidates/context rather than
|
||||||
falls back to Tantivy full-text "More Like This" similarity - see
|
propagating the exception - a vector-store outage should not block
|
||||||
_fulltext_similar_documents. On any retrieval failure, degrades to empty
|
classification, only its RAG-assisted enrichment.
|
||||||
candidates/context rather than propagating the exception - neither a
|
|
||||||
vector-store outage nor a search-index issue should block classification,
|
|
||||||
only its context-assisted enrichment.
|
|
||||||
"""
|
"""
|
||||||
assigned = get_assigned_metadata(document, user)
|
assigned = get_assigned_metadata(document, user)
|
||||||
ai_config = AIConfig()
|
|
||||||
try:
|
try:
|
||||||
if ai_config.llm_embedding_backend:
|
# None means "no restriction" to retrieve_similar_nodes. A superuser
|
||||||
# None means "no restriction" to retrieve_similar_nodes. A superuser
|
# (like no user at all) can see every document, so skip materializing
|
||||||
# (like no user at all) can see every document, so skip materializing
|
# every visible pk into a Python list and passing it through as an IN
|
||||||
# every visible pk into a Python list and passing it through as an IN
|
# filter: for a large library that is a wasted quadratic scan in the
|
||||||
# filter: for a large library that is a wasted quadratic scan in the
|
# vector store at best, and past ~32,763 documents a hard
|
||||||
# vector store at best, and past ~32,763 documents a hard
|
# sqlite3.OperationalError (SQLite's bound-parameter limit) at worst.
|
||||||
# sqlite3.OperationalError (SQLite's bound-parameter limit) at worst.
|
# get_objects_for_user_owner_aware() would return every Document for a
|
||||||
# permitted_object_ids() has its own superuser shortcut that would
|
# superuser anyway (guardian's own with_superuser shortcut), so this
|
||||||
# return every Document's id anyway, so this changes nothing about
|
# changes nothing about which documents are considered -- only how we
|
||||||
# which documents are considered -- only how we get there.
|
# get there.
|
||||||
visible_document_ids = (
|
visible_document_ids = (
|
||||||
None
|
None
|
||||||
if user is None or user.is_superuser
|
if user is None or user.is_superuser
|
||||||
else list(permitted_object_ids(user, Document, "view_document"))
|
else list(
|
||||||
)
|
get_objects_for_user_owner_aware(
|
||||||
nodes = retrieve_similar_nodes(
|
user,
|
||||||
document,
|
"view_document",
|
||||||
top_k=TAXONOMY_CANDIDATE_TOP_K,
|
Document,
|
||||||
document_ids=visible_document_ids,
|
).values_list("pk", flat=True),
|
||||||
)
|
|
||||||
similar_documents = _node_document_weights(nodes)
|
|
||||||
else:
|
|
||||||
# See _fulltext_similar_documents: it applies its own permission
|
|
||||||
# filter via `user`, so no visible-document-id list is needed here.
|
|
||||||
similar_documents = _fulltext_similar_documents(
|
|
||||||
document,
|
|
||||||
user,
|
|
||||||
top_k=TAXONOMY_CANDIDATE_TOP_K,
|
|
||||||
)
|
)
|
||||||
|
)
|
||||||
|
nodes = retrieve_similar_nodes(
|
||||||
|
document,
|
||||||
|
top_k=TAXONOMY_CANDIDATE_TOP_K,
|
||||||
|
document_ids=visible_document_ids,
|
||||||
|
)
|
||||||
|
|
||||||
candidates = build_taxonomy_candidates(similar_documents, user)
|
candidates = build_taxonomy_candidates(nodes, user)
|
||||||
|
|
||||||
similar_doc_ids = [s["document_id"] for s in similar_documents]
|
similar_docs = list(
|
||||||
docs_by_id = Document.objects.in_bulk(similar_doc_ids)
|
Document.objects.filter(pk__in=_node_document_ids(nodes))[:max_docs],
|
||||||
similar_docs = [
|
)
|
||||||
docs_by_id[doc_id] for doc_id in similar_doc_ids if doc_id in docs_by_id
|
|
||||||
][:max_docs]
|
|
||||||
context_blocks = []
|
context_blocks = []
|
||||||
for similar in similar_docs:
|
for similar in similar_docs:
|
||||||
text = similar.content[:1000] or ""
|
text = similar.content[:1000] or ""
|
||||||
@@ -247,8 +193,8 @@ def get_taxonomy_context(
|
|||||||
context_blocks.append(f"TITLE: {title}\n{text}")
|
context_blocks.append(f"TITLE: {title}\n{text}")
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.exception(
|
logger.exception(
|
||||||
"Failed to retrieve similar-document context for document %s; "
|
"Failed to retrieve RAG neighbours for document %s; continuing "
|
||||||
"continuing without taxonomy candidates or similar-document context.",
|
"without taxonomy candidates or similar-document context.",
|
||||||
document.pk,
|
document.pk,
|
||||||
)
|
)
|
||||||
return empty_taxonomy_candidates(), assigned, ""
|
return empty_taxonomy_candidates(), assigned, ""
|
||||||
@@ -331,14 +277,23 @@ def get_ai_document_classification(
|
|||||||
) -> ClassificationSuggestions:
|
) -> ClassificationSuggestions:
|
||||||
ai_config = AIConfig()
|
ai_config = AIConfig()
|
||||||
|
|
||||||
candidates, assigned, context = get_taxonomy_context(document, user)
|
if ai_config.llm_embedding_backend:
|
||||||
prompt = build_prompt_with_rag(
|
candidates, assigned, context = get_taxonomy_context(document, user)
|
||||||
document,
|
prompt = build_prompt_with_rag(
|
||||||
ai_config,
|
document,
|
||||||
candidates=candidates,
|
ai_config,
|
||||||
assigned=assigned,
|
candidates=candidates,
|
||||||
context=context,
|
assigned=assigned,
|
||||||
)
|
context=context,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
candidates = empty_taxonomy_candidates()
|
||||||
|
prompt = build_prompt_without_rag(
|
||||||
|
document,
|
||||||
|
ai_config,
|
||||||
|
candidates=candidates,
|
||||||
|
assigned=get_assigned_metadata(document, user),
|
||||||
|
)
|
||||||
|
|
||||||
client = AIClient()
|
client = AIClient()
|
||||||
# Hand the pooled DB connection back while the (slow) LLM query runs so it
|
# Hand the pooled DB connection back while the (slow) LLM query runs so it
|
||||||
|
|||||||
@@ -33,11 +33,6 @@ class TaxonomyCandidate(TypedDict):
|
|||||||
weight: float
|
weight: float
|
||||||
|
|
||||||
|
|
||||||
class SimilarDocument(TypedDict):
|
|
||||||
document_id: int
|
|
||||||
weight: float
|
|
||||||
|
|
||||||
|
|
||||||
class TaxonomyCandidates(TypedDict):
|
class TaxonomyCandidates(TypedDict):
|
||||||
tags: list[TaxonomyCandidate]
|
tags: list[TaxonomyCandidate]
|
||||||
document_types: list[TaxonomyCandidate]
|
document_types: list[TaxonomyCandidate]
|
||||||
@@ -110,10 +105,10 @@ def get_assigned_metadata(document: Document, user: User | None) -> AssignedMeta
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _node_document_weights(nodes: list["NodeWithScore"]) -> list[SimilarDocument]:
|
def _node_document_weights(nodes: list["NodeWithScore"]) -> dict[int, float]:
|
||||||
"""Sum each node's similarity score into its document_id (a document can
|
"""document_id -> that node's similarity score, summed if a document_id
|
||||||
appear via multiple chunks/nodes) and return one SimilarDocument per
|
appears more than once across the retrieved nodes (e.g. multiple chunks
|
||||||
distinct document_id."""
|
of the same source document)."""
|
||||||
weights: dict[int, float] = defaultdict(float)
|
weights: dict[int, float] = defaultdict(float)
|
||||||
for node in nodes:
|
for node in nodes:
|
||||||
document_id = node.metadata.get("document_id")
|
document_id = node.metadata.get("document_id")
|
||||||
@@ -126,10 +121,7 @@ def _node_document_weights(nodes: list["NodeWithScore"]) -> list[SimilarDocument
|
|||||||
weights[int(document_id)] += float(node.score or 0.0)
|
weights[int(document_id)] += float(node.score or 0.0)
|
||||||
except (TypeError, ValueError): # pragma: no cover
|
except (TypeError, ValueError): # pragma: no cover
|
||||||
continue
|
continue
|
||||||
return [
|
return weights
|
||||||
SimilarDocument(document_id=document_id, weight=weight)
|
|
||||||
for document_id, weight in weights.items()
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def _visible_ranked_candidates(
|
def _visible_ranked_candidates(
|
||||||
@@ -165,25 +157,20 @@ def _visible_ranked_candidates(
|
|||||||
|
|
||||||
|
|
||||||
def build_taxonomy_candidates(
|
def build_taxonomy_candidates(
|
||||||
similar_documents: list[SimilarDocument],
|
nodes: list["NodeWithScore"],
|
||||||
user: User | None,
|
user: User | None,
|
||||||
) -> TaxonomyCandidates:
|
) -> TaxonomyCandidates:
|
||||||
"""Resolve each similar document's id to a live Document, read its
|
"""Resolve each neighbour node's document_id to a live Document, read its
|
||||||
*current* tags/type/correspondent/storage_path via the ORM (never any
|
*current* tags/type/correspondent/storage_path via the ORM (never the
|
||||||
possibly-stale names an adapter's source might have cached), weight each
|
possibly-stale names cached in vector-index node metadata), weight each
|
||||||
distinct taxonomy object by aggregate similarity weight, permission-filter
|
distinct taxonomy object by aggregate neighbour similarity, permission-filter
|
||||||
against what ``user`` can see, and return each category ranked by weight
|
against what ``user`` can see, and return each category ranked by weight
|
||||||
and capped. ``similar_documents`` may come from either the vector-RAG
|
and capped.
|
||||||
adapter or the full-text fallback adapter - both produce this same shape.
|
|
||||||
"""
|
"""
|
||||||
if not similar_documents:
|
|
||||||
return empty_taxonomy_candidates()
|
|
||||||
|
|
||||||
# Both adapters guarantee at most one SimilarDocument per document_id, so
|
document_weights = _node_document_weights(nodes)
|
||||||
# this never silently drops a duplicate's weight.
|
if not document_weights:
|
||||||
document_weights: dict[int, float] = {
|
return empty_taxonomy_candidates()
|
||||||
s["document_id"]: s["weight"] for s in similar_documents
|
|
||||||
}
|
|
||||||
|
|
||||||
# Only .tags.all() needs prefetching (a reverse M2M, one extra query for
|
# Only .tags.all() needs prefetching (a reverse M2M, one extra query for
|
||||||
# the whole batch). document_type/correspondent/storage_path are read
|
# the whole batch). document_type/correspondent/storage_path are read
|
||||||
|
|||||||
@@ -1,4 +1,3 @@
|
|||||||
from collections.abc import Generator
|
|
||||||
from types import SimpleNamespace
|
from types import SimpleNamespace
|
||||||
from unittest.mock import MagicMock
|
from unittest.mock import MagicMock
|
||||||
from unittest.mock import patch
|
from unittest.mock import patch
|
||||||
@@ -8,13 +7,10 @@ import pytest_mock
|
|||||||
from django.test import override_settings
|
from django.test import override_settings
|
||||||
|
|
||||||
from documents.models import Document
|
from documents.models import Document
|
||||||
from documents.search import TantivyBackend
|
|
||||||
from documents.tests.factories import DocumentFactory
|
from documents.tests.factories import DocumentFactory
|
||||||
from documents.tests.factories import TagFactory
|
from documents.tests.factories import TagFactory
|
||||||
from documents.tests.factories import UserFactory
|
from documents.tests.factories import UserFactory
|
||||||
from paperless.config import AIConfig
|
from paperless.config import AIConfig
|
||||||
from paperless_ai.ai_classifier import TAXONOMY_CANDIDATE_TOP_K
|
|
||||||
from paperless_ai.ai_classifier import _fulltext_similar_documents
|
|
||||||
from paperless_ai.ai_classifier import _restrict_to_shown_candidates
|
from paperless_ai.ai_classifier import _restrict_to_shown_candidates
|
||||||
from paperless_ai.ai_classifier import build_localization_prompt
|
from paperless_ai.ai_classifier import build_localization_prompt
|
||||||
from paperless_ai.ai_classifier import build_prompt_with_rag
|
from paperless_ai.ai_classifier import build_prompt_with_rag
|
||||||
@@ -24,7 +20,6 @@ from paperless_ai.ai_classifier import get_language_name
|
|||||||
from paperless_ai.ai_classifier import get_taxonomy_context
|
from paperless_ai.ai_classifier import get_taxonomy_context
|
||||||
from paperless_ai.base_model import ClassificationSuggestions
|
from paperless_ai.base_model import ClassificationSuggestions
|
||||||
from paperless_ai.base_model import TaxonomyChoiceDict
|
from paperless_ai.base_model import TaxonomyChoiceDict
|
||||||
from paperless_ai.taxonomy import SimilarDocument
|
|
||||||
from paperless_ai.taxonomy import TaxonomyCandidate
|
from paperless_ai.taxonomy import TaxonomyCandidate
|
||||||
from paperless_ai.taxonomy import TaxonomyCandidates
|
from paperless_ai.taxonomy import TaxonomyCandidates
|
||||||
from paperless_ai.taxonomy import empty_taxonomy_candidates
|
from paperless_ai.taxonomy import empty_taxonomy_candidates
|
||||||
@@ -209,10 +204,12 @@ def test_use_rag_if_configured(
|
|||||||
|
|
||||||
@pytest.mark.django_db
|
@pytest.mark.django_db
|
||||||
@patch("paperless_ai.client.AIClient.run_llm_query")
|
@patch("paperless_ai.client.AIClient.run_llm_query")
|
||||||
@patch("paperless_ai.ai_classifier.build_prompt_with_rag")
|
@patch("paperless_ai.ai_classifier.build_prompt_without_rag")
|
||||||
|
@patch("paperless_ai.ai_classifier.AIConfig")
|
||||||
@override_settings(LLM_BACKEND="ollama", LLM_MODEL="some_model")
|
@override_settings(LLM_BACKEND="ollama", LLM_MODEL="some_model")
|
||||||
def test_use_rag_prompt_even_without_embedding_backend(
|
def test_use_without_rag_if_not_configured(
|
||||||
mock_build_prompt_with_rag,
|
mock_ai_config,
|
||||||
|
mock_build_prompt_without_rag,
|
||||||
mock_run_llm_query,
|
mock_run_llm_query,
|
||||||
mock_document,
|
mock_document,
|
||||||
):
|
):
|
||||||
@@ -222,13 +219,13 @@ def test_use_rag_prompt_even_without_embedding_backend(
|
|||||||
WHEN:
|
WHEN:
|
||||||
- get_ai_document_classification() is called
|
- get_ai_document_classification() is called
|
||||||
THEN:
|
THEN:
|
||||||
- The RAG-context prompt builder is still used (fed by the full-text
|
- The non-RAG prompt builder is used
|
||||||
fallback's context/candidates instead of the vector store's)
|
|
||||||
"""
|
"""
|
||||||
mock_build_prompt_with_rag.return_value = "Prompt with RAG"
|
mock_ai_config.return_value.llm_embedding_backend = None
|
||||||
|
mock_build_prompt_without_rag.return_value = "Prompt without RAG"
|
||||||
mock_run_llm_query.return_value = NESTED_SUGGESTIONS
|
mock_run_llm_query.return_value = NESTED_SUGGESTIONS
|
||||||
get_ai_document_classification(mock_document)
|
get_ai_document_classification(mock_document)
|
||||||
mock_build_prompt_with_rag.assert_called_once()
|
mock_build_prompt_without_rag.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.django_db
|
@pytest.mark.django_db
|
||||||
@@ -306,7 +303,6 @@ def test_build_localization_prompt_preserves_unicode_characters():
|
|||||||
|
|
||||||
|
|
||||||
@pytest.mark.django_db
|
@pytest.mark.django_db
|
||||||
@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
|
|
||||||
def test_get_taxonomy_context_assembles_rag_text_and_candidates():
|
def test_get_taxonomy_context_assembles_rag_text_and_candidates():
|
||||||
"""
|
"""
|
||||||
GIVEN:
|
GIVEN:
|
||||||
@@ -348,7 +344,6 @@ def test_get_taxonomy_context_assembles_rag_text_and_candidates():
|
|||||||
|
|
||||||
|
|
||||||
@pytest.mark.django_db
|
@pytest.mark.django_db
|
||||||
@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
|
|
||||||
def test_get_taxonomy_context_no_similar_docs():
|
def test_get_taxonomy_context_no_similar_docs():
|
||||||
"""
|
"""
|
||||||
GIVEN:
|
GIVEN:
|
||||||
@@ -372,67 +367,6 @@ def test_get_taxonomy_context_no_similar_docs():
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.django_db
|
|
||||||
def test_get_taxonomy_context_uses_fulltext_fallback_when_no_embedding_backend(
|
|
||||||
mocker: pytest_mock.MockerFixture,
|
|
||||||
) -> None:
|
|
||||||
"""
|
|
||||||
GIVEN:
|
|
||||||
- No LLM embedding backend is configured (the default test settings)
|
|
||||||
WHEN:
|
|
||||||
- get_taxonomy_context() is called
|
|
||||||
THEN:
|
|
||||||
- _fulltext_similar_documents() is called with the document, the user
|
|
||||||
and TAXONOMY_CANDIDATE_TOP_K
|
|
||||||
- retrieve_similar_nodes() (the vector path) is never called
|
|
||||||
"""
|
|
||||||
document = DocumentFactory.create(content="Some content")
|
|
||||||
mock_fulltext = mocker.patch(
|
|
||||||
"paperless_ai.ai_classifier._fulltext_similar_documents",
|
|
||||||
return_value=[],
|
|
||||||
)
|
|
||||||
mock_retrieve = mocker.patch("paperless_ai.ai_classifier.retrieve_similar_nodes")
|
|
||||||
|
|
||||||
get_taxonomy_context(document, user=None)
|
|
||||||
|
|
||||||
mock_fulltext.assert_called_once_with(
|
|
||||||
document,
|
|
||||||
None,
|
|
||||||
top_k=TAXONOMY_CANDIDATE_TOP_K,
|
|
||||||
)
|
|
||||||
mock_retrieve.assert_not_called()
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.django_db
|
|
||||||
@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
|
|
||||||
def test_get_taxonomy_context_uses_vector_path_when_embedding_backend_configured(
|
|
||||||
mocker: pytest_mock.MockerFixture,
|
|
||||||
) -> None:
|
|
||||||
"""
|
|
||||||
GIVEN:
|
|
||||||
- An LLM embedding backend is configured
|
|
||||||
WHEN:
|
|
||||||
- get_taxonomy_context() is called
|
|
||||||
THEN:
|
|
||||||
- retrieve_similar_nodes() (the vector path) is called
|
|
||||||
- _fulltext_similar_documents() (the no-embedding-backend fallback)
|
|
||||||
is never called
|
|
||||||
"""
|
|
||||||
document = DocumentFactory.create(content="Some content")
|
|
||||||
mock_retrieve = mocker.patch(
|
|
||||||
"paperless_ai.ai_classifier.retrieve_similar_nodes",
|
|
||||||
return_value=[],
|
|
||||||
)
|
|
||||||
mock_fulltext = mocker.patch(
|
|
||||||
"paperless_ai.ai_classifier._fulltext_similar_documents",
|
|
||||||
)
|
|
||||||
|
|
||||||
get_taxonomy_context(document, user=None)
|
|
||||||
|
|
||||||
mock_retrieve.assert_called_once()
|
|
||||||
mock_fulltext.assert_not_called()
|
|
||||||
|
|
||||||
|
|
||||||
class TestGetTaxonomyContextVisibility:
|
class TestGetTaxonomyContextVisibility:
|
||||||
"""get_taxonomy_context must not materialize every visible document id
|
"""get_taxonomy_context must not materialize every visible document id
|
||||||
for a user who can already see the whole library: a superuser (like no
|
for a user who can already see the whole library: a superuser (like no
|
||||||
@@ -445,7 +379,6 @@ class TestGetTaxonomyContextVisibility:
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
@pytest.mark.django_db
|
@pytest.mark.django_db
|
||||||
@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
|
|
||||||
def test_skips_permission_lookup_for_superuser(
|
def test_skips_permission_lookup_for_superuser(
|
||||||
self,
|
self,
|
||||||
mocker: pytest_mock.MockerFixture,
|
mocker: pytest_mock.MockerFixture,
|
||||||
@@ -464,18 +397,17 @@ class TestGetTaxonomyContextVisibility:
|
|||||||
"paperless_ai.ai_classifier.retrieve_similar_nodes",
|
"paperless_ai.ai_classifier.retrieve_similar_nodes",
|
||||||
return_value=[],
|
return_value=[],
|
||||||
)
|
)
|
||||||
mock_permitted = mocker.patch(
|
mock_get_objects = mocker.patch(
|
||||||
"paperless_ai.ai_classifier.permitted_object_ids",
|
"paperless_ai.ai_classifier.get_objects_for_user_owner_aware",
|
||||||
)
|
)
|
||||||
user = UserFactory.create(is_superuser=True)
|
user = UserFactory.create(is_superuser=True)
|
||||||
|
|
||||||
get_taxonomy_context(document, user)
|
get_taxonomy_context(document, user)
|
||||||
|
|
||||||
mock_permitted.assert_not_called()
|
mock_get_objects.assert_not_called()
|
||||||
assert mock_retrieve.call_args.kwargs["document_ids"] is None
|
assert mock_retrieve.call_args.kwargs["document_ids"] is None
|
||||||
|
|
||||||
@pytest.mark.django_db
|
@pytest.mark.django_db
|
||||||
@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
|
|
||||||
def test_skips_permission_lookup_when_no_user(
|
def test_skips_permission_lookup_when_no_user(
|
||||||
self,
|
self,
|
||||||
mocker: pytest_mock.MockerFixture,
|
mocker: pytest_mock.MockerFixture,
|
||||||
@@ -494,17 +426,16 @@ class TestGetTaxonomyContextVisibility:
|
|||||||
"paperless_ai.ai_classifier.retrieve_similar_nodes",
|
"paperless_ai.ai_classifier.retrieve_similar_nodes",
|
||||||
return_value=[],
|
return_value=[],
|
||||||
)
|
)
|
||||||
mock_permitted = mocker.patch(
|
mock_get_objects = mocker.patch(
|
||||||
"paperless_ai.ai_classifier.permitted_object_ids",
|
"paperless_ai.ai_classifier.get_objects_for_user_owner_aware",
|
||||||
)
|
)
|
||||||
|
|
||||||
get_taxonomy_context(document, None)
|
get_taxonomy_context(document, None)
|
||||||
|
|
||||||
mock_permitted.assert_not_called()
|
mock_get_objects.assert_not_called()
|
||||||
assert mock_retrieve.call_args.kwargs["document_ids"] is None
|
assert mock_retrieve.call_args.kwargs["document_ids"] is None
|
||||||
|
|
||||||
@pytest.mark.django_db
|
@pytest.mark.django_db
|
||||||
@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
|
|
||||||
def test_restricts_to_visible_documents_for_non_superuser(
|
def test_restricts_to_visible_documents_for_non_superuser(
|
||||||
self,
|
self,
|
||||||
mocker: pytest_mock.MockerFixture,
|
mocker: pytest_mock.MockerFixture,
|
||||||
@@ -515,7 +446,7 @@ class TestGetTaxonomyContextVisibility:
|
|||||||
WHEN:
|
WHEN:
|
||||||
- get_taxonomy_context() is called
|
- get_taxonomy_context() is called
|
||||||
THEN:
|
THEN:
|
||||||
- The user's permitted document ids are looked up and passed to
|
- The user's visible document ids are looked up and passed to
|
||||||
retrieve_similar_nodes() as a restriction
|
retrieve_similar_nodes() as a restriction
|
||||||
"""
|
"""
|
||||||
document = DocumentFactory.create(content="Some content")
|
document = DocumentFactory.create(content="Some content")
|
||||||
@@ -523,186 +454,21 @@ class TestGetTaxonomyContextVisibility:
|
|||||||
"paperless_ai.ai_classifier.retrieve_similar_nodes",
|
"paperless_ai.ai_classifier.retrieve_similar_nodes",
|
||||||
return_value=[],
|
return_value=[],
|
||||||
)
|
)
|
||||||
mock_permitted = mocker.patch(
|
mock_queryset = mocker.MagicMock()
|
||||||
"paperless_ai.ai_classifier.permitted_object_ids",
|
mock_queryset.values_list.return_value = [1, 2, 3]
|
||||||
return_value=[1, 2, 3],
|
mock_get_objects = mocker.patch(
|
||||||
|
"paperless_ai.ai_classifier.get_objects_for_user_owner_aware",
|
||||||
|
return_value=mock_queryset,
|
||||||
)
|
)
|
||||||
user = UserFactory.create(is_superuser=False)
|
user = UserFactory.create(is_superuser=False)
|
||||||
|
|
||||||
get_taxonomy_context(document, user)
|
get_taxonomy_context(document, user)
|
||||||
|
|
||||||
mock_permitted.assert_called_once_with(user, Document, "view_document")
|
mock_get_objects.assert_called_once_with(user, "view_document", Document)
|
||||||
assert mock_retrieve.call_args.kwargs["document_ids"] == [1, 2, 3]
|
assert mock_retrieve.call_args.kwargs["document_ids"] == [1, 2, 3]
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.django_db
|
@pytest.mark.django_db
|
||||||
class TestFulltextSimilarDocuments:
|
|
||||||
"""_fulltext_similar_documents is the no-embedding-backend fallback: it
|
|
||||||
asks the Tantivy full-text index for "More Like This" neighbours instead
|
|
||||||
of the vector store, and synthesizes a rank-based weight since Tantivy's
|
|
||||||
more_like_this_ids returns only an ordered id list, no scores.
|
|
||||||
"""
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def fulltext_backend(
|
|
||||||
self,
|
|
||||||
mocker: pytest_mock.MockerFixture,
|
|
||||||
) -> Generator[TantivyBackend, None, None]:
|
|
||||||
"""An in-memory Tantivy backend, wired up as the module-level
|
|
||||||
singleton _fulltext_similar_documents resolves via get_backend()."""
|
|
||||||
backend = TantivyBackend(path=None)
|
|
||||||
backend.open()
|
|
||||||
mocker.patch("documents.search.get_backend", return_value=backend)
|
|
||||||
try:
|
|
||||||
yield backend
|
|
||||||
finally:
|
|
||||||
backend.close()
|
|
||||||
|
|
||||||
def test_ranks_by_rank_based_weight_descending(
|
|
||||||
self,
|
|
||||||
fulltext_backend: TantivyBackend,
|
|
||||||
) -> None:
|
|
||||||
"""
|
|
||||||
GIVEN:
|
|
||||||
- A source document and two similar documents indexed in Tantivy
|
|
||||||
WHEN:
|
|
||||||
- _fulltext_similar_documents() is called
|
|
||||||
THEN:
|
|
||||||
- Each result's weight reflects its rank (first result weighted
|
|
||||||
higher than the second), not a raw similarity score
|
|
||||||
"""
|
|
||||||
source = DocumentFactory.create(content="quarterly financial report details")
|
|
||||||
first = DocumentFactory.create(content="quarterly financial report details")
|
|
||||||
second = DocumentFactory.create(content="financial report")
|
|
||||||
for doc in (source, first, second):
|
|
||||||
fulltext_backend.add_or_update(doc)
|
|
||||||
|
|
||||||
result = _fulltext_similar_documents(source, user=None, top_k=5)
|
|
||||||
|
|
||||||
assert len(result) == 2
|
|
||||||
weight_by_id = {s["document_id"]: s["weight"] for s in result}
|
|
||||||
assert weight_by_id[first.pk] > weight_by_id[second.pk]
|
|
||||||
|
|
||||||
def test_excludes_source_document(
|
|
||||||
self,
|
|
||||||
fulltext_backend: TantivyBackend,
|
|
||||||
) -> None:
|
|
||||||
"""
|
|
||||||
GIVEN:
|
|
||||||
- A source document indexed in Tantivy with no other documents
|
|
||||||
WHEN:
|
|
||||||
- _fulltext_similar_documents() is called
|
|
||||||
THEN:
|
|
||||||
- An empty list is returned - the source document is never its
|
|
||||||
own similar document
|
|
||||||
"""
|
|
||||||
source = DocumentFactory.create(content="unique unrelated content")
|
|
||||||
fulltext_backend.add_or_update(source)
|
|
||||||
|
|
||||||
result = _fulltext_similar_documents(source, user=None, top_k=5)
|
|
||||||
|
|
||||||
assert result == []
|
|
||||||
|
|
||||||
def test_empty_index_returns_empty_list(
|
|
||||||
self,
|
|
||||||
fulltext_backend: TantivyBackend,
|
|
||||||
) -> None:
|
|
||||||
"""
|
|
||||||
GIVEN:
|
|
||||||
- A document that has never been indexed (fresh/empty Tantivy index)
|
|
||||||
WHEN:
|
|
||||||
- _fulltext_similar_documents() is called
|
|
||||||
THEN:
|
|
||||||
- An empty list is returned rather than raising
|
|
||||||
"""
|
|
||||||
source = DocumentFactory.create(content="never indexed")
|
|
||||||
|
|
||||||
result = _fulltext_similar_documents(source, user=None, top_k=5)
|
|
||||||
|
|
||||||
assert result == []
|
|
||||||
|
|
||||||
def test_respects_top_k_limit(
|
|
||||||
self,
|
|
||||||
fulltext_backend: TantivyBackend,
|
|
||||||
) -> None:
|
|
||||||
"""
|
|
||||||
GIVEN:
|
|
||||||
- A source document and four similar documents indexed
|
|
||||||
WHEN:
|
|
||||||
- _fulltext_similar_documents() is called with top_k=2
|
|
||||||
THEN:
|
|
||||||
- At most 2 results are returned
|
|
||||||
"""
|
|
||||||
source = DocumentFactory.create(content="shared overlapping keyword text")
|
|
||||||
fulltext_backend.add_or_update(source)
|
|
||||||
for _ in range(4):
|
|
||||||
fulltext_backend.add_or_update(
|
|
||||||
DocumentFactory.create(content="shared overlapping keyword text"),
|
|
||||||
)
|
|
||||||
|
|
||||||
result = _fulltext_similar_documents(source, user=None, top_k=2)
|
|
||||||
|
|
||||||
assert len(result) == 2
|
|
||||||
|
|
||||||
def test_result_shape_is_similar_document(
|
|
||||||
self,
|
|
||||||
fulltext_backend: TantivyBackend,
|
|
||||||
) -> None:
|
|
||||||
"""
|
|
||||||
GIVEN:
|
|
||||||
- A source document and one similar document indexed
|
|
||||||
WHEN:
|
|
||||||
- _fulltext_similar_documents() is called
|
|
||||||
THEN:
|
|
||||||
- Each result is a SimilarDocument (document_id + weight only)
|
|
||||||
"""
|
|
||||||
source = DocumentFactory.create(content="shared content phrase")
|
|
||||||
other = DocumentFactory.create(content="shared content phrase")
|
|
||||||
fulltext_backend.add_or_update(source)
|
|
||||||
fulltext_backend.add_or_update(other)
|
|
||||||
|
|
||||||
result = _fulltext_similar_documents(source, user=None, top_k=5)
|
|
||||||
|
|
||||||
# rank 0 (the only/best result) with top_k=5 -> weight = top_k - rank = 5.0,
|
|
||||||
# per the "first result gets top_k, the last gets 1" formula.
|
|
||||||
assert result == [SimilarDocument(document_id=other.pk, weight=5.0)]
|
|
||||||
|
|
||||||
def test_superuser_sees_other_users_documents(
|
|
||||||
self,
|
|
||||||
fulltext_backend: TantivyBackend,
|
|
||||||
) -> None:
|
|
||||||
"""
|
|
||||||
GIVEN:
|
|
||||||
- A source document owned by one user and a similar document
|
|
||||||
owned by a different user, with no sharing between them
|
|
||||||
WHEN:
|
|
||||||
- _fulltext_similar_documents() is called with a superuser
|
|
||||||
THEN:
|
|
||||||
- The other user's document is still returned as a similar
|
|
||||||
document - a superuser must not be narrowed by the backend's
|
|
||||||
owner-based permission filter
|
|
||||||
"""
|
|
||||||
owner = UserFactory.create()
|
|
||||||
other_owner = UserFactory.create()
|
|
||||||
superuser = UserFactory.create(is_superuser=True)
|
|
||||||
source = DocumentFactory.create(
|
|
||||||
content="shared content phrase",
|
|
||||||
owner=owner,
|
|
||||||
)
|
|
||||||
other = DocumentFactory.create(
|
|
||||||
content="shared content phrase",
|
|
||||||
owner=other_owner,
|
|
||||||
)
|
|
||||||
fulltext_backend.add_or_update(source)
|
|
||||||
fulltext_backend.add_or_update(other)
|
|
||||||
|
|
||||||
result = _fulltext_similar_documents(source, user=superuser, top_k=5)
|
|
||||||
|
|
||||||
assert [s["document_id"] for s in result] == [other.pk]
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.django_db
|
|
||||||
@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
|
|
||||||
@patch("paperless_ai.ai_classifier.retrieve_similar_nodes")
|
@patch("paperless_ai.ai_classifier.retrieve_similar_nodes")
|
||||||
def test_get_taxonomy_context_retrieval_failure_degrades_to_no_hints(mock_retrieve):
|
def test_get_taxonomy_context_retrieval_failure_degrades_to_no_hints(mock_retrieve):
|
||||||
"""
|
"""
|
||||||
@@ -729,7 +495,6 @@ def test_get_taxonomy_context_retrieval_failure_degrades_to_no_hints(mock_retrie
|
|||||||
|
|
||||||
|
|
||||||
@pytest.mark.django_db
|
@pytest.mark.django_db
|
||||||
@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
|
|
||||||
@patch("paperless_ai.ai_classifier.build_taxonomy_candidates")
|
@patch("paperless_ai.ai_classifier.build_taxonomy_candidates")
|
||||||
@patch("paperless_ai.ai_classifier.retrieve_similar_nodes")
|
@patch("paperless_ai.ai_classifier.retrieve_similar_nodes")
|
||||||
def test_get_taxonomy_context_candidate_building_failure_degrades_to_no_hints(
|
def test_get_taxonomy_context_candidate_building_failure_degrades_to_no_hints(
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import json
|
import json
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
import pytest_mock
|
import pytest_mock
|
||||||
@@ -10,7 +11,6 @@ from documents.tests.factories import StoragePathFactory
|
|||||||
from documents.tests.factories import TagFactory
|
from documents.tests.factories import TagFactory
|
||||||
from documents.tests.factories import UserFactory
|
from documents.tests.factories import UserFactory
|
||||||
from paperless_ai.taxonomy import AssignedMetadata
|
from paperless_ai.taxonomy import AssignedMetadata
|
||||||
from paperless_ai.taxonomy import SimilarDocument
|
|
||||||
from paperless_ai.taxonomy import TaxonomyCandidates
|
from paperless_ai.taxonomy import TaxonomyCandidates
|
||||||
from paperless_ai.taxonomy import build_taxonomy_candidates
|
from paperless_ai.taxonomy import build_taxonomy_candidates
|
||||||
from paperless_ai.taxonomy import format_taxonomy_for_prompt
|
from paperless_ai.taxonomy import format_taxonomy_for_prompt
|
||||||
@@ -132,8 +132,9 @@ class TestGetAssignedMetadata:
|
|||||||
assert result["tags"] == ["Owned By Someone Else"]
|
assert result["tags"] == ["Owned By Someone Else"]
|
||||||
|
|
||||||
|
|
||||||
def make_similar(document_id: int, weight: float) -> SimilarDocument:
|
def make_node(document_id: int, score: float) -> SimpleNamespace:
|
||||||
return SimilarDocument(document_id=document_id, weight=weight)
|
"""A stand-in for NodeWithScore: only ``.metadata``/``.score`` are read."""
|
||||||
|
return SimpleNamespace(metadata={"document_id": str(document_id)}, score=score)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.django_db
|
@pytest.mark.django_db
|
||||||
@@ -169,9 +170,9 @@ class TestBuildTaxonomyCandidates:
|
|||||||
doc_a.tags.add(tag)
|
doc_a.tags.add(tag)
|
||||||
doc_b = DocumentFactory.create()
|
doc_b = DocumentFactory.create()
|
||||||
doc_b.tags.add(tag)
|
doc_b.tags.add(tag)
|
||||||
similar_documents = [make_similar(doc_a.pk, 0.9), make_similar(doc_b.pk, 0.4)]
|
nodes = [make_node(doc_a.pk, 0.9), make_node(doc_b.pk, 0.4)]
|
||||||
|
|
||||||
result = build_taxonomy_candidates(similar_documents, user=None)
|
result = build_taxonomy_candidates(nodes, user=None)
|
||||||
|
|
||||||
assert len(result["tags"]) == 1
|
assert len(result["tags"]) == 1
|
||||||
assert result["tags"][0]["id"] == tag.pk
|
assert result["tags"][0]["id"] == tag.pk
|
||||||
@@ -196,9 +197,9 @@ class TestBuildTaxonomyCandidates:
|
|||||||
document.tags.add(tag)
|
document.tags.add(tag)
|
||||||
tag.name = "New Name"
|
tag.name = "New Name"
|
||||||
tag.save()
|
tag.save()
|
||||||
similar_documents = [make_similar(document.pk, 0.5)]
|
nodes = [make_node(document.pk, 0.5)]
|
||||||
|
|
||||||
result = build_taxonomy_candidates(similar_documents, user=None)
|
result = build_taxonomy_candidates(nodes, user=None)
|
||||||
|
|
||||||
assert result["tags"][0]["name"] == "New Name"
|
assert result["tags"][0]["name"] == "New Name"
|
||||||
|
|
||||||
@@ -218,9 +219,9 @@ class TestBuildTaxonomyCandidates:
|
|||||||
document = DocumentFactory.create()
|
document = DocumentFactory.create()
|
||||||
document.tags.add(tag)
|
document.tags.add(tag)
|
||||||
tag.delete()
|
tag.delete()
|
||||||
similar_documents = [make_similar(document.pk, 0.5)]
|
nodes = [make_node(document.pk, 0.5)]
|
||||||
|
|
||||||
result = build_taxonomy_candidates(similar_documents, user=None)
|
result = build_taxonomy_candidates(nodes, user=None)
|
||||||
|
|
||||||
assert result["tags"] == []
|
assert result["tags"] == []
|
||||||
|
|
||||||
@@ -239,12 +240,9 @@ class TestBuildTaxonomyCandidates:
|
|||||||
strong_doc.tags.add(strong_tag)
|
strong_doc.tags.add(strong_tag)
|
||||||
weak_doc = DocumentFactory.create()
|
weak_doc = DocumentFactory.create()
|
||||||
weak_doc.tags.add(weak_tag)
|
weak_doc.tags.add(weak_tag)
|
||||||
similar_documents = [
|
nodes = [make_node(strong_doc.pk, 0.9), make_node(weak_doc.pk, 0.1)]
|
||||||
make_similar(strong_doc.pk, 0.9),
|
|
||||||
make_similar(weak_doc.pk, 0.1),
|
|
||||||
]
|
|
||||||
|
|
||||||
result = build_taxonomy_candidates(similar_documents, user=None)
|
result = build_taxonomy_candidates(nodes, user=None)
|
||||||
|
|
||||||
assert [c["name"] for c in result["tags"]] == ["Strong", "Weak"]
|
assert [c["name"] for c in result["tags"]] == ["Strong", "Weak"]
|
||||||
|
|
||||||
@@ -260,9 +258,9 @@ class TestBuildTaxonomyCandidates:
|
|||||||
document = DocumentFactory.create()
|
document = DocumentFactory.create()
|
||||||
for i in range(15):
|
for i in range(15):
|
||||||
document.tags.add(TagFactory.create(name=f"Tag{i}"))
|
document.tags.add(TagFactory.create(name=f"Tag{i}"))
|
||||||
similar_documents = [make_similar(document.pk, 0.5)]
|
nodes = [make_node(document.pk, 0.5)]
|
||||||
|
|
||||||
result = build_taxonomy_candidates(similar_documents, user=None)
|
result = build_taxonomy_candidates(nodes, user=None)
|
||||||
|
|
||||||
assert len(result["tags"]) == 10
|
assert len(result["tags"]) == 10
|
||||||
|
|
||||||
@@ -276,12 +274,12 @@ class TestBuildTaxonomyCandidates:
|
|||||||
- Only 5 correspondents are returned
|
- Only 5 correspondents are returned
|
||||||
"""
|
"""
|
||||||
correspondents = CorrespondentFactory.create_batch(7)
|
correspondents = CorrespondentFactory.create_batch(7)
|
||||||
similar_documents = [
|
nodes = [
|
||||||
make_similar(DocumentFactory.create(correspondent=c).pk, 0.5)
|
make_node(DocumentFactory.create(correspondent=c).pk, 0.5)
|
||||||
for c in correspondents
|
for c in correspondents
|
||||||
]
|
]
|
||||||
|
|
||||||
result = build_taxonomy_candidates(similar_documents, user=None)
|
result = build_taxonomy_candidates(nodes, user=None)
|
||||||
|
|
||||||
assert len(result["correspondents"]) == 5
|
assert len(result["correspondents"]) == 5
|
||||||
|
|
||||||
@@ -296,9 +294,9 @@ class TestBuildTaxonomyCandidates:
|
|||||||
"""
|
"""
|
||||||
document_type = DocumentTypeFactory.create(name="Invoice")
|
document_type = DocumentTypeFactory.create(name="Invoice")
|
||||||
document = DocumentFactory.create(document_type=document_type)
|
document = DocumentFactory.create(document_type=document_type)
|
||||||
similar_documents = [make_similar(document.pk, 0.5)]
|
nodes = [make_node(document.pk, 0.5)]
|
||||||
|
|
||||||
result = build_taxonomy_candidates(similar_documents, user=None)
|
result = build_taxonomy_candidates(nodes, user=None)
|
||||||
|
|
||||||
assert len(result["document_types"]) == 1
|
assert len(result["document_types"]) == 1
|
||||||
assert result["document_types"][0]["id"] == document_type.pk
|
assert result["document_types"][0]["id"] == document_type.pk
|
||||||
@@ -314,12 +312,12 @@ class TestBuildTaxonomyCandidates:
|
|||||||
- Only 5 document_types are returned
|
- Only 5 document_types are returned
|
||||||
"""
|
"""
|
||||||
document_types = DocumentTypeFactory.create_batch(7)
|
document_types = DocumentTypeFactory.create_batch(7)
|
||||||
similar_documents = [
|
nodes = [
|
||||||
make_similar(DocumentFactory.create(document_type=dt).pk, 0.5)
|
make_node(DocumentFactory.create(document_type=dt).pk, 0.5)
|
||||||
for dt in document_types
|
for dt in document_types
|
||||||
]
|
]
|
||||||
|
|
||||||
result = build_taxonomy_candidates(similar_documents, user=None)
|
result = build_taxonomy_candidates(nodes, user=None)
|
||||||
|
|
||||||
assert len(result["document_types"]) == 5
|
assert len(result["document_types"]) == 5
|
||||||
|
|
||||||
@@ -334,9 +332,9 @@ class TestBuildTaxonomyCandidates:
|
|||||||
"""
|
"""
|
||||||
storage_path = StoragePathFactory.create(name="Invoices")
|
storage_path = StoragePathFactory.create(name="Invoices")
|
||||||
document = DocumentFactory.create(storage_path=storage_path)
|
document = DocumentFactory.create(storage_path=storage_path)
|
||||||
similar_documents = [make_similar(document.pk, 0.5)]
|
nodes = [make_node(document.pk, 0.5)]
|
||||||
|
|
||||||
result = build_taxonomy_candidates(similar_documents, user=None)
|
result = build_taxonomy_candidates(nodes, user=None)
|
||||||
|
|
||||||
assert len(result["storage_paths"]) == 1
|
assert len(result["storage_paths"]) == 1
|
||||||
assert result["storage_paths"][0]["id"] == storage_path.pk
|
assert result["storage_paths"][0]["id"] == storage_path.pk
|
||||||
@@ -352,12 +350,12 @@ class TestBuildTaxonomyCandidates:
|
|||||||
- Only 5 storage_paths are returned
|
- Only 5 storage_paths are returned
|
||||||
"""
|
"""
|
||||||
storage_paths = StoragePathFactory.create_batch(7)
|
storage_paths = StoragePathFactory.create_batch(7)
|
||||||
similar_documents = [
|
nodes = [
|
||||||
make_similar(DocumentFactory.create(storage_path=sp).pk, 0.5)
|
make_node(DocumentFactory.create(storage_path=sp).pk, 0.5)
|
||||||
for sp in storage_paths
|
for sp in storage_paths
|
||||||
]
|
]
|
||||||
|
|
||||||
result = build_taxonomy_candidates(similar_documents, user=None)
|
result = build_taxonomy_candidates(nodes, user=None)
|
||||||
|
|
||||||
assert len(result["storage_paths"]) == 5
|
assert len(result["storage_paths"]) == 5
|
||||||
|
|
||||||
@@ -377,14 +375,14 @@ class TestBuildTaxonomyCandidates:
|
|||||||
tag = TagFactory.create(name="Restricted")
|
tag = TagFactory.create(name="Restricted")
|
||||||
document = DocumentFactory.create()
|
document = DocumentFactory.create()
|
||||||
document.tags.add(tag)
|
document.tags.add(tag)
|
||||||
similar_documents = [make_similar(document.pk, 0.5)]
|
nodes = [make_node(document.pk, 0.5)]
|
||||||
user = UserFactory.create()
|
user = UserFactory.create()
|
||||||
mocker.patch(
|
mocker.patch(
|
||||||
"documents.permissions.permitted_object_ids",
|
"documents.permissions.permitted_object_ids",
|
||||||
return_value=[], # user cannot see this tag
|
return_value=[], # user cannot see this tag
|
||||||
)
|
)
|
||||||
|
|
||||||
result = build_taxonomy_candidates(similar_documents, user=user)
|
result = build_taxonomy_candidates(nodes, user=user)
|
||||||
|
|
||||||
assert result["tags"] == []
|
assert result["tags"] == []
|
||||||
|
|
||||||
@@ -414,10 +412,10 @@ class TestBuildTaxonomyCandidates:
|
|||||||
tag.save()
|
tag.save()
|
||||||
document = DocumentFactory.create()
|
document = DocumentFactory.create()
|
||||||
document.tags.add(tag)
|
document.tags.add(tag)
|
||||||
similar_documents = [make_similar(document.pk, 0.5)]
|
nodes = [make_node(document.pk, 0.5)]
|
||||||
spy = mocker.patch("documents.permissions.permitted_object_ids")
|
spy = mocker.patch("documents.permissions.permitted_object_ids")
|
||||||
|
|
||||||
result = build_taxonomy_candidates(similar_documents, user=None)
|
result = build_taxonomy_candidates(nodes, user=None)
|
||||||
|
|
||||||
assert result["tags"][0]["name"] == "Owned"
|
assert result["tags"][0]["name"] == "Owned"
|
||||||
spy.assert_not_called()
|
spy.assert_not_called()
|
||||||
|
|||||||
Reference in New Issue
Block a user