Compare commits

..
Author SHA1 Message Date
stumpylog d706d2a9fe Mark empty-pks early-return in set_permissions_for_objects as no-cover
Defensive guard for an edge case (all requested pks already gone/invalid)
rather than a path normal usage exercises; matches the existing
pragma: no cover convention elsewhere in this file.
2026-08-27 09:42:21 -07:00
stumpylog 5caa3327fd Fix: use .distinct() for existing-grant lookup, drop flaky query-count invariant tests
.distinct() lets the database dedupe identity ids server-side instead of
transferring one row per (object, grantee) match and deduping in Python --
was the dominant cost on a large selection with existing grants.

Also replaced the two query-count-equality tests (bulk_edit and the
bulk_edit_objects API path) with plain functional-correctness checks at
both batch sizes.  Hopefully stops that flake.
2026-08-27 09:42:21 -07:00
stumpylog c72c8d1574 Perf: avoid unnecessary full-row fetches in batch permission assignment
set_permissions_for_objects now takes a model + pks instead of instances,
and identity filtering resolves straight to ids, so bulk-editing
permissions no longer materializes full Document/User/Group rows just to
read their pk/id. Row construction for bulk_create is also chunked to
bound peak memory for very large "apply to all" operations.
2026-08-27 09:42:21 -07:00
stumpylog b1f5445689 Perf: batch guardian permission assignment in bulk-edit
bulk_edit.set_permissions and BulkEditObjectPermissionsView both
looped documents/objects and called set_permissions_for_object per
object, which itself calls guardian's assign_perm/remove_perm once
per (object, user) pair -- ~10-20+ queries per object, scaling with
selection size.

Added set_permissions_for_objects, a bulk equivalent that resolves
existing permission holders once across the whole batch (not once per
object) and applies changes with a small, batch-size-independent
number of queries per action instead of one per (object, user) pair.
2026-08-27 09:42:21 -07:00
9 changed files with 473 additions and 427 deletions
+7 -4
View File
@@ -27,7 +27,7 @@ from documents.models import DocumentType
from documents.models import PaperlessTask from documents.models import PaperlessTask
from documents.models import StoragePath from documents.models import StoragePath
from documents.models import Tag from documents.models import Tag
from documents.permissions import set_permissions_for_object from documents.permissions import set_permissions_for_objects
from documents.plugins.helpers import DocumentsStatusManager from documents.plugins.helpers import DocumentsStatusManager
from documents.tasks import bulk_update_documents from documents.tasks import bulk_update_documents
from documents.tasks import consume_file from documents.tasks import consume_file
@@ -430,10 +430,13 @@ def set_permissions(
else: else:
qs.update(owner=owner) qs.update(owner=owner)
for doc in qs:
set_permissions_for_object(permissions=set_permissions, object=doc, merge=merge)
affected_docs = list(qs.values_list("pk", flat=True)) affected_docs = list(qs.values_list("pk", flat=True))
set_permissions_for_objects(
permissions=set_permissions,
model=Document,
pks=affected_docs,
merge=merge,
)
bulk_update_documents.apply_async( bulk_update_documents.apply_async(
kwargs={"document_ids": affected_docs}, kwargs={"document_ids": affected_docs},
+178
View File
@@ -173,6 +173,184 @@ def set_permissions_for_object(
) )
def _resolve_permissions(codenames: set[str], ctype: ContentType) -> list[Permission]:
"""
Resolves `codenames` to Permission rows, raising like the single-object
assign_perm() this bulk path replaces does (via a `.get()` internally)
if any codename doesn't exist -- e.g. a client-supplied action name that
was never validated (BulkEditObjectsSerializer._validate_permissions
calls validate_set_permissions() only for its side-effecting id checks
and discards the filtered dict it returns, so an unrecognized action key
reaches this function as-is). A plain `.filter()` with no existence
check would otherwise silently build zero rows and no-op instead of
reporting the bad input.
"""
permission_objs = list(
Permission.objects.filter(content_type=ctype, codename__in=codenames),
)
missing = codenames - {p.codename for p in permission_objs}
if missing:
raise Permission.DoesNotExist(
f"Permission matching query does not exist for codename(s): "
f"{', '.join(sorted(missing))}",
)
return permission_objs
# Target number of permission rows to build in Python before handing them to
# bulk_create -- keeps peak memory bounded for a large "apply to all" call,
# independent of bulk_create's own batch_size (which only caps the size of
# each INSERT statement, not how many row objects exist in memory at once).
_PERMISSION_ROW_CHUNK_SIZE = 5000
def _apply_bulk_permission_entry(
*,
perm_model: type[UserObjectPermission] | type[GroupObjectPermission],
identity_model: type[User] | type[Group],
identity_field: str,
ids: list[int],
codename: str,
permission_objs: list[Permission],
ctype: ContentType,
object_pks: list[str],
merge: bool,
) -> None:
# Only the ids are needed to build permission rows (via `<field>_id=`),
# so avoid fetching full User/Group rows for identities that may not
# even end up being granted anything new.
add_ids = set(
identity_model.objects.filter(id__in=ids).values_list("id", flat=True),
)
if not merge:
existing_ids = set(
perm_model.objects.filter(
content_type=ctype,
object_pk__in=object_pks,
permission__codename=codename,
)
.values_list(f"{identity_field}_id", flat=True)
.distinct(),
)
remove_ids = existing_ids - add_ids
if remove_ids:
perm_model.objects.filter(
content_type=ctype,
object_pk__in=object_pks,
permission__codename=codename,
**{f"{identity_field}_id__in": remove_ids},
).delete()
if not add_ids:
return
rows_per_pk = len(permission_objs) * len(add_ids)
pks_per_chunk = max(1, _PERMISSION_ROW_CHUNK_SIZE // rows_per_pk)
for start in range(0, len(object_pks), pks_per_chunk):
pk_chunk = object_pks[start : start + pks_per_chunk]
rows = [
perm_model(
content_type=ctype,
object_pk=pk,
permission=permission_obj,
**{f"{identity_field}_id": identity_id},
)
for permission_obj in permission_objs
for pk in pk_chunk
for identity_id in add_ids
]
# ignore_conflicts skips only rows that already exist as an exact
# (identity, permission, object) match -- the same de-dup the
# underlying (user|group, permission, object_pk) unique constraint
# already enforces for the single-object assign_perm() this
# replaces, so it doesn't change what counts as "already granted".
# batch_size caps how many rows go into a single INSERT so a huge
# chunk doesn't build one enormous statement.
perm_model.objects.bulk_create(rows, ignore_conflicts=True, batch_size=1000)
def set_permissions_for_objects(
permissions: dict,
model: type[Model],
pks: QuerySet | list,
*,
merge: bool = False,
) -> None:
"""
Bulk equivalent of set_permissions_for_object: applies the same
permission changes to every object identified by `pks` at once.
Takes a model + pks (rather than model instances) deliberately -- the
permission rows built below only ever need `pk`, `content_type`, and
identity ids, so callers shouldn't have to fetch full rows (with every
other field) just to hand them to this function.
Deliberately does not use guardian's queryset/list-aware assign_perm:
passing a list as the object routes to bulk_assign_perm, which skips
creating a direct permission row for anyone who already has the
permission via ANY group membership (it checks
ObjectPermissionChecker.has_perm, which is group-inheritance-aware) --
unlike the single-object assign_perm this replaces, which always
ensures a direct row via get_or_create regardless of group-derived
access. Losing that guarantee would mean a later revocation of the
group's grant silently strips access an admin explicitly asked to be
direct. Bulk-creating rows straight against the permission models
instead (see _apply_bulk_permission_entry) preserves the original
always-create-a-direct-row semantics while still batching every object
and every identity into one query per action, rather than one query per
(object, user) pair.
"""
object_pks = [str(pk) for pk in pks]
if not object_pks: # pragma: no cover
return
model_name = model.__name__.lower()
ctype = ContentType.objects.get_for_model(model)
for action, entry in permissions.items():
codename = f"{action}_{model_name}"
implied_codenames = {codename}
if action == "change":
# change gives view too
implied_codenames.add(f"view_{model_name}")
# Resolved once per action (not once per users/groups branch) and
# shared between both below -- also where an unrecognized action
# name (see _resolve_permissions) is caught.
permission_objs = (
_resolve_permissions(implied_codenames, ctype)
if "users" in entry or "groups" in entry
else []
)
if "users" in entry:
_apply_bulk_permission_entry(
perm_model=UserObjectPermission,
identity_model=User,
identity_field="user",
ids=entry["users"],
codename=codename,
permission_objs=permission_objs,
ctype=ctype,
object_pks=object_pks,
merge=merge,
)
if "groups" in entry:
_apply_bulk_permission_entry(
perm_model=GroupObjectPermission,
identity_model=Group,
identity_field="group",
ids=entry["groups"],
codename=codename,
permission_objs=permission_objs,
ctype=ctype,
object_pks=object_pks,
merge=merge,
)
def permitted_object_ids( def permitted_object_ids(
user: User | None, user: User | None,
model: type[Model], model: type[Model],
+44
View File
@@ -2,10 +2,13 @@ import datetime
import json import json
from unittest import mock from unittest import mock
from django.contrib.auth.models import Group
from django.contrib.auth.models import Permission from django.contrib.auth.models import Permission
from django.contrib.auth.models import User from django.contrib.auth.models import User
from django.test import override_settings from django.test import override_settings
from guardian.shortcuts import assign_perm from guardian.shortcuts import assign_perm
from guardian.shortcuts import get_groups_with_perms
from guardian.shortcuts import get_users_with_perms
from rest_framework import status from rest_framework import status
from rest_framework.test import APITestCase from rest_framework.test import APITestCase
@@ -815,6 +818,47 @@ class TestBulkEditObjects(APITestCase):
self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(StoragePath.objects.count(), 0) self.assertEqual(StoragePath.objects.count(), 0)
def test_bulk_objects_set_permissions_batched_across_object_count(
self,
) -> None:
"""
GIVEN:
- Many tags are being bulk-edited to set permissions at once
WHEN:
- bulk_edit_objects API endpoint is called with set_permissions
operation over a small batch vs. a much larger one
THEN:
- Permissions are applied correctly at both scales
"""
group1 = Group.objects.create(name="perm-group")
permissions = {
"view": {"users": [self.user1.id, self.user2.id], "groups": [group1.id]},
"change": {"users": [self.user1.id], "groups": [group1.id]},
}
def run_with_n_tags(n: int) -> None:
tags = [Tag.objects.create(name=f"perm-tag-{n}-{i}") for i in range(n)]
response = self.client.post(
"/api/bulk_edit_objects/",
json.dumps(
{
"objects": [t.id for t in tags],
"object_type": "tags",
"operation": "set_permissions",
"permissions": permissions,
"merge": False,
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
for tag in tags:
self.assertEqual(get_users_with_perms(tag).count(), 2)
self.assertEqual(get_groups_with_perms(tag).count(), 1)
run_with_n_tags(5)
run_with_n_tags(50)
def test_bulk_objects_delete_all_filtered(self) -> None: def test_bulk_objects_delete_all_filtered(self) -> None:
""" """
GIVEN: GIVEN:
+116
View File
@@ -5,6 +5,7 @@ from unittest import mock
import pikepdf import pikepdf
from django.contrib.auth.models import Group from django.contrib.auth.models import Group
from django.contrib.auth.models import Permission
from django.contrib.auth.models import User from django.contrib.auth.models import User
from django.test import TestCase from django.test import TestCase
from guardian.shortcuts import assign_perm from guardian.shortcuts import assign_perm
@@ -19,6 +20,7 @@ from documents.models import Document
from documents.models import DocumentType from documents.models import DocumentType
from documents.models import StoragePath from documents.models import StoragePath
from documents.models import Tag from documents.models import Tag
from documents.permissions import set_permissions_for_objects
from documents.tests.utils import DirectoriesMixin from documents.tests.utils import DirectoriesMixin
@@ -510,6 +512,120 @@ class TestBulkEdit(DirectoriesMixin, TestCase):
) )
self.assertEqual(groups_with_perms.count(), 2) self.assertEqual(groups_with_perms.count(), 2)
@mock.patch("documents.tasks.bulk_update_documents.apply_async")
def test_set_permissions_batched_across_document_count(
self,
m,
) -> None:
"""
GIVEN:
- Many documents are being bulk-edited to set permissions at once
WHEN:
- set_permissions runs over a small batch vs. a much larger one
THEN:
- Permissions are applied correctly at both scales
"""
permissions = {
"view": {
"users": [self.user1.id, self.user2.id],
"groups": [self.group2.id],
},
"change": {
"users": [self.user1.id],
"groups": [self.group2.id],
},
}
def run_with_n_documents(n: int) -> None:
docs = [
Document.objects.create(checksum=f"perm-{n}-{i}", title=f"perm-{n}-{i}")
for i in range(n)
]
bulk_edit.set_permissions(
[doc.id for doc in docs],
set_permissions=permissions,
owner=self.owner,
merge=False,
)
for doc in docs:
self.assertEqual(get_users_with_perms(doc).count(), 2)
self.assertEqual(get_groups_with_perms(doc).count(), 1)
run_with_n_documents(5)
run_with_n_documents(50)
@mock.patch("documents.tasks.bulk_update_documents.apply_async")
def test_set_permissions_grants_direct_perm_even_if_already_granted_via_group(
self,
m,
) -> None:
"""
GIVEN:
- A user already has view access to a document via group
membership, with no direct grant of their own
WHEN:
- set_permissions explicitly grants that same user direct view
access via bulk_edit
THEN:
- A direct permission grant is created for the user, not skipped
because they already have equivalent access via the group
Regression test: guardian's queryset-aware assign_perm() (routed to
when the target is a list/queryset) skips creating a direct row for
anyone whose ObjectPermissionChecker.has_perm() already returns True
-- which includes group-derived access. The single-object assign_perm
this bulk path replaces has no such check; it always ensures a
direct row via get_or_create. Losing that guarantee would mean
revoking the group's grant later silently strips access that was
supposed to be explicit.
"""
self.doc1.owner = self.user1
self.doc1.save()
self.user1.groups.add(self.group1)
assign_perm("view_document", self.group1, self.doc1)
bulk_edit.set_permissions(
[self.doc1.id],
set_permissions={
"view": {"users": [self.user1.id], "groups": []},
},
merge=True,
)
direct_users = get_users_with_perms(
self.doc1,
only_with_perms_in=["view_document"],
with_group_users=False,
)
self.assertIn(self.user1, direct_users)
def test_set_permissions_for_objects_raises_for_unknown_action(self) -> None:
"""
GIVEN:
- An unrecognized permission action name with users to grant it
to
WHEN:
- set_permissions_for_objects is called
THEN:
- Permission.DoesNotExist is raised, not a silent no-op
Regression test: the endpoint that calls this
(BulkEditObjectPermissionsView) never actually validates action
names against the raw client-supplied permissions dict --
BulkEditObjectsSerializer._validate_permissions calls
validate_set_permissions() only for its side-effecting user/group id
checks and discards the filtered dict it returns -- so a bogus
action key reaches this function as-is. Resolving the Permission via
a bare `.filter()` (which returns empty instead of raising) would
silently drop the grant and report success.
"""
with self.assertRaises(Permission.DoesNotExist):
set_permissions_for_objects(
{"not_a_real_action": {"users": [self.user1.id], "groups": []}},
Document,
[self.doc1.pk],
)
@mock.patch("documents.models.Document.delete") @mock.patch("documents.models.Document.delete")
def test_delete_documents_old_uuid_field(self, m) -> None: def test_delete_documents_old_uuid_field(self, m) -> None:
m.side_effect = Exception("Data too long for column 'transaction_id' at row 1") m.side_effect = Exception("Data too long for column 'transaction_id' at row 1")
+7 -7
View File
@@ -178,7 +178,7 @@ from documents.permissions import has_perms_owner_aware
from documents.permissions import has_system_status_permission from documents.permissions import has_system_status_permission
from documents.permissions import permitted_document_ids from documents.permissions import permitted_document_ids
from documents.permissions import permitted_object_ids from documents.permissions import permitted_object_ids
from documents.permissions import set_permissions_for_object from documents.permissions import set_permissions_for_objects
from documents.plugins.date_parsing import get_date_parser from documents.plugins.date_parsing import get_date_parser
from documents.schema import generate_object_with_permissions_schema from documents.schema import generate_object_with_permissions_schema
from documents.search import SearchHit from documents.search import SearchHit
@@ -4914,12 +4914,12 @@ class BulkEditObjectsView(PassUserMixin):
qs_owner_update.update(owner=owner) qs_owner_update.update(owner=owner)
if "permissions" in serializer.validated_data: if "permissions" in serializer.validated_data:
for obj in qs: set_permissions_for_objects(
set_permissions_for_object( permissions=permissions,
permissions=permissions, model=object_class,
object=obj, pks=qs.values_list("pk", flat=True),
merge=merge, merge=merge,
) )
except Exception as e: except Exception as e:
logger.warning( logger.warning(
+54 -99
View File
@@ -5,14 +5,13 @@ from django.conf import settings
from django.contrib.auth.models import User from django.contrib.auth.models import User
from documents.models import Document from documents.models import Document
from documents.permissions import permitted_object_ids from documents.permissions import get_objects_for_user_owner_aware
from documents.permissions import restrict_queryset_to_visible
from documents.permissions import user_is_unrestricted
from paperless.config import AIConfig from paperless.config import AIConfig
from paperless_ai.base_model import ClassificationSuggestions from paperless_ai.base_model import ClassificationSuggestions
from paperless_ai.base_model import TaxonomyChoiceDict from paperless_ai.base_model import TaxonomyChoiceDict
from paperless_ai.client import AIClient from paperless_ai.client import AIClient
from paperless_ai.db import db_connection_released from paperless_ai.db import db_connection_released
from paperless_ai.indexing import _node_document_ids
from paperless_ai.indexing import retrieve_similar_nodes from paperless_ai.indexing import retrieve_similar_nodes
from paperless_ai.indexing import truncate_content from paperless_ai.indexing import truncate_content
from paperless_ai.prompts.context import ClassificationPromptContext from paperless_ai.prompts.context import ClassificationPromptContext
@@ -20,9 +19,7 @@ from paperless_ai.prompts.context import LocalizationPromptContext
from paperless_ai.prompts.context import RagContextPromptContext from paperless_ai.prompts.context import RagContextPromptContext
from paperless_ai.prompts.render import render_prompt from paperless_ai.prompts.render import render_prompt
from paperless_ai.taxonomy import AssignedMetadata from paperless_ai.taxonomy import AssignedMetadata
from paperless_ai.taxonomy import SimilarDocument
from paperless_ai.taxonomy import TaxonomyCandidates from paperless_ai.taxonomy import TaxonomyCandidates
from paperless_ai.taxonomy import _node_document_weights
from paperless_ai.taxonomy import build_taxonomy_candidates from paperless_ai.taxonomy import build_taxonomy_candidates
from paperless_ai.taxonomy import empty_taxonomy_candidates from paperless_ai.taxonomy import empty_taxonomy_candidates
from paperless_ai.taxonomy import format_taxonomy_for_prompt from paperless_ai.taxonomy import format_taxonomy_for_prompt
@@ -42,48 +39,6 @@ logger = logging.getLogger("paperless_ai.rag_classifier")
TAXONOMY_CANDIDATE_TOP_K = 15 TAXONOMY_CANDIDATE_TOP_K = 15
def _fulltext_similar_documents(
document: Document,
user: User | None,
top_k: int,
) -> list[SimilarDocument]:
"""Rank-based fallback when no embedding backend is configured. Uses
Tantivy's "More Like This" (term-overlap similarity) instead of vector
similarity - cruder, but far better than no candidates at all.
more_like_this_ids returns only a ranked ID list, no scores, so weight is
synthesized from rank (descending from top_k) rather than claiming a
similarity magnitude that doesn't exist. An unrestricted user (none, or an
active superuser - see user_is_unrestricted) is normalized to ``None``
before calling, since the backend's permission filter has no superuser
short-circuit of its own. Results are re-checked with
restrict_queryset_to_visible() since Tantivy's indexed permission fields
lag the DB via async reindexing.
"""
from documents.search import get_backend
unrestricted = user_is_unrestricted(user)
search_user = None if unrestricted else user
backend = get_backend()
similar_ids = backend.more_like_this_ids(
document.pk,
user=search_user,
limit=top_k,
)
if not unrestricted:
allowed_ids = set(
restrict_queryset_to_visible(
Document.objects.filter(pk__in=similar_ids),
user,
"view_document",
).values_list("pk", flat=True),
)
similar_ids = [doc_id for doc_id in similar_ids if doc_id in allowed_ids]
return [
SimilarDocument(document_id=doc_id, weight=float(top_k - rank))
for rank, doc_id in enumerate(similar_ids)
]
def get_language_name(language_code: str) -> str: def get_language_name(language_code: str) -> str:
normalized_language_code = language_code.lower() normalized_language_code = language_code.lower()
for code, name in settings.LANGUAGES: for code, name in settings.LANGUAGES:
@@ -192,54 +147,45 @@ def get_taxonomy_context(
user: User | None = None, user: User | None = None,
max_docs: int = 5, max_docs: int = 5,
) -> tuple[TaxonomyCandidates, AssignedMetadata, str]: ) -> tuple[TaxonomyCandidates, AssignedMetadata, str]:
"""One retrieval feeds both taxonomy candidates and RAG text context. Uses """One retrieval feeds both taxonomy candidates and RAG text context.
vector similarity when an embedding backend is configured, otherwise On any retrieval failure, degrades to empty candidates/context rather than
falls back to Tantivy full-text "More Like This" similarity - see propagating the exception - a vector-store outage should not block
_fulltext_similar_documents. On any retrieval failure, degrades to empty classification, only its RAG-assisted enrichment.
candidates/context rather than propagating the exception - neither a
vector-store outage nor a search-index issue should block classification,
only its context-assisted enrichment.
""" """
assigned = get_assigned_metadata(document, user) assigned = get_assigned_metadata(document, user)
ai_config = AIConfig()
try: try:
if ai_config.llm_embedding_backend: # None means "no restriction" to retrieve_similar_nodes. A superuser
# None means "no restriction" to retrieve_similar_nodes. A superuser # (like no user at all) can see every document, so skip materializing
# (like no user at all) can see every document, so skip materializing # every visible pk into a Python list and passing it through as an IN
# every visible pk into a Python list and passing it through as an IN # filter: for a large library that is a wasted quadratic scan in the
# filter: for a large library that is a wasted quadratic scan in the # vector store at best, and past ~32,763 documents a hard
# vector store at best, and past ~32,763 documents a hard # sqlite3.OperationalError (SQLite's bound-parameter limit) at worst.
# sqlite3.OperationalError (SQLite's bound-parameter limit) at worst. # get_objects_for_user_owner_aware() would return every Document for a
# permitted_object_ids() has its own superuser shortcut that would # superuser anyway (guardian's own with_superuser shortcut), so this
# return every Document's id anyway, so this changes nothing about # changes nothing about which documents are considered -- only how we
# which documents are considered -- only how we get there. # get there.
visible_document_ids = ( visible_document_ids = (
None None
if user is None or user.is_superuser if user is None or user.is_superuser
else list(permitted_object_ids(user, Document, "view_document")) else list(
) get_objects_for_user_owner_aware(
nodes = retrieve_similar_nodes( user,
document, "view_document",
top_k=TAXONOMY_CANDIDATE_TOP_K, Document,
document_ids=visible_document_ids, ).values_list("pk", flat=True),
)
similar_documents = _node_document_weights(nodes)
else:
# See _fulltext_similar_documents: it applies its own permission
# filter via `user`, so no visible-document-id list is needed here.
similar_documents = _fulltext_similar_documents(
document,
user,
top_k=TAXONOMY_CANDIDATE_TOP_K,
) )
)
nodes = retrieve_similar_nodes(
document,
top_k=TAXONOMY_CANDIDATE_TOP_K,
document_ids=visible_document_ids,
)
candidates = build_taxonomy_candidates(similar_documents, user) candidates = build_taxonomy_candidates(nodes, user)
similar_doc_ids = [s["document_id"] for s in similar_documents] similar_docs = list(
docs_by_id = Document.objects.in_bulk(similar_doc_ids) Document.objects.filter(pk__in=_node_document_ids(nodes))[:max_docs],
similar_docs = [ )
docs_by_id[doc_id] for doc_id in similar_doc_ids if doc_id in docs_by_id
][:max_docs]
context_blocks = [] context_blocks = []
for similar in similar_docs: for similar in similar_docs:
text = similar.content[:1000] or "" text = similar.content[:1000] or ""
@@ -247,8 +193,8 @@ def get_taxonomy_context(
context_blocks.append(f"TITLE: {title}\n{text}") context_blocks.append(f"TITLE: {title}\n{text}")
except Exception: except Exception:
logger.exception( logger.exception(
"Failed to retrieve similar-document context for document %s; " "Failed to retrieve RAG neighbours for document %s; continuing "
"continuing without taxonomy candidates or similar-document context.", "without taxonomy candidates or similar-document context.",
document.pk, document.pk,
) )
return empty_taxonomy_candidates(), assigned, "" return empty_taxonomy_candidates(), assigned, ""
@@ -331,14 +277,23 @@ def get_ai_document_classification(
) -> ClassificationSuggestions: ) -> ClassificationSuggestions:
ai_config = AIConfig() ai_config = AIConfig()
candidates, assigned, context = get_taxonomy_context(document, user) if ai_config.llm_embedding_backend:
prompt = build_prompt_with_rag( candidates, assigned, context = get_taxonomy_context(document, user)
document, prompt = build_prompt_with_rag(
ai_config, document,
candidates=candidates, ai_config,
assigned=assigned, candidates=candidates,
context=context, assigned=assigned,
) context=context,
)
else:
candidates = empty_taxonomy_candidates()
prompt = build_prompt_without_rag(
document,
ai_config,
candidates=candidates,
assigned=get_assigned_metadata(document, user),
)
client = AIClient() client = AIClient()
# Hand the pooled DB connection back while the (slow) LLM query runs so it # Hand the pooled DB connection back while the (slow) LLM query runs so it
+14 -27
View File
@@ -33,11 +33,6 @@ class TaxonomyCandidate(TypedDict):
weight: float weight: float
class SimilarDocument(TypedDict):
document_id: int
weight: float
class TaxonomyCandidates(TypedDict): class TaxonomyCandidates(TypedDict):
tags: list[TaxonomyCandidate] tags: list[TaxonomyCandidate]
document_types: list[TaxonomyCandidate] document_types: list[TaxonomyCandidate]
@@ -110,10 +105,10 @@ def get_assigned_metadata(document: Document, user: User | None) -> AssignedMeta
) )
def _node_document_weights(nodes: list["NodeWithScore"]) -> list[SimilarDocument]: def _node_document_weights(nodes: list["NodeWithScore"]) -> dict[int, float]:
"""Sum each node's similarity score into its document_id (a document can """document_id -> that node's similarity score, summed if a document_id
appear via multiple chunks/nodes) and return one SimilarDocument per appears more than once across the retrieved nodes (e.g. multiple chunks
distinct document_id.""" of the same source document)."""
weights: dict[int, float] = defaultdict(float) weights: dict[int, float] = defaultdict(float)
for node in nodes: for node in nodes:
document_id = node.metadata.get("document_id") document_id = node.metadata.get("document_id")
@@ -126,10 +121,7 @@ def _node_document_weights(nodes: list["NodeWithScore"]) -> list[SimilarDocument
weights[int(document_id)] += float(node.score or 0.0) weights[int(document_id)] += float(node.score or 0.0)
except (TypeError, ValueError): # pragma: no cover except (TypeError, ValueError): # pragma: no cover
continue continue
return [ return weights
SimilarDocument(document_id=document_id, weight=weight)
for document_id, weight in weights.items()
]
def _visible_ranked_candidates( def _visible_ranked_candidates(
@@ -165,25 +157,20 @@ def _visible_ranked_candidates(
def build_taxonomy_candidates( def build_taxonomy_candidates(
similar_documents: list[SimilarDocument], nodes: list["NodeWithScore"],
user: User | None, user: User | None,
) -> TaxonomyCandidates: ) -> TaxonomyCandidates:
"""Resolve each similar document's id to a live Document, read its """Resolve each neighbour node's document_id to a live Document, read its
*current* tags/type/correspondent/storage_path via the ORM (never any *current* tags/type/correspondent/storage_path via the ORM (never the
possibly-stale names an adapter's source might have cached), weight each possibly-stale names cached in vector-index node metadata), weight each
distinct taxonomy object by aggregate similarity weight, permission-filter distinct taxonomy object by aggregate neighbour similarity, permission-filter
against what ``user`` can see, and return each category ranked by weight against what ``user`` can see, and return each category ranked by weight
and capped. ``similar_documents`` may come from either the vector-RAG and capped.
adapter or the full-text fallback adapter - both produce this same shape.
""" """
if not similar_documents:
return empty_taxonomy_candidates()
# Both adapters guarantee at most one SimilarDocument per document_id, so document_weights = _node_document_weights(nodes)
# this never silently drops a duplicate's weight. if not document_weights:
document_weights: dict[int, float] = { return empty_taxonomy_candidates()
s["document_id"]: s["weight"] for s in similar_documents
}
# Only .tags.all() needs prefetching (a reverse M2M, one extra query for # Only .tags.all() needs prefetching (a reverse M2M, one extra query for
# the whole batch). document_type/correspondent/storage_path are read # the whole batch). document_type/correspondent/storage_path are read
+22 -257
View File
@@ -1,4 +1,3 @@
from collections.abc import Generator
from types import SimpleNamespace from types import SimpleNamespace
from unittest.mock import MagicMock from unittest.mock import MagicMock
from unittest.mock import patch from unittest.mock import patch
@@ -8,13 +7,10 @@ import pytest_mock
from django.test import override_settings from django.test import override_settings
from documents.models import Document from documents.models import Document
from documents.search import TantivyBackend
from documents.tests.factories import DocumentFactory from documents.tests.factories import DocumentFactory
from documents.tests.factories import TagFactory from documents.tests.factories import TagFactory
from documents.tests.factories import UserFactory from documents.tests.factories import UserFactory
from paperless.config import AIConfig from paperless.config import AIConfig
from paperless_ai.ai_classifier import TAXONOMY_CANDIDATE_TOP_K
from paperless_ai.ai_classifier import _fulltext_similar_documents
from paperless_ai.ai_classifier import _restrict_to_shown_candidates from paperless_ai.ai_classifier import _restrict_to_shown_candidates
from paperless_ai.ai_classifier import build_localization_prompt from paperless_ai.ai_classifier import build_localization_prompt
from paperless_ai.ai_classifier import build_prompt_with_rag from paperless_ai.ai_classifier import build_prompt_with_rag
@@ -24,7 +20,6 @@ from paperless_ai.ai_classifier import get_language_name
from paperless_ai.ai_classifier import get_taxonomy_context from paperless_ai.ai_classifier import get_taxonomy_context
from paperless_ai.base_model import ClassificationSuggestions from paperless_ai.base_model import ClassificationSuggestions
from paperless_ai.base_model import TaxonomyChoiceDict from paperless_ai.base_model import TaxonomyChoiceDict
from paperless_ai.taxonomy import SimilarDocument
from paperless_ai.taxonomy import TaxonomyCandidate from paperless_ai.taxonomy import TaxonomyCandidate
from paperless_ai.taxonomy import TaxonomyCandidates from paperless_ai.taxonomy import TaxonomyCandidates
from paperless_ai.taxonomy import empty_taxonomy_candidates from paperless_ai.taxonomy import empty_taxonomy_candidates
@@ -209,10 +204,12 @@ def test_use_rag_if_configured(
@pytest.mark.django_db @pytest.mark.django_db
@patch("paperless_ai.client.AIClient.run_llm_query") @patch("paperless_ai.client.AIClient.run_llm_query")
@patch("paperless_ai.ai_classifier.build_prompt_with_rag") @patch("paperless_ai.ai_classifier.build_prompt_without_rag")
@patch("paperless_ai.ai_classifier.AIConfig")
@override_settings(LLM_BACKEND="ollama", LLM_MODEL="some_model") @override_settings(LLM_BACKEND="ollama", LLM_MODEL="some_model")
def test_use_rag_prompt_even_without_embedding_backend( def test_use_without_rag_if_not_configured(
mock_build_prompt_with_rag, mock_ai_config,
mock_build_prompt_without_rag,
mock_run_llm_query, mock_run_llm_query,
mock_document, mock_document,
): ):
@@ -222,13 +219,13 @@ def test_use_rag_prompt_even_without_embedding_backend(
WHEN: WHEN:
- get_ai_document_classification() is called - get_ai_document_classification() is called
THEN: THEN:
- The RAG-context prompt builder is still used (fed by the full-text - The non-RAG prompt builder is used
fallback's context/candidates instead of the vector store's)
""" """
mock_build_prompt_with_rag.return_value = "Prompt with RAG" mock_ai_config.return_value.llm_embedding_backend = None
mock_build_prompt_without_rag.return_value = "Prompt without RAG"
mock_run_llm_query.return_value = NESTED_SUGGESTIONS mock_run_llm_query.return_value = NESTED_SUGGESTIONS
get_ai_document_classification(mock_document) get_ai_document_classification(mock_document)
mock_build_prompt_with_rag.assert_called_once() mock_build_prompt_without_rag.assert_called_once()
@pytest.mark.django_db @pytest.mark.django_db
@@ -306,7 +303,6 @@ def test_build_localization_prompt_preserves_unicode_characters():
@pytest.mark.django_db @pytest.mark.django_db
@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
def test_get_taxonomy_context_assembles_rag_text_and_candidates(): def test_get_taxonomy_context_assembles_rag_text_and_candidates():
""" """
GIVEN: GIVEN:
@@ -348,7 +344,6 @@ def test_get_taxonomy_context_assembles_rag_text_and_candidates():
@pytest.mark.django_db @pytest.mark.django_db
@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
def test_get_taxonomy_context_no_similar_docs(): def test_get_taxonomy_context_no_similar_docs():
""" """
GIVEN: GIVEN:
@@ -372,67 +367,6 @@ def test_get_taxonomy_context_no_similar_docs():
} }
@pytest.mark.django_db
def test_get_taxonomy_context_uses_fulltext_fallback_when_no_embedding_backend(
mocker: pytest_mock.MockerFixture,
) -> None:
"""
GIVEN:
- No LLM embedding backend is configured (the default test settings)
WHEN:
- get_taxonomy_context() is called
THEN:
- _fulltext_similar_documents() is called with the document, the user
and TAXONOMY_CANDIDATE_TOP_K
- retrieve_similar_nodes() (the vector path) is never called
"""
document = DocumentFactory.create(content="Some content")
mock_fulltext = mocker.patch(
"paperless_ai.ai_classifier._fulltext_similar_documents",
return_value=[],
)
mock_retrieve = mocker.patch("paperless_ai.ai_classifier.retrieve_similar_nodes")
get_taxonomy_context(document, user=None)
mock_fulltext.assert_called_once_with(
document,
None,
top_k=TAXONOMY_CANDIDATE_TOP_K,
)
mock_retrieve.assert_not_called()
@pytest.mark.django_db
@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
def test_get_taxonomy_context_uses_vector_path_when_embedding_backend_configured(
mocker: pytest_mock.MockerFixture,
) -> None:
"""
GIVEN:
- An LLM embedding backend is configured
WHEN:
- get_taxonomy_context() is called
THEN:
- retrieve_similar_nodes() (the vector path) is called
- _fulltext_similar_documents() (the no-embedding-backend fallback)
is never called
"""
document = DocumentFactory.create(content="Some content")
mock_retrieve = mocker.patch(
"paperless_ai.ai_classifier.retrieve_similar_nodes",
return_value=[],
)
mock_fulltext = mocker.patch(
"paperless_ai.ai_classifier._fulltext_similar_documents",
)
get_taxonomy_context(document, user=None)
mock_retrieve.assert_called_once()
mock_fulltext.assert_not_called()
class TestGetTaxonomyContextVisibility: class TestGetTaxonomyContextVisibility:
"""get_taxonomy_context must not materialize every visible document id """get_taxonomy_context must not materialize every visible document id
for a user who can already see the whole library: a superuser (like no for a user who can already see the whole library: a superuser (like no
@@ -445,7 +379,6 @@ class TestGetTaxonomyContextVisibility:
""" """
@pytest.mark.django_db @pytest.mark.django_db
@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
def test_skips_permission_lookup_for_superuser( def test_skips_permission_lookup_for_superuser(
self, self,
mocker: pytest_mock.MockerFixture, mocker: pytest_mock.MockerFixture,
@@ -464,18 +397,17 @@ class TestGetTaxonomyContextVisibility:
"paperless_ai.ai_classifier.retrieve_similar_nodes", "paperless_ai.ai_classifier.retrieve_similar_nodes",
return_value=[], return_value=[],
) )
mock_permitted = mocker.patch( mock_get_objects = mocker.patch(
"paperless_ai.ai_classifier.permitted_object_ids", "paperless_ai.ai_classifier.get_objects_for_user_owner_aware",
) )
user = UserFactory.create(is_superuser=True) user = UserFactory.create(is_superuser=True)
get_taxonomy_context(document, user) get_taxonomy_context(document, user)
mock_permitted.assert_not_called() mock_get_objects.assert_not_called()
assert mock_retrieve.call_args.kwargs["document_ids"] is None assert mock_retrieve.call_args.kwargs["document_ids"] is None
@pytest.mark.django_db @pytest.mark.django_db
@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
def test_skips_permission_lookup_when_no_user( def test_skips_permission_lookup_when_no_user(
self, self,
mocker: pytest_mock.MockerFixture, mocker: pytest_mock.MockerFixture,
@@ -494,17 +426,16 @@ class TestGetTaxonomyContextVisibility:
"paperless_ai.ai_classifier.retrieve_similar_nodes", "paperless_ai.ai_classifier.retrieve_similar_nodes",
return_value=[], return_value=[],
) )
mock_permitted = mocker.patch( mock_get_objects = mocker.patch(
"paperless_ai.ai_classifier.permitted_object_ids", "paperless_ai.ai_classifier.get_objects_for_user_owner_aware",
) )
get_taxonomy_context(document, None) get_taxonomy_context(document, None)
mock_permitted.assert_not_called() mock_get_objects.assert_not_called()
assert mock_retrieve.call_args.kwargs["document_ids"] is None assert mock_retrieve.call_args.kwargs["document_ids"] is None
@pytest.mark.django_db @pytest.mark.django_db
@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
def test_restricts_to_visible_documents_for_non_superuser( def test_restricts_to_visible_documents_for_non_superuser(
self, self,
mocker: pytest_mock.MockerFixture, mocker: pytest_mock.MockerFixture,
@@ -515,7 +446,7 @@ class TestGetTaxonomyContextVisibility:
WHEN: WHEN:
- get_taxonomy_context() is called - get_taxonomy_context() is called
THEN: THEN:
- The user's permitted document ids are looked up and passed to - The user's visible document ids are looked up and passed to
retrieve_similar_nodes() as a restriction retrieve_similar_nodes() as a restriction
""" """
document = DocumentFactory.create(content="Some content") document = DocumentFactory.create(content="Some content")
@@ -523,186 +454,21 @@ class TestGetTaxonomyContextVisibility:
"paperless_ai.ai_classifier.retrieve_similar_nodes", "paperless_ai.ai_classifier.retrieve_similar_nodes",
return_value=[], return_value=[],
) )
mock_permitted = mocker.patch( mock_queryset = mocker.MagicMock()
"paperless_ai.ai_classifier.permitted_object_ids", mock_queryset.values_list.return_value = [1, 2, 3]
return_value=[1, 2, 3], mock_get_objects = mocker.patch(
"paperless_ai.ai_classifier.get_objects_for_user_owner_aware",
return_value=mock_queryset,
) )
user = UserFactory.create(is_superuser=False) user = UserFactory.create(is_superuser=False)
get_taxonomy_context(document, user) get_taxonomy_context(document, user)
mock_permitted.assert_called_once_with(user, Document, "view_document") mock_get_objects.assert_called_once_with(user, "view_document", Document)
assert mock_retrieve.call_args.kwargs["document_ids"] == [1, 2, 3] assert mock_retrieve.call_args.kwargs["document_ids"] == [1, 2, 3]
@pytest.mark.django_db @pytest.mark.django_db
class TestFulltextSimilarDocuments:
"""_fulltext_similar_documents is the no-embedding-backend fallback: it
asks the Tantivy full-text index for "More Like This" neighbours instead
of the vector store, and synthesizes a rank-based weight since Tantivy's
more_like_this_ids returns only an ordered id list, no scores.
"""
@pytest.fixture
def fulltext_backend(
self,
mocker: pytest_mock.MockerFixture,
) -> Generator[TantivyBackend, None, None]:
"""An in-memory Tantivy backend, wired up as the module-level
singleton _fulltext_similar_documents resolves via get_backend()."""
backend = TantivyBackend(path=None)
backend.open()
mocker.patch("documents.search.get_backend", return_value=backend)
try:
yield backend
finally:
backend.close()
def test_ranks_by_rank_based_weight_descending(
self,
fulltext_backend: TantivyBackend,
) -> None:
"""
GIVEN:
- A source document and two similar documents indexed in Tantivy
WHEN:
- _fulltext_similar_documents() is called
THEN:
- Each result's weight reflects its rank (first result weighted
higher than the second), not a raw similarity score
"""
source = DocumentFactory.create(content="quarterly financial report details")
first = DocumentFactory.create(content="quarterly financial report details")
second = DocumentFactory.create(content="financial report")
for doc in (source, first, second):
fulltext_backend.add_or_update(doc)
result = _fulltext_similar_documents(source, user=None, top_k=5)
assert len(result) == 2
weight_by_id = {s["document_id"]: s["weight"] for s in result}
assert weight_by_id[first.pk] > weight_by_id[second.pk]
def test_excludes_source_document(
self,
fulltext_backend: TantivyBackend,
) -> None:
"""
GIVEN:
- A source document indexed in Tantivy with no other documents
WHEN:
- _fulltext_similar_documents() is called
THEN:
- An empty list is returned - the source document is never its
own similar document
"""
source = DocumentFactory.create(content="unique unrelated content")
fulltext_backend.add_or_update(source)
result = _fulltext_similar_documents(source, user=None, top_k=5)
assert result == []
def test_empty_index_returns_empty_list(
self,
fulltext_backend: TantivyBackend,
) -> None:
"""
GIVEN:
- A document that has never been indexed (fresh/empty Tantivy index)
WHEN:
- _fulltext_similar_documents() is called
THEN:
- An empty list is returned rather than raising
"""
source = DocumentFactory.create(content="never indexed")
result = _fulltext_similar_documents(source, user=None, top_k=5)
assert result == []
def test_respects_top_k_limit(
self,
fulltext_backend: TantivyBackend,
) -> None:
"""
GIVEN:
- A source document and four similar documents indexed
WHEN:
- _fulltext_similar_documents() is called with top_k=2
THEN:
- At most 2 results are returned
"""
source = DocumentFactory.create(content="shared overlapping keyword text")
fulltext_backend.add_or_update(source)
for _ in range(4):
fulltext_backend.add_or_update(
DocumentFactory.create(content="shared overlapping keyword text"),
)
result = _fulltext_similar_documents(source, user=None, top_k=2)
assert len(result) == 2
def test_result_shape_is_similar_document(
self,
fulltext_backend: TantivyBackend,
) -> None:
"""
GIVEN:
- A source document and one similar document indexed
WHEN:
- _fulltext_similar_documents() is called
THEN:
- Each result is a SimilarDocument (document_id + weight only)
"""
source = DocumentFactory.create(content="shared content phrase")
other = DocumentFactory.create(content="shared content phrase")
fulltext_backend.add_or_update(source)
fulltext_backend.add_or_update(other)
result = _fulltext_similar_documents(source, user=None, top_k=5)
# rank 0 (the only/best result) with top_k=5 -> weight = top_k - rank = 5.0,
# per the "first result gets top_k, the last gets 1" formula.
assert result == [SimilarDocument(document_id=other.pk, weight=5.0)]
def test_superuser_sees_other_users_documents(
self,
fulltext_backend: TantivyBackend,
) -> None:
"""
GIVEN:
- A source document owned by one user and a similar document
owned by a different user, with no sharing between them
WHEN:
- _fulltext_similar_documents() is called with a superuser
THEN:
- The other user's document is still returned as a similar
document - a superuser must not be narrowed by the backend's
owner-based permission filter
"""
owner = UserFactory.create()
other_owner = UserFactory.create()
superuser = UserFactory.create(is_superuser=True)
source = DocumentFactory.create(
content="shared content phrase",
owner=owner,
)
other = DocumentFactory.create(
content="shared content phrase",
owner=other_owner,
)
fulltext_backend.add_or_update(source)
fulltext_backend.add_or_update(other)
result = _fulltext_similar_documents(source, user=superuser, top_k=5)
assert [s["document_id"] for s in result] == [other.pk]
@pytest.mark.django_db
@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
@patch("paperless_ai.ai_classifier.retrieve_similar_nodes") @patch("paperless_ai.ai_classifier.retrieve_similar_nodes")
def test_get_taxonomy_context_retrieval_failure_degrades_to_no_hints(mock_retrieve): def test_get_taxonomy_context_retrieval_failure_degrades_to_no_hints(mock_retrieve):
""" """
@@ -729,7 +495,6 @@ def test_get_taxonomy_context_retrieval_failure_degrades_to_no_hints(mock_retrie
@pytest.mark.django_db @pytest.mark.django_db
@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
@patch("paperless_ai.ai_classifier.build_taxonomy_candidates") @patch("paperless_ai.ai_classifier.build_taxonomy_candidates")
@patch("paperless_ai.ai_classifier.retrieve_similar_nodes") @patch("paperless_ai.ai_classifier.retrieve_similar_nodes")
def test_get_taxonomy_context_candidate_building_failure_degrades_to_no_hints( def test_get_taxonomy_context_candidate_building_failure_degrades_to_no_hints(
+31 -33
View File
@@ -1,4 +1,5 @@
import json import json
from types import SimpleNamespace
import pytest import pytest
import pytest_mock import pytest_mock
@@ -10,7 +11,6 @@ from documents.tests.factories import StoragePathFactory
from documents.tests.factories import TagFactory from documents.tests.factories import TagFactory
from documents.tests.factories import UserFactory from documents.tests.factories import UserFactory
from paperless_ai.taxonomy import AssignedMetadata from paperless_ai.taxonomy import AssignedMetadata
from paperless_ai.taxonomy import SimilarDocument
from paperless_ai.taxonomy import TaxonomyCandidates from paperless_ai.taxonomy import TaxonomyCandidates
from paperless_ai.taxonomy import build_taxonomy_candidates from paperless_ai.taxonomy import build_taxonomy_candidates
from paperless_ai.taxonomy import format_taxonomy_for_prompt from paperless_ai.taxonomy import format_taxonomy_for_prompt
@@ -132,8 +132,9 @@ class TestGetAssignedMetadata:
assert result["tags"] == ["Owned By Someone Else"] assert result["tags"] == ["Owned By Someone Else"]
def make_similar(document_id: int, weight: float) -> SimilarDocument: def make_node(document_id: int, score: float) -> SimpleNamespace:
return SimilarDocument(document_id=document_id, weight=weight) """A stand-in for NodeWithScore: only ``.metadata``/``.score`` are read."""
return SimpleNamespace(metadata={"document_id": str(document_id)}, score=score)
@pytest.mark.django_db @pytest.mark.django_db
@@ -169,9 +170,9 @@ class TestBuildTaxonomyCandidates:
doc_a.tags.add(tag) doc_a.tags.add(tag)
doc_b = DocumentFactory.create() doc_b = DocumentFactory.create()
doc_b.tags.add(tag) doc_b.tags.add(tag)
similar_documents = [make_similar(doc_a.pk, 0.9), make_similar(doc_b.pk, 0.4)] nodes = [make_node(doc_a.pk, 0.9), make_node(doc_b.pk, 0.4)]
result = build_taxonomy_candidates(similar_documents, user=None) result = build_taxonomy_candidates(nodes, user=None)
assert len(result["tags"]) == 1 assert len(result["tags"]) == 1
assert result["tags"][0]["id"] == tag.pk assert result["tags"][0]["id"] == tag.pk
@@ -196,9 +197,9 @@ class TestBuildTaxonomyCandidates:
document.tags.add(tag) document.tags.add(tag)
tag.name = "New Name" tag.name = "New Name"
tag.save() tag.save()
similar_documents = [make_similar(document.pk, 0.5)] nodes = [make_node(document.pk, 0.5)]
result = build_taxonomy_candidates(similar_documents, user=None) result = build_taxonomy_candidates(nodes, user=None)
assert result["tags"][0]["name"] == "New Name" assert result["tags"][0]["name"] == "New Name"
@@ -218,9 +219,9 @@ class TestBuildTaxonomyCandidates:
document = DocumentFactory.create() document = DocumentFactory.create()
document.tags.add(tag) document.tags.add(tag)
tag.delete() tag.delete()
similar_documents = [make_similar(document.pk, 0.5)] nodes = [make_node(document.pk, 0.5)]
result = build_taxonomy_candidates(similar_documents, user=None) result = build_taxonomy_candidates(nodes, user=None)
assert result["tags"] == [] assert result["tags"] == []
@@ -239,12 +240,9 @@ class TestBuildTaxonomyCandidates:
strong_doc.tags.add(strong_tag) strong_doc.tags.add(strong_tag)
weak_doc = DocumentFactory.create() weak_doc = DocumentFactory.create()
weak_doc.tags.add(weak_tag) weak_doc.tags.add(weak_tag)
similar_documents = [ nodes = [make_node(strong_doc.pk, 0.9), make_node(weak_doc.pk, 0.1)]
make_similar(strong_doc.pk, 0.9),
make_similar(weak_doc.pk, 0.1),
]
result = build_taxonomy_candidates(similar_documents, user=None) result = build_taxonomy_candidates(nodes, user=None)
assert [c["name"] for c in result["tags"]] == ["Strong", "Weak"] assert [c["name"] for c in result["tags"]] == ["Strong", "Weak"]
@@ -260,9 +258,9 @@ class TestBuildTaxonomyCandidates:
document = DocumentFactory.create() document = DocumentFactory.create()
for i in range(15): for i in range(15):
document.tags.add(TagFactory.create(name=f"Tag{i}")) document.tags.add(TagFactory.create(name=f"Tag{i}"))
similar_documents = [make_similar(document.pk, 0.5)] nodes = [make_node(document.pk, 0.5)]
result = build_taxonomy_candidates(similar_documents, user=None) result = build_taxonomy_candidates(nodes, user=None)
assert len(result["tags"]) == 10 assert len(result["tags"]) == 10
@@ -276,12 +274,12 @@ class TestBuildTaxonomyCandidates:
- Only 5 correspondents are returned - Only 5 correspondents are returned
""" """
correspondents = CorrespondentFactory.create_batch(7) correspondents = CorrespondentFactory.create_batch(7)
similar_documents = [ nodes = [
make_similar(DocumentFactory.create(correspondent=c).pk, 0.5) make_node(DocumentFactory.create(correspondent=c).pk, 0.5)
for c in correspondents for c in correspondents
] ]
result = build_taxonomy_candidates(similar_documents, user=None) result = build_taxonomy_candidates(nodes, user=None)
assert len(result["correspondents"]) == 5 assert len(result["correspondents"]) == 5
@@ -296,9 +294,9 @@ class TestBuildTaxonomyCandidates:
""" """
document_type = DocumentTypeFactory.create(name="Invoice") document_type = DocumentTypeFactory.create(name="Invoice")
document = DocumentFactory.create(document_type=document_type) document = DocumentFactory.create(document_type=document_type)
similar_documents = [make_similar(document.pk, 0.5)] nodes = [make_node(document.pk, 0.5)]
result = build_taxonomy_candidates(similar_documents, user=None) result = build_taxonomy_candidates(nodes, user=None)
assert len(result["document_types"]) == 1 assert len(result["document_types"]) == 1
assert result["document_types"][0]["id"] == document_type.pk assert result["document_types"][0]["id"] == document_type.pk
@@ -314,12 +312,12 @@ class TestBuildTaxonomyCandidates:
- Only 5 document_types are returned - Only 5 document_types are returned
""" """
document_types = DocumentTypeFactory.create_batch(7) document_types = DocumentTypeFactory.create_batch(7)
similar_documents = [ nodes = [
make_similar(DocumentFactory.create(document_type=dt).pk, 0.5) make_node(DocumentFactory.create(document_type=dt).pk, 0.5)
for dt in document_types for dt in document_types
] ]
result = build_taxonomy_candidates(similar_documents, user=None) result = build_taxonomy_candidates(nodes, user=None)
assert len(result["document_types"]) == 5 assert len(result["document_types"]) == 5
@@ -334,9 +332,9 @@ class TestBuildTaxonomyCandidates:
""" """
storage_path = StoragePathFactory.create(name="Invoices") storage_path = StoragePathFactory.create(name="Invoices")
document = DocumentFactory.create(storage_path=storage_path) document = DocumentFactory.create(storage_path=storage_path)
similar_documents = [make_similar(document.pk, 0.5)] nodes = [make_node(document.pk, 0.5)]
result = build_taxonomy_candidates(similar_documents, user=None) result = build_taxonomy_candidates(nodes, user=None)
assert len(result["storage_paths"]) == 1 assert len(result["storage_paths"]) == 1
assert result["storage_paths"][0]["id"] == storage_path.pk assert result["storage_paths"][0]["id"] == storage_path.pk
@@ -352,12 +350,12 @@ class TestBuildTaxonomyCandidates:
- Only 5 storage_paths are returned - Only 5 storage_paths are returned
""" """
storage_paths = StoragePathFactory.create_batch(7) storage_paths = StoragePathFactory.create_batch(7)
similar_documents = [ nodes = [
make_similar(DocumentFactory.create(storage_path=sp).pk, 0.5) make_node(DocumentFactory.create(storage_path=sp).pk, 0.5)
for sp in storage_paths for sp in storage_paths
] ]
result = build_taxonomy_candidates(similar_documents, user=None) result = build_taxonomy_candidates(nodes, user=None)
assert len(result["storage_paths"]) == 5 assert len(result["storage_paths"]) == 5
@@ -377,14 +375,14 @@ class TestBuildTaxonomyCandidates:
tag = TagFactory.create(name="Restricted") tag = TagFactory.create(name="Restricted")
document = DocumentFactory.create() document = DocumentFactory.create()
document.tags.add(tag) document.tags.add(tag)
similar_documents = [make_similar(document.pk, 0.5)] nodes = [make_node(document.pk, 0.5)]
user = UserFactory.create() user = UserFactory.create()
mocker.patch( mocker.patch(
"documents.permissions.permitted_object_ids", "documents.permissions.permitted_object_ids",
return_value=[], # user cannot see this tag return_value=[], # user cannot see this tag
) )
result = build_taxonomy_candidates(similar_documents, user=user) result = build_taxonomy_candidates(nodes, user=user)
assert result["tags"] == [] assert result["tags"] == []
@@ -414,10 +412,10 @@ class TestBuildTaxonomyCandidates:
tag.save() tag.save()
document = DocumentFactory.create() document = DocumentFactory.create()
document.tags.add(tag) document.tags.add(tag)
similar_documents = [make_similar(document.pk, 0.5)] nodes = [make_node(document.pk, 0.5)]
spy = mocker.patch("documents.permissions.permitted_object_ids") spy = mocker.patch("documents.permissions.permitted_object_ids")
result = build_taxonomy_candidates(similar_documents, user=None) result = build_taxonomy_candidates(nodes, user=None)
assert result["tags"][0]["name"] == "Owned" assert result["tags"][0]["name"] == "Owned"
spy.assert_not_called() spy.assert_not_called()