amorfo77/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2026-08-02 09:02:18 +00:00
Code Issues Projects Releases Wiki Activity

Feature: Replace Whoosh with tantivy search backend (#12471)

Browse Source 
Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
Co-authored-by: Antoine Mérino <3023499+Merinorus@users.noreply.github.com>
This commit is contained in:
Trenton H
2026-04-02 12:38:22 -07:00
committed by GitHub
co-authored by Claude Sonnet 4.6 Antoine Mérino
parent e01a762e81
commit aed9abe48c
52 changed files with 4050 additions and 1708 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/documents/views.py
+121 -115
View File
@@ -100,7 +100,6 @@ from rest_framework.viewsets import ReadOnlyModelViewSet
from rest_framework.viewsets import ViewSet
from documents import bulk_edit
from documents import index
from documents.bulk_download import ArchiveOnlyStrategy
from documents.bulk_download import OriginalAndArchiveStrategy
from documents.bulk_download import OriginalsOnlyStrategy
@@ -1029,9 +1028,9 @@ class DocumentViewSet(
response_data["content"] = content_doc.content
response = Response(response_data)
from documents import index
from documents.search import get_backend
index.add_or_update_document(refreshed_doc)
get_backend().add_or_update(refreshed_doc)
document_updated.send(
sender=self.__class__,
@@ -1060,9 +1059,9 @@ class DocumentViewSet(
return Response({"results": serializer.data, "selection_data": selection_data})
def destroy(self, request, *args, **kwargs):
from documents import index
from documents.search import get_backend
index.remove_document_from_index(self.get_object())
get_backend().remove(self.get_object().pk)
try:
return super().destroy(request, *args, **kwargs)
except Exception as e:
@@ -1469,9 +1468,9 @@ class DocumentViewSet(
doc.modified = timezone.now()
doc.save()
from documents import index
from documents.search import get_backend
index.add_or_update_document(doc)
get_backend().add_or_update(doc)
notes = serializer.to_representation(doc).get("notes")
@@ -1506,9 +1505,9 @@ class DocumentViewSet(
doc.modified = timezone.now()
doc.save()
from documents import index
from documents.search import get_backend
index.add_or_update_document(doc)
get_backend().add_or_update(doc)
notes = serializer.to_representation(doc).get("notes")
@@ -1820,12 +1819,13 @@ class DocumentViewSet(
"Cannot delete the root/original version. Delete the document instead.",
)
from documents import index
from documents.search import get_backend
index.remove_document_from_index(version_doc)
_backend = get_backend()
_backend.remove(version_doc.pk)
version_doc_id = version_doc.id
version_doc.delete()
index.add_or_update_document(root_doc)
_backend.add_or_update(root_doc)
if settings.AUDIT_LOG_ENABLED:
actor = (
request.user if request.user and request.user.is_authenticated else None
@@ -2025,10 +2025,6 @@ class ChatStreamingView(GenericAPIView):
),
)
class UnifiedSearchViewSet(DocumentViewSet):
def __init__(self, *args: Any, **kwargs: Any) -> None:
super().__init__(*args, **kwargs)
self.searcher = None
def get_serializer_class(self):
if self._is_search_request():
return SearchResultSerializer
@@ -2041,17 +2037,34 @@ class UnifiedSearchViewSet(DocumentViewSet):
or "more_like_id" in self.request.query_params
)
def filter_queryset(self, queryset):
filtered_queryset = super().filter_queryset(queryset)
def list(self, request, *args, **kwargs):
if not self._is_search_request():
return super().list(request)
if self._is_search_request():
if "query" in self.request.query_params:
from documents import index
from documents.search import TantivyRelevanceList
from documents.search import get_backend
query_class = index.DelayedFullTextQuery
elif "more_like_id" in self.request.query_params:
try:
backend = get_backend()
# ORM-filtered queryset: permissions + field filters + ordering (DRF backends applied)
filtered_qs = self.filter_queryset(self.get_queryset())
user = None if request.user.is_superuser else request.user
if "query" in request.query_params:
query_str = request.query_params["query"]
results = backend.search(
query_str,
user=user,
page=1,
page_size=10000,
sort_field=None,
sort_reverse=False,
)
else:
# more_like_id — validate permission on the seed document first
try:
more_like_doc_id = int(self.request.query_params["more_like_id"])
more_like_doc_id = int(request.query_params["more_like_id"])
more_like_doc = Document.objects.select_related("owner").get(
pk=more_like_doc_id,
)
@@ -2059,76 +2072,71 @@ class UnifiedSearchViewSet(DocumentViewSet):
raise PermissionDenied(_("Invalid more_like_id"))
if not has_perms_owner_aware(
self.request.user,
request.user,
"view_document",
more_like_doc,
):
raise PermissionDenied(_("Insufficient permissions."))
from documents import index
query_class = index.DelayedMoreLikeThisQuery
else:
raise ValueError
return query_class(
self.searcher,
self.request.query_params,
self.paginator.get_page_size(self.request),
filter_queryset=filtered_queryset,
)
else:
return filtered_queryset
def list(self, request, *args, **kwargs):
if self._is_search_request():
from documents import index
try:
with index.open_index_searcher() as s:
self.searcher = s
queryset = self.filter_queryset(self.get_queryset())
page = self.paginate_queryset(queryset)
serializer = self.get_serializer(page, many=True)
response = self.get_paginated_response(serializer.data)
response.data["corrected_query"] = (
queryset.suggested_correction
if hasattr(queryset, "suggested_correction")
else None
)
if get_boolean(
str(
request.query_params.get(
"include_selection_data",
"false",
),
),
):
result_ids = queryset.get_result_ids()
response.data["selection_data"] = (
self._get_selection_data_for_queryset(
Document.objects.filter(pk__in=result_ids),
)
)
return response
except NotFound:
raise
except PermissionDenied as e:
invalid_more_like_id_message = _("Invalid more_like_id")
if str(e.detail) == str(invalid_more_like_id_message):
return HttpResponseForbidden(invalid_more_like_id_message)
return HttpResponseForbidden(_("Insufficient permissions."))
except Exception as e:
logger.warning(f"An error occurred listing search results: {e!s}")
return HttpResponseBadRequest(
"Error listing search results, check logs for more detail.",
results = backend.more_like_this(
more_like_doc_id,
user=user,
page=1,
page_size=10000,
)
else:
return super().list(request)
hits_by_id = {h["id"]: h for h in results.hits}
# Determine sort order: no ordering param -> Tantivy relevance; otherwise -> ORM order
ordering_param = request.query_params.get("ordering", "").lstrip("-")
if not ordering_param:
# Preserve Tantivy relevance order; intersect with ORM-visible IDs
orm_ids = set(filtered_qs.values_list("pk", flat=True))
ordered_hits = [h for h in results.hits if h["id"] in orm_ids]
else:
# Use ORM ordering (already applied by DocumentsOrderingFilter)
hit_ids = set(hits_by_id.keys())
orm_ordered_ids = filtered_qs.filter(id__in=hit_ids).values_list(
"pk",
flat=True,
)
ordered_hits = [
hits_by_id[pk] for pk in orm_ordered_ids if pk in hits_by_id
]
rl = TantivyRelevanceList(ordered_hits)
page = self.paginate_queryset(rl)
if page is not None:
serializer = self.get_serializer(page, many=True)
response = self.get_paginated_response(serializer.data)
response.data["corrected_query"] = None
if get_boolean(
str(request.query_params.get("include_selection_data", "false")),
):
all_ids = [h["id"] for h in ordered_hits]
response.data["selection_data"] = (
self._get_selection_data_for_queryset(
filtered_qs.filter(pk__in=all_ids),
)
)
return response
serializer = self.get_serializer(ordered_hits, many=True)
return Response(serializer.data)
except NotFound:
raise
except PermissionDenied as e:
invalid_more_like_id_message = _("Invalid more_like_id")
if str(e.detail) == str(invalid_more_like_id_message):
return HttpResponseForbidden(invalid_more_like_id_message)
return HttpResponseForbidden(_("Insufficient permissions."))
except Exception as e:
logger.warning(f"An error occurred listing search results: {e!s}")
return HttpResponseBadRequest(
"Error listing search results, check logs for more detail.",
)
@action(detail=False, methods=["GET"], name="Get Next ASN")
def next_asn(self, request, *args, **kwargs):
@@ -2946,18 +2954,9 @@ class SearchAutoCompleteView(GenericAPIView):
else:
limit = 10
from documents import index
from documents.search import get_backend
ix = index.open_index()
return Response(
index.autocomplete(
ix,
term,
limit,
user,
),
)
return Response(get_backend().autocomplete(term, limit, user))
@extend_schema_view(
@@ -3023,20 +3022,21 @@ class GlobalSearchView(PassUserMixin):
# First search by title
docs = all_docs.filter(title__icontains=query)
if not db_only and len(docs) < OBJECT_LIMIT:
# If we don't have enough results, search by content
from documents import index
# If we don't have enough results, search by content.
# Over-fetch from Tantivy (no permission filter) and rely on
# the ORM all_docs queryset for authoritative permission gating.
from documents.search import get_backend
with index.open_index_searcher() as s:
fts_query = index.DelayedFullTextQuery(
s,
request.query_params,
OBJECT_LIMIT,
filter_queryset=all_docs,
)
results = fts_query[0:1]
docs = docs | Document.objects.filter(
id__in=[r["id"] for r in results],
)
fts_results = get_backend().search(
query,
user=None,
page=1,
page_size=1000,
sort_field=None,
sort_reverse=False,
)
fts_ids = {h["id"] for h in fts_results.hits}
docs = docs | all_docs.filter(id__in=fts_ids)
docs = docs[:OBJECT_LIMIT]
saved_views = (
get_objects_for_user_owner_aware(
@@ -4279,10 +4279,16 @@ class SystemStatusView(PassUserMixin):
index_error = None
try:
ix = index.open_index()
from documents.search import get_backend
get_backend() # triggers open/rebuild; raises on error
index_status = "OK"
index_last_modified = make_aware(
datetime.fromtimestamp(ix.last_modified()),
# Use the most-recently modified file in the index directory as a proxy
# for last index write time (Tantivy has no single last_modified() call).
index_dir = settings.INDEX_DIR
mtimes = [p.stat().st_mtime for p in index_dir.iterdir() if p.is_file()]
index_last_modified = (
make_aware(datetime.fromtimestamp(max(mtimes))) if mtimes else None
)
except Exception as e:
index_status = "ERROR"