diff --git a/src/documents/search/_backend.py b/src/documents/search/_backend.py index df2b73f7d..476315d57 100644 --- a/src/documents/search/_backend.py +++ b/src/documents/search/_backend.py @@ -132,9 +132,9 @@ class TantivyRelevanceList: stop = key.stop or len(self._ordered_ids) # DRF slices to extract the current page. If the slice aligns # with our pre-fetched page_hits, return them directly. - if start == self._page_offset and stop <= self._page_offset + len( - self._page_hits, - ): + # We only check start — DRF always slices with stop=start+page_size, + # which exceeds page_hits length on the last page. + if start == self._page_offset: return self._page_hits[: stop - start] # Fallback: return stub dicts (no highlights). return [ @@ -507,6 +507,7 @@ class TantivyBackend: doc_ids: list[int], *, search_mode: SearchMode = SearchMode.QUERY, + rank_start: int = 1, ) -> list[SearchHit]: """ Generate SearchHit dicts with highlights for specific document IDs. @@ -525,6 +526,8 @@ class TantivyBackend: query: The search query (used for snippet generation) doc_ids: Ordered list of document IDs to generate hits for search_mode: Query parsing mode (for building the snippet query) + rank_start: Starting rank value (1-based absolute position in the + full result set; pass ``page_offset + 1`` for paginated calls) Returns: List of SearchHit dicts in the same order as doc_ids @@ -540,8 +543,9 @@ class TantivyBackend: notes_snippet_generator = None hits: list[SearchHit] = [] - for rank, doc_id in enumerate(doc_ids, start=1): - # Look up document by ID + for rank, doc_id in enumerate(doc_ids, start=rank_start): + # Look up document by ID, scoring against the user query so that + # the returned SearchHit carries a real BM25 relevance score. id_query = tantivy.Query.range_query( self._schema, "id", @@ -549,12 +553,18 @@ class TantivyBackend: doc_id, doc_id, ) - results = searcher.search(id_query, limit=1) + scored_query = tantivy.Query.boolean_query( + [ + (tantivy.Occur.Must, user_query), + (tantivy.Occur.Must, id_query), + ], + ) + results = searcher.search(scored_query, limit=1) if not results.hits: continue - doc_address = results.hits[0][1] + score, doc_address = results.hits[0] actual_doc = searcher.doc(doc_address) doc_dict = actual_doc.to_dict() @@ -590,7 +600,7 @@ class TantivyBackend: hits.append( SearchHit( id=doc_id, - score=0.0, + score=score, rank=rank, highlights=highlights, ), @@ -684,6 +694,8 @@ class TantivyBackend: """ self._ensure_open() normalized_term = ascii_fold(term.lower()) + if not normalized_term: + return [] searcher = self._index.searcher() diff --git a/src/documents/views.py b/src/documents/views.py index 250aaad32..2fa1491a5 100644 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -247,6 +247,13 @@ if settings.AUDIT_LOG_ENABLED: logger = logging.getLogger("paperless.api") +# Crossover point for intersect_and_order: below this count use a targeted +# IN-clause query; at or above this count fall back to a full-table scan + +# Python set intersection. The IN-clause is faster for small result sets but +# degrades on SQLite with thousands of parameters. PostgreSQL handles large IN +# clauses efficiently, so this threshold mainly protects SQLite users. +_TANTIVY_INTERSECT_THRESHOLD = 5_000 + class IndexView(TemplateView): template_name = "index.html" @@ -2089,12 +2096,9 @@ class UnifiedSearchViewSet(DocumentViewSet): page_num = int(request.query_params.get("page", 1)) except (TypeError, ValueError): page_num = 1 - try: - page_size = int( - request.query_params.get("page_size", self.paginator.page_size), - ) - except (TypeError, ValueError): - page_size = self.paginator.page_size + page_size = ( + self.paginator.get_page_size(request) or self.paginator.page_size + ) return sort_field_name, sort_reverse, use_tantivy_sort, page_num, page_size @@ -2105,12 +2109,23 @@ class UnifiedSearchViewSet(DocumentViewSet): use_tantivy_sort: bool, ) -> list[int]: """Intersect search IDs with ORM-visible IDs, preserving order.""" - orm_ids = set(filtered_qs.values_list("pk", flat=True)) + if not all_ids: + return [] if use_tantivy_sort: - return [doc_id for doc_id in all_ids if doc_id in orm_ids] - id_set = set(all_ids) & orm_ids + if len(all_ids) <= _TANTIVY_INTERSECT_THRESHOLD: + # Small result set: targeted IN-clause avoids a full-table scan. + visible_ids = set( + filtered_qs.filter(pk__in=all_ids).values_list("pk", flat=True), + ) + else: + # Large result set: full-table scan + Python intersection is faster + # than a large IN-clause on SQLite. + visible_ids = set( + filtered_qs.values_list("pk", flat=True), + ) + return [doc_id for doc_id in all_ids if doc_id in visible_ids] return list( - filtered_qs.filter(id__in=id_set).values_list("pk", flat=True), + filtered_qs.filter(id__in=all_ids).values_list("pk", flat=True), ) def run_text_search( @@ -2148,6 +2163,7 @@ class UnifiedSearchViewSet(DocumentViewSet): query_str, page_ids, search_mode=search_mode, + rank_start=page_offset + 1, ) return ordered_ids, page_hits, page_offset