Fixes copilot found issues, try to tune the filtering as suggested

src/documents/search/_backend.py

@@ -132,9 +132,9 @@ class TantivyRelevanceList:
         stop = key.stop or len(self._ordered_ids)
         # DRF slices to extract the current page.  If the slice aligns
         # with our pre-fetched page_hits, return them directly.
-        if start == self._page_offset and stop <= self._page_offset + len(
-            self._page_hits,
-        ):
+        # We only check start — DRF always slices with stop=start+page_size,
+        # which exceeds page_hits length on the last page.
+        if start == self._page_offset:
             return self._page_hits[: stop - start]
         # Fallback: return stub dicts (no highlights).
         return [
@@ -507,6 +507,7 @@ class TantivyBackend:
         doc_ids: list[int],
         *,
         search_mode: SearchMode = SearchMode.QUERY,
+        rank_start: int = 1,
     ) -> list[SearchHit]:
         """
         Generate SearchHit dicts with highlights for specific document IDs.
@@ -525,6 +526,8 @@ class TantivyBackend:
             query: The search query (used for snippet generation)
             doc_ids: Ordered list of document IDs to generate hits for
             search_mode: Query parsing mode (for building the snippet query)
+            rank_start: Starting rank value (1-based absolute position in the
+                full result set; pass ``page_offset + 1`` for paginated calls)
 
         Returns:
             List of SearchHit dicts in the same order as doc_ids
@@ -540,8 +543,9 @@ class TantivyBackend:
         notes_snippet_generator = None
         hits: list[SearchHit] = []
 
-        for rank, doc_id in enumerate(doc_ids, start=1):
-            # Look up document by ID
+        for rank, doc_id in enumerate(doc_ids, start=rank_start):
+            # Look up document by ID, scoring against the user query so that
+            # the returned SearchHit carries a real BM25 relevance score.
             id_query = tantivy.Query.range_query(
                 self._schema,
                 "id",
@@ -549,12 +553,18 @@ class TantivyBackend:
                 doc_id,
                 doc_id,
             )
-            results = searcher.search(id_query, limit=1)
+            scored_query = tantivy.Query.boolean_query(
+                [
+                    (tantivy.Occur.Must, user_query),
+                    (tantivy.Occur.Must, id_query),
+                ],
+            )
+            results = searcher.search(scored_query, limit=1)
 
             if not results.hits:
                 continue
 
-            doc_address = results.hits[0][1]
+            score, doc_address = results.hits[0]
             actual_doc = searcher.doc(doc_address)
             doc_dict = actual_doc.to_dict()
 
@@ -590,7 +600,7 @@ class TantivyBackend:
             hits.append(
                 SearchHit(
                     id=doc_id,
-                    score=0.0,
+                    score=score,
                     rank=rank,
                     highlights=highlights,
                 ),
@@ -684,6 +694,8 @@ class TantivyBackend:
         """
         self._ensure_open()
         normalized_term = ascii_fold(term.lower())
+        if not normalized_term:
+            return []
 
         searcher = self._index.searcher()
 

src/documents/views.py

@@ -247,6 +247,13 @@ if settings.AUDIT_LOG_ENABLED:
 
 logger = logging.getLogger("paperless.api")
 
+# Crossover point for intersect_and_order: below this count use a targeted
+# IN-clause query; at or above this count fall back to a full-table scan +
+# Python set intersection.  The IN-clause is faster for small result sets but
+# degrades on SQLite with thousands of parameters.  PostgreSQL handles large IN
+# clauses efficiently, so this threshold mainly protects SQLite users.
+_TANTIVY_INTERSECT_THRESHOLD = 5_000
+
 
 class IndexView(TemplateView):
     template_name = "index.html"
@@ -2089,12 +2096,9 @@ class UnifiedSearchViewSet(DocumentViewSet):
                 page_num = int(request.query_params.get("page", 1))
             except (TypeError, ValueError):
                 page_num = 1
-            try:
-                page_size = int(
-                    request.query_params.get("page_size", self.paginator.page_size),
-                )
-            except (TypeError, ValueError):
-                page_size = self.paginator.page_size
+            page_size = (
+                self.paginator.get_page_size(request) or self.paginator.page_size
+            )
 
             return sort_field_name, sort_reverse, use_tantivy_sort, page_num, page_size
 
@@ -2105,12 +2109,23 @@ class UnifiedSearchViewSet(DocumentViewSet):
             use_tantivy_sort: bool,
         ) -> list[int]:
             """Intersect search IDs with ORM-visible IDs, preserving order."""
-            orm_ids = set(filtered_qs.values_list("pk", flat=True))
+            if not all_ids:
+                return []
             if use_tantivy_sort:
-                return [doc_id for doc_id in all_ids if doc_id in orm_ids]
-            id_set = set(all_ids) & orm_ids
+                if len(all_ids) <= _TANTIVY_INTERSECT_THRESHOLD:
+                    # Small result set: targeted IN-clause avoids a full-table scan.
+                    visible_ids = set(
+                        filtered_qs.filter(pk__in=all_ids).values_list("pk", flat=True),
+                    )
+                else:
+                    # Large result set: full-table scan + Python intersection is faster
+                    # than a large IN-clause on SQLite.
+                    visible_ids = set(
+                        filtered_qs.values_list("pk", flat=True),
+                    )
+                return [doc_id for doc_id in all_ids if doc_id in visible_ids]
             return list(
-                filtered_qs.filter(id__in=id_set).values_list("pk", flat=True),
+                filtered_qs.filter(id__in=all_ids).values_list("pk", flat=True),
             )
 
         def run_text_search(
@@ -2148,6 +2163,7 @@ class UnifiedSearchViewSet(DocumentViewSet):
                 query_str,
                 page_ids,
                 search_mode=search_mode,
+                rank_start=page_offset + 1,
             )
             return ordered_ids, page_hits, page_offset