Enhancement: unify text search to use tantivy (#12485)

src/documents/search/__init__.py

@@ -1,4 +1,5 @@
 from documents.search._backend import SearchIndexLockError
+from documents.search._backend import SearchMode
 from documents.search._backend import SearchResults
 from documents.search._backend import TantivyBackend
 from documents.search._backend import TantivyRelevanceList
@@ -10,6 +11,7 @@ from documents.search._schema import wipe_index
 
 __all__ = [
     "SearchIndexLockError",
+    "SearchMode",
     "SearchResults",
     "TantivyBackend",
     "TantivyRelevanceList",

src/documents/search/_backend.py

@@ -2,11 +2,11 @@ from __future__ import annotations
 
 import logging
 import threading
-import unicodedata
 from collections import Counter
 from dataclasses import dataclass
 from datetime import UTC
 from datetime import datetime
+from enum import StrEnum
 from typing import TYPE_CHECKING
 from typing import Self
 from typing import TypedDict
@@ -19,7 +19,10 @@ from django.conf import settings
 from django.utils.timezone import get_current_timezone
 from guardian.shortcuts import get_users_with_perms
 
+from documents.search._normalize import ascii_fold
 from documents.search._query import build_permission_filter
+from documents.search._query import parse_simple_text_query
+from documents.search._query import parse_simple_title_query
 from documents.search._query import parse_user_query
 from documents.search._schema import _write_sentinels
 from documents.search._schema import build_schema
@@ -45,14 +48,10 @@ _AUTOCOMPLETE_REGEX_TIMEOUT = 1.0  # seconds; guards against ReDoS on untrusted
 T = TypeVar("T")
 
 
-def _ascii_fold(s: str) -> str:
-    """
-    Normalize unicode to ASCII equivalent characters for search consistency.
-
-    Converts accented characters (e.g., "café") to their ASCII base forms ("cafe")
-    to enable cross-language searching without requiring exact diacritic matching.
-    """
-    return unicodedata.normalize("NFD", s).encode("ascii", "ignore").decode()
+class SearchMode(StrEnum):
+    QUERY = "query"
+    TEXT = "text"
+    TITLE = "title"
 
 
 def _extract_autocomplete_words(text_sources: list[str]) -> set[str]:
@@ -74,7 +73,7 @@ def _extract_autocomplete_words(text_sources: list[str]) -> set[str]:
             )
             continue
         for token in tokens:
-            normalized = _ascii_fold(token.lower())
+            normalized = ascii_fold(token.lower())
             if normalized:
                 words.add(normalized)
     return words
@@ -294,8 +293,10 @@ class TantivyBackend:
         doc.add_text("checksum", document.checksum)
         doc.add_text("title", document.title)
         doc.add_text("title_sort", document.title)
+        doc.add_text("simple_title", document.title)
         doc.add_text("content", content)
         doc.add_text("bigram_content", content)
+        doc.add_text("simple_content", content)
 
         # Original filename - only add if not None/empty
         if document.original_filename:
@@ -433,6 +434,7 @@ class TantivyBackend:
         sort_field: str | None,
         *,
         sort_reverse: bool,
+        search_mode: SearchMode = SearchMode.QUERY,
     ) -> SearchResults:
         """
         Execute a search query against the document index.
@@ -441,20 +443,32 @@ class TantivyBackend:
         permission filtering before executing against Tantivy. Supports both
         relevance-based and field-based sorting.
 
+        QUERY search mode supports natural date keywords, field filters, etc.
+        TITLE search mode treats the query as plain text to search for in title only
+        TEXT search mode treats the query as plain text to search for in title and content
+
         Args:
-            query: User's search query (supports natural date keywords, field filters)
+            query: User's search query
             user: User for permission filtering (None for superuser/no filtering)
             page: Page number (1-indexed) for pagination
             page_size: Number of results per page
             sort_field: Field to sort by (None for relevance ranking)
             sort_reverse: Whether to reverse the sort order
+            search_mode: "query" for advanced Tantivy syntax, "text" for
+                plain-text search over title and content only, "title" for
+                plain-text search over title only
 
         Returns:
             SearchResults with hits, total count, and processed query
         """
         self._ensure_open()
         tz = get_current_timezone()
-        user_query = parse_user_query(self._index, query, tz)
+        if search_mode is SearchMode.TEXT:
+            user_query = parse_simple_text_query(self._index, query)
+        elif search_mode is SearchMode.TITLE:
+            user_query = parse_simple_title_query(self._index, query)
+        else:
+            user_query = parse_user_query(self._index, query, tz)
 
         # Apply permission filter if user is not None (not superuser)
         if user is not None:
@@ -594,7 +608,7 @@ class TantivyBackend:
             List of word suggestions ordered by frequency, then alphabetically
         """
         self._ensure_open()
-        normalized_term = _ascii_fold(term.lower())
+        normalized_term = ascii_fold(term.lower())
 
         searcher = self._index.searcher()
 

src/documents/search/_normalize.py

@@ -0,0 +1,8 @@
+from __future__ import annotations
+
+import unicodedata
+
+
+def ascii_fold(text: str) -> str:
+    """Normalize unicode text to ASCII equivalents for search consistency."""
+    return unicodedata.normalize("NFD", text).encode("ascii", "ignore").decode()

src/documents/search/_query.py

@@ -12,6 +12,8 @@ import tantivy
 from dateutil.relativedelta import relativedelta
 from django.conf import settings
 
+from documents.search._normalize import ascii_fold
+
 if TYPE_CHECKING:
     from datetime import tzinfo
 
@@ -51,6 +53,7 @@ _WHOOSH_REL_RANGE_RE = regex.compile(
 )
 # Whoosh-style 8-digit date: field:YYYYMMDD — field-aware so timezone can be applied correctly
 _DATE8_RE = regex.compile(r"(?P<field>\w+):(?P<date8>\d{8})\b")
+_SIMPLE_QUERY_TOKEN_RE = regex.compile(r"\S+")
 
 
 def _fmt(dt: datetime) -> str:
@@ -436,7 +439,37 @@ DEFAULT_SEARCH_FIELDS = [
     "document_type",
     "tag",
 ]
+SIMPLE_SEARCH_FIELDS = ["simple_title", "simple_content"]
+TITLE_SEARCH_FIELDS = ["simple_title"]
 _FIELD_BOOSTS = {"title": 2.0}
+_SIMPLE_FIELD_BOOSTS = {"simple_title": 2.0}
+
+
+def _build_simple_field_query(
+    index: tantivy.Index,
+    field: str,
+    tokens: list[str],
+) -> tantivy.Query:
+    patterns = []
+    for idx, token in enumerate(tokens):
+        escaped = regex.escape(token)
+        # For multi-token substring search, only the first token can begin mid-word.
+        # Later tokens follow a whitespace boundary in the original query, so anchor
+        # them to the start of the next indexed token to reduce false positives like
+        # matching "Z-Berichte 16" for the query "Z-Berichte 6".
+        if idx == 0:
+            patterns.append(f".*{escaped}.*")
+        else:
+            patterns.append(f"{escaped}.*")
+    if len(patterns) == 1:
+        query = tantivy.Query.regex_query(index.schema, field, patterns[0])
+    else:
+        query = tantivy.Query.regex_phrase_query(index.schema, field, patterns)
+
+    boost = _SIMPLE_FIELD_BOOSTS.get(field, 1.0)
+    if boost > 1.0:
+        return tantivy.Query.boost_query(query, boost)
+    return query
 
 
 def parse_user_query(
@@ -495,3 +528,52 @@ def parse_user_query(
         )
 
     return exact
+
+
+def parse_simple_query(
+    index: tantivy.Index,
+    raw_query: str,
+    fields: list[str],
+) -> tantivy.Query:
+    """
+    Parse a plain-text query using Tantivy over a restricted field set.
+
+    Query string is escaped and normalized to be treated as "simple" text query.
+    """
+    tokens = [
+        ascii_fold(token.lower())
+        for token in _SIMPLE_QUERY_TOKEN_RE.findall(raw_query, timeout=_REGEX_TIMEOUT)
+    ]
+    tokens = [token for token in tokens if token]
+    if not tokens:
+        return tantivy.Query.empty_query()
+
+    field_queries = [
+        (tantivy.Occur.Should, _build_simple_field_query(index, field, tokens))
+        for field in fields
+    ]
+    if len(field_queries) == 1:
+        return field_queries[0][1]
+    return tantivy.Query.boolean_query(field_queries)
+
+
+def parse_simple_text_query(
+    index: tantivy.Index,
+    raw_query: str,
+) -> tantivy.Query:
+    """
+    Parse a plain-text query over title/content for simple search inputs.
+    """
+
+    return parse_simple_query(index, raw_query, SIMPLE_SEARCH_FIELDS)
+
+
+def parse_simple_title_query(
+    index: tantivy.Index,
+    raw_query: str,
+) -> tantivy.Query:
+    """
+    Parse a plain-text query over the title field only.
+    """
+
+    return parse_simple_query(index, raw_query, TITLE_SEARCH_FIELDS)

src/documents/search/_schema.py

@@ -53,6 +53,18 @@ def build_schema() -> tantivy.Schema:
     # CJK support - not stored, indexed only
     sb.add_text_field("bigram_content", stored=False, tokenizer_name="bigram_analyzer")
 
+    # Simple substring search support for title/content - not stored, indexed only
+    sb.add_text_field(
+        "simple_title",
+        stored=False,
+        tokenizer_name="simple_search_analyzer",
+    )
+    sb.add_text_field(
+        "simple_content",
+        stored=False,
+        tokenizer_name="simple_search_analyzer",
+    )
+
     # Autocomplete prefix scan - stored, not indexed
     sb.add_text_field("autocomplete_word", stored=True, tokenizer_name="raw")
 

src/documents/search/_tokenizer.py

@@ -70,6 +70,7 @@ def register_tokenizers(index: tantivy.Index, language: str | None) -> None:
     index.register_tokenizer("paperless_text", _paperless_text(language))
     index.register_tokenizer("simple_analyzer", _simple_analyzer())
     index.register_tokenizer("bigram_analyzer", _bigram_analyzer())
+    index.register_tokenizer("simple_search_analyzer", _simple_search_analyzer())
     # Fast-field tokenizer required for fast=True text fields in the schema
     index.register_fast_field_tokenizer("simple_analyzer", _simple_analyzer())
 
@@ -114,3 +115,16 @@ def _bigram_analyzer() -> tantivy.TextAnalyzer:
         .filter(tantivy.Filter.lowercase())
         .build()
     )
+
+
+def _simple_search_analyzer() -> tantivy.TextAnalyzer:
+    """Tokenizer for simple substring search fields: non-whitespace chunks -> remove_long(65) -> lowercase -> ascii_fold."""
+    return (
+        tantivy.TextAnalyzerBuilder(
+            tantivy.Tokenizer.regex(r"\S+"),
+        )
+        .filter(tantivy.Filter.remove_long(65))
+        .filter(tantivy.Filter.lowercase())
+        .filter(tantivy.Filter.ascii_fold())
+        .build()
+    )