Enhancement: unify text search to use tantivy (#12485)

src/documents/search/_tokenizer.py

@@ -70,6 +70,7 @@ def register_tokenizers(index: tantivy.Index, language: str | None) -> None:
     index.register_tokenizer("paperless_text", _paperless_text(language))
     index.register_tokenizer("simple_analyzer", _simple_analyzer())
     index.register_tokenizer("bigram_analyzer", _bigram_analyzer())
+    index.register_tokenizer("simple_search_analyzer", _simple_search_analyzer())
     # Fast-field tokenizer required for fast=True text fields in the schema
     index.register_fast_field_tokenizer("simple_analyzer", _simple_analyzer())
 
@@ -114,3 +115,16 @@ def _bigram_analyzer() -> tantivy.TextAnalyzer:
         .filter(tantivy.Filter.lowercase())
         .build()
     )
+
+
+def _simple_search_analyzer() -> tantivy.TextAnalyzer:
+    """Tokenizer for simple substring search fields: non-whitespace chunks -> remove_long(65) -> lowercase -> ascii_fold."""
+    return (
+        tantivy.TextAnalyzerBuilder(
+            tantivy.Tokenizer.regex(r"\S+"),
+        )
+        .filter(tantivy.Filter.remove_long(65))
+        .filter(tantivy.Filter.lowercase())
+        .filter(tantivy.Filter.ascii_fold())
+        .build()
+    )