feat(search): add whoosh-compat FieldRegistry construction

src/documents/search/_registry.py

@@ -0,0 +1,64 @@
+from __future__ import annotations
+
+from whoosh_compat import FieldKind
+from whoosh_compat import FieldRegistry
+from whoosh_compat import FieldSpec
+
+from documents.search._fields import PUBLIC_FIELDS
+from documents.search._tokenizer import ascii_fold
+from documents.search._tokenizer import paperless_text_analyzer
+
+_registry_cache: dict[str | None, FieldRegistry] = {}
+
+
+def _identity_analyzer(text: str) -> list[str]:
+    """Analyzer for KEYWORD fields indexed with the raw tokenizer (no splitting)."""
+    return [text]
+
+
+def _pattern_normalizer(text: str) -> str:
+    """Normalize wildcard/regex query patterns: lowercase -> ascii_fold.
+
+    Mirrors the lowercase -> ascii_fold steps of the index-time analyzers
+    (paperless_text) without stemming, so pattern queries (e.g. "run*")
+    match tokens that were folded the same way at index time but are not
+    run through a stemmer, which would corrupt wildcard/regex semantics.
+    """
+    return ascii_fold(text.lower())
+
+
+def get_field_registry(language: str | None) -> FieldRegistry:
+    """Build (or return the cached) FieldRegistry for the given search language.
+
+    Cached keyed by language, rebuilt on the same trigger register_tokenizers()
+    uses (settings.SEARCH_LANGUAGE change) — a fresh call with a new language
+    builds and caches a new registry rather than mutating the old one.
+    """
+    if language in _registry_cache:
+        return _registry_cache[language]
+
+    text_analyzer = paperless_text_analyzer(language).analyze
+
+    specs = []
+    for field in PUBLIC_FIELDS:
+        if field.kind is FieldKind.KEYWORD:
+            analyzer = _identity_analyzer
+        else:
+            analyzer = text_analyzer
+        specs.append(
+            FieldSpec(
+                name=field.name,
+                kind=field.kind,
+                aliases=field.aliases,
+                comma_values=field.comma_values,
+                analyzer=analyzer,
+                pattern_normalizer=_pattern_normalizer,
+                date_only=field.date_only,
+                fast=field.fast,
+                subpaths=field.subpaths,
+            ),
+        )
+
+    registry = FieldRegistry(specs)
+    _registry_cache[language] = registry
+    return registry

src/documents/search/_tokenizer.py

@@ -71,7 +71,7 @@ def register_tokenizers(index: tantivy.Index, language: str | None) -> None:
         use fast=True and Tantivy requires fast-field tokenizers to exist
         even for documents that omit those fields.
     """
-    index.register_tokenizer("paperless_text", _paperless_text(language))
+    index.register_tokenizer("paperless_text", paperless_text_analyzer(language))
     index.register_tokenizer("simple_analyzer", _simple_analyzer())
     index.register_tokenizer("bigram_analyzer", _bigram_analyzer())
     index.register_tokenizer("simple_search_analyzer", _simple_search_analyzer())
@@ -79,7 +79,7 @@ def register_tokenizers(index: tantivy.Index, language: str | None) -> None:
     index.register_fast_field_tokenizer("simple_analyzer", _simple_analyzer())
 
 
-def _paperless_text(language: str | None) -> tantivy.TextAnalyzer:
+def paperless_text_analyzer(language: str | None) -> tantivy.TextAnalyzer:
     """Main full-text tokenizer for content, title, etc: simple -> remove_long(129) -> lowercase -> ascii_fold [-> stemmer]"""
     builder = (
         tantivy.TextAnalyzerBuilder(tantivy.Tokenizer.simple())

src/documents/tests/search/test_registry.py

@@ -0,0 +1,122 @@
+from whoosh_compat import FieldKind
+
+from documents.search._registry import get_field_registry
+
+
+class TestFieldRegistry:
+    def test_internal_id_fields_are_not_registered(self) -> None:
+        registry = get_field_registry(None)
+        for name in (
+            "tag_id",
+            "owner_id",
+            "viewer_id",
+            "correspondent_id",
+            "document_type_id",
+            "storage_path_id",
+            "viewer_group_id",
+        ):
+            assert name not in registry
+
+    def test_type_alias_resolves_to_document_type(self) -> None:
+        registry = get_field_registry(None)
+        ref = registry.make_ref("type")
+        assert ref is not None
+        resolved = registry.resolve(ref)
+        assert resolved is not None
+        assert resolved.spec.name == "document_type"
+
+    def test_path_alias_resolves_to_storage_path(self) -> None:
+        registry = get_field_registry(None)
+        ref = registry.make_ref("path")
+        assert ref is not None
+        resolved = registry.resolve(ref)
+        assert resolved is not None
+        assert resolved.spec.name == "storage_path"
+
+    def test_notes_json_subpaths_resolve(self) -> None:
+        registry = get_field_registry(None)
+        ref = registry.make_ref("notes.user")
+        assert ref is not None
+        resolved = registry.resolve(ref)
+        assert resolved is not None
+        assert resolved.spec.name == "notes"
+        assert resolved.json_path == "user"
+        assert resolved.is_subpath is True
+
+    def test_custom_fields_json_subpaths_resolve(self) -> None:
+        registry = get_field_registry(None)
+        for raw in ("custom_fields.name", "custom_fields.value"):
+            ref = registry.make_ref(raw)
+            assert ref is not None
+            assert registry.resolve(ref) is not None
+
+    def test_unregistered_json_subpath_does_not_resolve(self) -> None:
+        registry = get_field_registry(None)
+        # An unregistered subpath is not even a valid FieldRef: make_ref
+        # returns None for a dotted name whose subpath isn't registered
+        # (it doesn't produce a ref for resolve() to then reject).
+        assert registry.make_ref("notes.bogus") is None
+
+    def test_tag_is_comma_values(self) -> None:
+        registry = get_field_registry(None)
+        ref = registry.make_ref("tag")
+        assert ref is not None
+        resolved = registry.resolve(ref)
+        assert resolved is not None
+        assert resolved.spec.comma_values is True
+
+    def test_created_is_date_kind(self) -> None:
+        registry = get_field_registry(None)
+        ref = registry.make_ref("created")
+        assert ref is not None
+        resolved = registry.resolve(ref)
+        assert resolved is not None
+        assert resolved.spec.kind is FieldKind.DATE
+        assert resolved.spec.date_only is True
+
+    def test_analyzer_lowercases_and_ascii_folds(self) -> None:
+        # title uses the paperless_text analyzer: simple -> remove_long ->
+        # lowercase -> ascii_fold [-> stemmer]. With no language configured
+        # (None), no stemmer runs, so "Café" folds to the single token "cafe".
+        registry = get_field_registry(None)
+        ref = registry.make_ref("title")
+        assert ref is not None
+        resolved = registry.resolve(ref)
+        assert resolved is not None
+        assert resolved.spec.analyzer is not None
+        assert resolved.spec.analyzer("Café") == ["cafe"]
+
+    def test_checksum_analyzer_is_identity_single_token(self) -> None:
+        # checksum uses the raw tokenizer at index time (no splitting).
+        registry = get_field_registry(None)
+        ref = registry.make_ref("checksum")
+        assert ref is not None
+        resolved = registry.resolve(ref)
+        assert resolved is not None
+        assert resolved.spec.analyzer("ABC-123") == ["ABC-123"]
+
+    def test_pattern_normalizer_is_ascii_fold_only_no_stemming(self) -> None:
+        registry = get_field_registry(None)
+        ref = registry.make_ref("title")
+        assert ref is not None
+        resolved = registry.resolve(ref)
+        assert resolved is not None
+        assert resolved.spec.pattern_normalizer is not None
+        # "running" must NOT be stemmed to "run" by the pattern normalizer,
+        # only case/accent-folded — even with English stemming configured.
+        registry_en = get_field_registry("en")
+        ref_en = registry_en.make_ref("title")
+        assert ref_en is not None
+        resolved_en = registry_en.resolve(ref_en)
+        assert resolved_en is not None
+        assert resolved_en.spec.pattern_normalizer("Running") == "running"
+
+    def test_registry_is_cached_per_language(self) -> None:
+        a = get_field_registry("en")
+        b = get_field_registry("en")
+        assert a is b
+
+    def test_registry_rebuilds_on_language_change(self) -> None:
+        a = get_field_registry("en")
+        b = get_field_registry("de")
+        assert a is not b

src/documents/tests/search/test_tokenizer.py

@@ -7,8 +7,8 @@ import pytest
 import tantivy
 
 from documents.search._tokenizer import _bigram_analyzer
-from documents.search._tokenizer import _paperless_text
 from documents.search._tokenizer import _simple_search_analyzer
+from documents.search._tokenizer import paperless_text_analyzer
 from documents.search._tokenizer import register_tokenizers
 
 if TYPE_CHECKING:
@@ -25,7 +25,7 @@ class TestTokenizers:
         sb.add_text_field("content", stored=True, tokenizer_name="paperless_text")
         schema = sb.build()
         idx = tantivy.Index(schema, path=None)
-        idx.register_tokenizer("paperless_text", _paperless_text(""))
+        idx.register_tokenizer("paperless_text", paperless_text_analyzer(""))
         return idx
 
     @pytest.fixture