diff --git a/src/documents/search/_registry.py b/src/documents/search/_registry.py new file mode 100644 index 000000000..2a4a98d26 --- /dev/null +++ b/src/documents/search/_registry.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +from whoosh_compat import FieldKind +from whoosh_compat import FieldRegistry +from whoosh_compat import FieldSpec + +from documents.search._fields import PUBLIC_FIELDS +from documents.search._tokenizer import ascii_fold +from documents.search._tokenizer import paperless_text_analyzer + +_registry_cache: dict[str | None, FieldRegistry] = {} + + +def _identity_analyzer(text: str) -> list[str]: + """Analyzer for KEYWORD fields indexed with the raw tokenizer (no splitting).""" + return [text] + + +def _pattern_normalizer(text: str) -> str: + """Normalize wildcard/regex query patterns: lowercase -> ascii_fold. + + Mirrors the lowercase -> ascii_fold steps of the index-time analyzers + (paperless_text) without stemming, so pattern queries (e.g. "run*") + match tokens that were folded the same way at index time but are not + run through a stemmer, which would corrupt wildcard/regex semantics. + """ + return ascii_fold(text.lower()) + + +def get_field_registry(language: str | None) -> FieldRegistry: + """Build (or return the cached) FieldRegistry for the given search language. + + Cached keyed by language, rebuilt on the same trigger register_tokenizers() + uses (settings.SEARCH_LANGUAGE change) — a fresh call with a new language + builds and caches a new registry rather than mutating the old one. + """ + if language in _registry_cache: + return _registry_cache[language] + + text_analyzer = paperless_text_analyzer(language).analyze + + specs = [] + for field in PUBLIC_FIELDS: + if field.kind is FieldKind.KEYWORD: + analyzer = _identity_analyzer + else: + analyzer = text_analyzer + specs.append( + FieldSpec( + name=field.name, + kind=field.kind, + aliases=field.aliases, + comma_values=field.comma_values, + analyzer=analyzer, + pattern_normalizer=_pattern_normalizer, + date_only=field.date_only, + fast=field.fast, + subpaths=field.subpaths, + ), + ) + + registry = FieldRegistry(specs) + _registry_cache[language] = registry + return registry diff --git a/src/documents/search/_tokenizer.py b/src/documents/search/_tokenizer.py index c84dd9093..11c2f5ba4 100644 --- a/src/documents/search/_tokenizer.py +++ b/src/documents/search/_tokenizer.py @@ -71,7 +71,7 @@ def register_tokenizers(index: tantivy.Index, language: str | None) -> None: use fast=True and Tantivy requires fast-field tokenizers to exist even for documents that omit those fields. """ - index.register_tokenizer("paperless_text", _paperless_text(language)) + index.register_tokenizer("paperless_text", paperless_text_analyzer(language)) index.register_tokenizer("simple_analyzer", _simple_analyzer()) index.register_tokenizer("bigram_analyzer", _bigram_analyzer()) index.register_tokenizer("simple_search_analyzer", _simple_search_analyzer()) @@ -79,7 +79,7 @@ def register_tokenizers(index: tantivy.Index, language: str | None) -> None: index.register_fast_field_tokenizer("simple_analyzer", _simple_analyzer()) -def _paperless_text(language: str | None) -> tantivy.TextAnalyzer: +def paperless_text_analyzer(language: str | None) -> tantivy.TextAnalyzer: """Main full-text tokenizer for content, title, etc: simple -> remove_long(129) -> lowercase -> ascii_fold [-> stemmer]""" builder = ( tantivy.TextAnalyzerBuilder(tantivy.Tokenizer.simple()) diff --git a/src/documents/tests/search/test_registry.py b/src/documents/tests/search/test_registry.py new file mode 100644 index 000000000..8e850aa22 --- /dev/null +++ b/src/documents/tests/search/test_registry.py @@ -0,0 +1,122 @@ +from whoosh_compat import FieldKind + +from documents.search._registry import get_field_registry + + +class TestFieldRegistry: + def test_internal_id_fields_are_not_registered(self) -> None: + registry = get_field_registry(None) + for name in ( + "tag_id", + "owner_id", + "viewer_id", + "correspondent_id", + "document_type_id", + "storage_path_id", + "viewer_group_id", + ): + assert name not in registry + + def test_type_alias_resolves_to_document_type(self) -> None: + registry = get_field_registry(None) + ref = registry.make_ref("type") + assert ref is not None + resolved = registry.resolve(ref) + assert resolved is not None + assert resolved.spec.name == "document_type" + + def test_path_alias_resolves_to_storage_path(self) -> None: + registry = get_field_registry(None) + ref = registry.make_ref("path") + assert ref is not None + resolved = registry.resolve(ref) + assert resolved is not None + assert resolved.spec.name == "storage_path" + + def test_notes_json_subpaths_resolve(self) -> None: + registry = get_field_registry(None) + ref = registry.make_ref("notes.user") + assert ref is not None + resolved = registry.resolve(ref) + assert resolved is not None + assert resolved.spec.name == "notes" + assert resolved.json_path == "user" + assert resolved.is_subpath is True + + def test_custom_fields_json_subpaths_resolve(self) -> None: + registry = get_field_registry(None) + for raw in ("custom_fields.name", "custom_fields.value"): + ref = registry.make_ref(raw) + assert ref is not None + assert registry.resolve(ref) is not None + + def test_unregistered_json_subpath_does_not_resolve(self) -> None: + registry = get_field_registry(None) + # An unregistered subpath is not even a valid FieldRef: make_ref + # returns None for a dotted name whose subpath isn't registered + # (it doesn't produce a ref for resolve() to then reject). + assert registry.make_ref("notes.bogus") is None + + def test_tag_is_comma_values(self) -> None: + registry = get_field_registry(None) + ref = registry.make_ref("tag") + assert ref is not None + resolved = registry.resolve(ref) + assert resolved is not None + assert resolved.spec.comma_values is True + + def test_created_is_date_kind(self) -> None: + registry = get_field_registry(None) + ref = registry.make_ref("created") + assert ref is not None + resolved = registry.resolve(ref) + assert resolved is not None + assert resolved.spec.kind is FieldKind.DATE + assert resolved.spec.date_only is True + + def test_analyzer_lowercases_and_ascii_folds(self) -> None: + # title uses the paperless_text analyzer: simple -> remove_long -> + # lowercase -> ascii_fold [-> stemmer]. With no language configured + # (None), no stemmer runs, so "Café" folds to the single token "cafe". + registry = get_field_registry(None) + ref = registry.make_ref("title") + assert ref is not None + resolved = registry.resolve(ref) + assert resolved is not None + assert resolved.spec.analyzer is not None + assert resolved.spec.analyzer("Café") == ["cafe"] + + def test_checksum_analyzer_is_identity_single_token(self) -> None: + # checksum uses the raw tokenizer at index time (no splitting). + registry = get_field_registry(None) + ref = registry.make_ref("checksum") + assert ref is not None + resolved = registry.resolve(ref) + assert resolved is not None + assert resolved.spec.analyzer("ABC-123") == ["ABC-123"] + + def test_pattern_normalizer_is_ascii_fold_only_no_stemming(self) -> None: + registry = get_field_registry(None) + ref = registry.make_ref("title") + assert ref is not None + resolved = registry.resolve(ref) + assert resolved is not None + assert resolved.spec.pattern_normalizer is not None + # "running" must NOT be stemmed to "run" by the pattern normalizer, + # only case/accent-folded — even with English stemming configured. + registry_en = get_field_registry("en") + ref_en = registry_en.make_ref("title") + assert ref_en is not None + resolved_en = registry_en.resolve(ref_en) + assert resolved_en is not None + assert resolved_en.spec.pattern_normalizer("Running") == "running" + + def test_registry_is_cached_per_language(self) -> None: + a = get_field_registry("en") + b = get_field_registry("en") + assert a is b + + def test_registry_rebuilds_on_language_change(self) -> None: + a = get_field_registry("en") + b = get_field_registry("de") + assert a is not b diff --git a/src/documents/tests/search/test_tokenizer.py b/src/documents/tests/search/test_tokenizer.py index bd03cdf09..0bcf1ce3a 100644 --- a/src/documents/tests/search/test_tokenizer.py +++ b/src/documents/tests/search/test_tokenizer.py @@ -7,8 +7,8 @@ import pytest import tantivy from documents.search._tokenizer import _bigram_analyzer -from documents.search._tokenizer import _paperless_text from documents.search._tokenizer import _simple_search_analyzer +from documents.search._tokenizer import paperless_text_analyzer from documents.search._tokenizer import register_tokenizers if TYPE_CHECKING: @@ -25,7 +25,7 @@ class TestTokenizers: sb.add_text_field("content", stored=True, tokenizer_name="paperless_text") schema = sb.build() idx = tantivy.Index(schema, path=None) - idx.register_tokenizer("paperless_text", _paperless_text("")) + idx.register_tokenizer("paperless_text", paperless_text_analyzer("")) return idx @pytest.fixture