mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-11-22 13:24:22 +00:00
feat(search): add whoosh-compat FieldRegistry construction
This commit is contained in:
1 parent
d0e93790ce
commit
bc5360cd6b
4 files changed
+190
-4
No files matched your search
@@ -0,0 +1,64 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from whoosh_compat import FieldKind
|
||||||
|
from whoosh_compat import FieldRegistry
|
||||||
|
from whoosh_compat import FieldSpec
|
||||||
|
|
||||||
|
from documents.search._fields import PUBLIC_FIELDS
|
||||||
|
from documents.search._tokenizer import ascii_fold
|
||||||
|
from documents.search._tokenizer import paperless_text_analyzer
|
||||||
|
|
||||||
|
_registry_cache: dict[str | None, FieldRegistry] = {}
|
||||||
|
|
||||||
|
|
||||||
|
def _identity_analyzer(text: str) -> list[str]:
|
||||||
|
"""Analyzer for KEYWORD fields indexed with the raw tokenizer (no splitting)."""
|
||||||
|
return [text]
|
||||||
|
|
||||||
|
|
||||||
|
def _pattern_normalizer(text: str) -> str:
|
||||||
|
"""Normalize wildcard/regex query patterns: lowercase -> ascii_fold.
|
||||||
|
|
||||||
|
Mirrors the lowercase -> ascii_fold steps of the index-time analyzers
|
||||||
|
(paperless_text) without stemming, so pattern queries (e.g. "run*")
|
||||||
|
match tokens that were folded the same way at index time but are not
|
||||||
|
run through a stemmer, which would corrupt wildcard/regex semantics.
|
||||||
|
"""
|
||||||
|
return ascii_fold(text.lower())
|
||||||
|
|
||||||
|
|
||||||
|
def get_field_registry(language: str | None) -> FieldRegistry:
|
||||||
|
"""Build (or return the cached) FieldRegistry for the given search language.
|
||||||
|
|
||||||
|
Cached keyed by language, rebuilt on the same trigger register_tokenizers()
|
||||||
|
uses (settings.SEARCH_LANGUAGE change) — a fresh call with a new language
|
||||||
|
builds and caches a new registry rather than mutating the old one.
|
||||||
|
"""
|
||||||
|
if language in _registry_cache:
|
||||||
|
return _registry_cache[language]
|
||||||
|
|
||||||
|
text_analyzer = paperless_text_analyzer(language).analyze
|
||||||
|
|
||||||
|
specs = []
|
||||||
|
for field in PUBLIC_FIELDS:
|
||||||
|
if field.kind is FieldKind.KEYWORD:
|
||||||
|
analyzer = _identity_analyzer
|
||||||
|
else:
|
||||||
|
analyzer = text_analyzer
|
||||||
|
specs.append(
|
||||||
|
FieldSpec(
|
||||||
|
name=field.name,
|
||||||
|
kind=field.kind,
|
||||||
|
aliases=field.aliases,
|
||||||
|
comma_values=field.comma_values,
|
||||||
|
analyzer=analyzer,
|
||||||
|
pattern_normalizer=_pattern_normalizer,
|
||||||
|
date_only=field.date_only,
|
||||||
|
fast=field.fast,
|
||||||
|
subpaths=field.subpaths,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
registry = FieldRegistry(specs)
|
||||||
|
_registry_cache[language] = registry
|
||||||
|
return registry
|
||||||
@@ -71,7 +71,7 @@ def register_tokenizers(index: tantivy.Index, language: str | None) -> None:
|
|||||||
use fast=True and Tantivy requires fast-field tokenizers to exist
|
use fast=True and Tantivy requires fast-field tokenizers to exist
|
||||||
even for documents that omit those fields.
|
even for documents that omit those fields.
|
||||||
"""
|
"""
|
||||||
index.register_tokenizer("paperless_text", _paperless_text(language))
|
index.register_tokenizer("paperless_text", paperless_text_analyzer(language))
|
||||||
index.register_tokenizer("simple_analyzer", _simple_analyzer())
|
index.register_tokenizer("simple_analyzer", _simple_analyzer())
|
||||||
index.register_tokenizer("bigram_analyzer", _bigram_analyzer())
|
index.register_tokenizer("bigram_analyzer", _bigram_analyzer())
|
||||||
index.register_tokenizer("simple_search_analyzer", _simple_search_analyzer())
|
index.register_tokenizer("simple_search_analyzer", _simple_search_analyzer())
|
||||||
@@ -79,7 +79,7 @@ def register_tokenizers(index: tantivy.Index, language: str | None) -> None:
|
|||||||
index.register_fast_field_tokenizer("simple_analyzer", _simple_analyzer())
|
index.register_fast_field_tokenizer("simple_analyzer", _simple_analyzer())
|
||||||
|
|
||||||
|
|
||||||
def _paperless_text(language: str | None) -> tantivy.TextAnalyzer:
|
def paperless_text_analyzer(language: str | None) -> tantivy.TextAnalyzer:
|
||||||
"""Main full-text tokenizer for content, title, etc: simple -> remove_long(129) -> lowercase -> ascii_fold [-> stemmer]"""
|
"""Main full-text tokenizer for content, title, etc: simple -> remove_long(129) -> lowercase -> ascii_fold [-> stemmer]"""
|
||||||
builder = (
|
builder = (
|
||||||
tantivy.TextAnalyzerBuilder(tantivy.Tokenizer.simple())
|
tantivy.TextAnalyzerBuilder(tantivy.Tokenizer.simple())
|
||||||
|
|||||||
@@ -0,0 +1,122 @@
|
|||||||
|
from whoosh_compat import FieldKind
|
||||||
|
|
||||||
|
from documents.search._registry import get_field_registry
|
||||||
|
|
||||||
|
|
||||||
|
class TestFieldRegistry:
|
||||||
|
def test_internal_id_fields_are_not_registered(self) -> None:
|
||||||
|
registry = get_field_registry(None)
|
||||||
|
for name in (
|
||||||
|
"tag_id",
|
||||||
|
"owner_id",
|
||||||
|
"viewer_id",
|
||||||
|
"correspondent_id",
|
||||||
|
"document_type_id",
|
||||||
|
"storage_path_id",
|
||||||
|
"viewer_group_id",
|
||||||
|
):
|
||||||
|
assert name not in registry
|
||||||
|
|
||||||
|
def test_type_alias_resolves_to_document_type(self) -> None:
|
||||||
|
registry = get_field_registry(None)
|
||||||
|
ref = registry.make_ref("type")
|
||||||
|
assert ref is not None
|
||||||
|
resolved = registry.resolve(ref)
|
||||||
|
assert resolved is not None
|
||||||
|
assert resolved.spec.name == "document_type"
|
||||||
|
|
||||||
|
def test_path_alias_resolves_to_storage_path(self) -> None:
|
||||||
|
registry = get_field_registry(None)
|
||||||
|
ref = registry.make_ref("path")
|
||||||
|
assert ref is not None
|
||||||
|
resolved = registry.resolve(ref)
|
||||||
|
assert resolved is not None
|
||||||
|
assert resolved.spec.name == "storage_path"
|
||||||
|
|
||||||
|
def test_notes_json_subpaths_resolve(self) -> None:
|
||||||
|
registry = get_field_registry(None)
|
||||||
|
ref = registry.make_ref("notes.user")
|
||||||
|
assert ref is not None
|
||||||
|
resolved = registry.resolve(ref)
|
||||||
|
assert resolved is not None
|
||||||
|
assert resolved.spec.name == "notes"
|
||||||
|
assert resolved.json_path == "user"
|
||||||
|
assert resolved.is_subpath is True
|
||||||
|
|
||||||
|
def test_custom_fields_json_subpaths_resolve(self) -> None:
|
||||||
|
registry = get_field_registry(None)
|
||||||
|
for raw in ("custom_fields.name", "custom_fields.value"):
|
||||||
|
ref = registry.make_ref(raw)
|
||||||
|
assert ref is not None
|
||||||
|
assert registry.resolve(ref) is not None
|
||||||
|
|
||||||
|
def test_unregistered_json_subpath_does_not_resolve(self) -> None:
|
||||||
|
registry = get_field_registry(None)
|
||||||
|
# An unregistered subpath is not even a valid FieldRef: make_ref
|
||||||
|
# returns None for a dotted name whose subpath isn't registered
|
||||||
|
# (it doesn't produce a ref for resolve() to then reject).
|
||||||
|
assert registry.make_ref("notes.bogus") is None
|
||||||
|
|
||||||
|
def test_tag_is_comma_values(self) -> None:
|
||||||
|
registry = get_field_registry(None)
|
||||||
|
ref = registry.make_ref("tag")
|
||||||
|
assert ref is not None
|
||||||
|
resolved = registry.resolve(ref)
|
||||||
|
assert resolved is not None
|
||||||
|
assert resolved.spec.comma_values is True
|
||||||
|
|
||||||
|
def test_created_is_date_kind(self) -> None:
|
||||||
|
registry = get_field_registry(None)
|
||||||
|
ref = registry.make_ref("created")
|
||||||
|
assert ref is not None
|
||||||
|
resolved = registry.resolve(ref)
|
||||||
|
assert resolved is not None
|
||||||
|
assert resolved.spec.kind is FieldKind.DATE
|
||||||
|
assert resolved.spec.date_only is True
|
||||||
|
|
||||||
|
def test_analyzer_lowercases_and_ascii_folds(self) -> None:
|
||||||
|
# title uses the paperless_text analyzer: simple -> remove_long ->
|
||||||
|
# lowercase -> ascii_fold [-> stemmer]. With no language configured
|
||||||
|
# (None), no stemmer runs, so "Café" folds to the single token "cafe".
|
||||||
|
registry = get_field_registry(None)
|
||||||
|
ref = registry.make_ref("title")
|
||||||
|
assert ref is not None
|
||||||
|
resolved = registry.resolve(ref)
|
||||||
|
assert resolved is not None
|
||||||
|
assert resolved.spec.analyzer is not None
|
||||||
|
assert resolved.spec.analyzer("Café") == ["cafe"]
|
||||||
|
|
||||||
|
def test_checksum_analyzer_is_identity_single_token(self) -> None:
|
||||||
|
# checksum uses the raw tokenizer at index time (no splitting).
|
||||||
|
registry = get_field_registry(None)
|
||||||
|
ref = registry.make_ref("checksum")
|
||||||
|
assert ref is not None
|
||||||
|
resolved = registry.resolve(ref)
|
||||||
|
assert resolved is not None
|
||||||
|
assert resolved.spec.analyzer("ABC-123") == ["ABC-123"]
|
||||||
|
|
||||||
|
def test_pattern_normalizer_is_ascii_fold_only_no_stemming(self) -> None:
|
||||||
|
registry = get_field_registry(None)
|
||||||
|
ref = registry.make_ref("title")
|
||||||
|
assert ref is not None
|
||||||
|
resolved = registry.resolve(ref)
|
||||||
|
assert resolved is not None
|
||||||
|
assert resolved.spec.pattern_normalizer is not None
|
||||||
|
# "running" must NOT be stemmed to "run" by the pattern normalizer,
|
||||||
|
# only case/accent-folded — even with English stemming configured.
|
||||||
|
registry_en = get_field_registry("en")
|
||||||
|
ref_en = registry_en.make_ref("title")
|
||||||
|
assert ref_en is not None
|
||||||
|
resolved_en = registry_en.resolve(ref_en)
|
||||||
|
assert resolved_en is not None
|
||||||
|
assert resolved_en.spec.pattern_normalizer("Running") == "running"
|
||||||
|
|
||||||
|
def test_registry_is_cached_per_language(self) -> None:
|
||||||
|
a = get_field_registry("en")
|
||||||
|
b = get_field_registry("en")
|
||||||
|
assert a is b
|
||||||
|
|
||||||
|
def test_registry_rebuilds_on_language_change(self) -> None:
|
||||||
|
a = get_field_registry("en")
|
||||||
|
b = get_field_registry("de")
|
||||||
|
assert a is not b
|
||||||
@@ -7,8 +7,8 @@ import pytest
|
|||||||
import tantivy
|
import tantivy
|
||||||
|
|
||||||
from documents.search._tokenizer import _bigram_analyzer
|
from documents.search._tokenizer import _bigram_analyzer
|
||||||
from documents.search._tokenizer import _paperless_text
|
|
||||||
from documents.search._tokenizer import _simple_search_analyzer
|
from documents.search._tokenizer import _simple_search_analyzer
|
||||||
|
from documents.search._tokenizer import paperless_text_analyzer
|
||||||
from documents.search._tokenizer import register_tokenizers
|
from documents.search._tokenizer import register_tokenizers
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
@@ -25,7 +25,7 @@ class TestTokenizers:
|
|||||||
sb.add_text_field("content", stored=True, tokenizer_name="paperless_text")
|
sb.add_text_field("content", stored=True, tokenizer_name="paperless_text")
|
||||||
schema = sb.build()
|
schema = sb.build()
|
||||||
idx = tantivy.Index(schema, path=None)
|
idx = tantivy.Index(schema, path=None)
|
||||||
idx.register_tokenizer("paperless_text", _paperless_text(""))
|
idx.register_tokenizer("paperless_text", paperless_text_analyzer(""))
|
||||||
return idx
|
return idx
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
|
|||||||
Reference in new issue
Block a user