diff --git a/docs/usage.md b/docs/usage.md index f4661d6eb..fd2ff3b99 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -922,9 +922,16 @@ original_filename:invoice.pdf Matching inexact words: ``` -produ*name +invoice* +title:Invoice* ``` +Wildcards are matched against the _stemmed_ terms stored in the index, not +against the words as they appear in the document. A trailing `*` therefore +works as expected (`invoice*` finds "invoice", "invoices" and "invoiced"), but +a pattern whose text continues past where stemming cuts a word off cannot +match: `productname` is indexed as `productnam`, so `produ*name` finds nothing. + Matching natural date keywords: Multi-word date keywords work quoted or unquoted after a date field diff --git a/src/documents/search/_registry.py b/src/documents/search/_registry.py index ad5b22baf..da6a6e678 100644 --- a/src/documents/search/_registry.py +++ b/src/documents/search/_registry.py @@ -1,6 +1,7 @@ from __future__ import annotations import dataclasses +from typing import TYPE_CHECKING from whoosh_compat import FieldKind from whoosh_compat import FieldRegistry @@ -8,6 +9,10 @@ from whoosh_compat import FieldRegistry from documents.search._fields import PUBLIC_FIELDS from documents.search._tokenizer import ascii_fold from documents.search._tokenizer import paperless_text_analyzer +from documents.search._tokenizer import stem_pattern_text + +if TYPE_CHECKING: + from collections.abc import Callable _registry_cache: dict[str | None, FieldRegistry] = {} @@ -17,15 +22,26 @@ def _identity_analyzer(text: str) -> list[str]: return [text] -def _pattern_normalizer(text: str) -> str: - """Normalize wildcard/regex query patterns: lowercase -> ascii_fold. +def _make_pattern_normalizer(language: str | None) -> Callable[[str], str]: + """Build the wildcard/regex literal-run normalizer for a search language.""" - Mirrors the lowercase -> ascii_fold steps of the index-time analyzers - (paperless_text) without stemming, so pattern queries (e.g. "run*") - match tokens that were folded the same way at index time but are not - run through a stemmer, which would corrupt wildcard/regex semantics. - """ - return ascii_fold(text.lower()) + def _pattern_normalizer(text: str) -> str: + """Normalize a literal run so it can match indexed terms. + + Index terms go through lowercase -> ascii_fold -> stem, so a pattern + that skips stemming can never match one: "invoice*" would look for a + term starting with "invoice" while the index holds "invoic". The run is + therefore stemmed here too. + + A stem can be longer than the fragment the user typed, though, and a + longer prefix matches nothing while a shorter one only widens recall, + so the stem is used only when it is no longer than the typed run. + """ + folded = ascii_fold(text.lower()) + stemmed = stem_pattern_text(folded, language) + return folded if len(stemmed) > len(folded) else stemmed + + return _pattern_normalizer def get_field_registry(language: str | None) -> FieldRegistry: @@ -39,6 +55,7 @@ def get_field_registry(language: str | None) -> FieldRegistry: return _registry_cache[language] text_analyzer = paperless_text_analyzer(language).analyze + pattern_normalizer = _make_pattern_normalizer(language) specs = [ dataclasses.replace( @@ -46,7 +63,7 @@ def get_field_registry(language: str | None) -> FieldRegistry: analyzer=_identity_analyzer if field.kind is FieldKind.KEYWORD else text_analyzer, - pattern_normalizer=_pattern_normalizer, + pattern_normalizer=pattern_normalizer, ) for field in PUBLIC_FIELDS ] diff --git a/src/documents/search/_tokenizer.py b/src/documents/search/_tokenizer.py index 11c2f5ba4..fddddbc9c 100644 --- a/src/documents/search/_tokenizer.py +++ b/src/documents/search/_tokenizer.py @@ -1,6 +1,7 @@ from __future__ import annotations import logging +from functools import cache from typing import Final import tantivy @@ -100,6 +101,51 @@ def paperless_text_analyzer(language: str | None) -> tantivy.TextAnalyzer: return builder.build() +@cache +def _pattern_stemmer(language: str | None) -> tantivy.TextAnalyzer | None: + """The stemming tail of paperless_text_analyzer, over a whole literal run. + + Same language gate and same Snowball stemmer paperless_text_analyzer + applies at index time, so query patterns follow SEARCH_LANGUAGE. Returns + None when that gate disables stemming; paperless_text_analyzer already + warns about an unsupported language, so this stays quiet. + + The raw tokenizer keeps the run whole (a wildcard literal is a fragment, + not necessarily a word), and remove_long is kept so an over-long run is + treated the same way the index treats it. + """ + if not language: + return None + tantivy_lang = _LANGUAGE_MAP.get(language.lower()) + if tantivy_lang is None: + return None + return ( + tantivy.TextAnalyzerBuilder(tantivy.Tokenizer.raw()) + .filter(tantivy.Filter.remove_long(_TOKEN_REMOVE_LONG_LIMIT)) + .filter(tantivy.Filter.stemmer(tantivy_lang)) + .build() + ) + + +def stem_pattern_text(text: str, language: str | None) -> str: + """Stem an already lowercased/ascii-folded run the way index terms are. + + Returns text unchanged when stemming is disabled for language, and also + when the stem step does not yield exactly one token: remove_long drops an + over-long run entirely, and there is no single stem to substitute for a run + that analyzes to several. Falling back to the text as typed is the safe + direction for a pattern prefix, since it can only be as narrow as it was + before stemming was considered. + """ + analyzer = _pattern_stemmer(language) + if analyzer is None: + return text + tokens = analyzer.analyze(text) + if len(tokens) != 1: + return text + return tokens[0] + + def _simple_analyzer() -> tantivy.TextAnalyzer: """Tokenizer for shadow sort fields (title_sort, correspondent_sort, type_sort): simple -> lowercase -> ascii_fold.""" return ( diff --git a/src/documents/tests/search/test_pattern_stemming.py b/src/documents/tests/search/test_pattern_stemming.py new file mode 100644 index 000000000..6fddec710 --- /dev/null +++ b/src/documents/tests/search/test_pattern_stemming.py @@ -0,0 +1,128 @@ +"""Wildcard patterns must match a stemmed index. + +Query patterns are normalized but were not stemmed, while index terms are +stemmed, so the natural spelling of a prefix search matched nothing: +``invoice*`` found no document although ``invoic*`` did. v2's index was +UNSTEMMED (whoosh ``TEXT()`` defaults to ``StandardAnalyzer``), so this +regressed against both baselines. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from documents.models import Document +from documents.search._registry import _make_pattern_normalizer + +if TYPE_CHECKING: + from documents.search._backend import TantivyBackend + +pytestmark = [pytest.mark.search, pytest.mark.django_db] + +CONTENT = ( + "invoice total due for electricity from both companies, " + "payments made to the university library" +) + + +def _matched_ids(backend: TantivyBackend, query: str) -> set[int]: + return set(backend.search_ids(query, user=None)) + + +@pytest.fixture +def indexed_doc(backend: TantivyBackend) -> Document: + doc = Document.objects.create( + title="Invoice 2020 productname", + content=CONTENT, + checksum="pattern-stemming-1", + archive_serial_number=900, + ) + backend.add_or_update(doc) + return doc + + +class TestPrefixStemming: + @pytest.mark.parametrize( + "query", + [ + "invoice*", + "electricity*", + "companies*", + "payments*", + "library*", + "title:Invoice*", + ], + ) + def test_full_word_prefix_matches_its_stem( + self, + backend: TantivyBackend, + indexed_doc: Document, + query: str, + ) -> None: + assert _matched_ids(backend, query) == {indexed_doc.id} + + @pytest.mark.parametrize("query", ["invoic*", "electr*", "payment*"]) + def test_already_stemmed_prefix_still_matches( + self, + backend: TantivyBackend, + indexed_doc: Document, + query: str, + ) -> None: + assert _matched_ids(backend, query) == {indexed_doc.id} + + @pytest.mark.parametrize("query", ["univers*", "librar*"]) + def test_partial_prefix_is_not_lengthened_by_its_stem( + self, + backend: TantivyBackend, + indexed_doc: Document, + query: str, + ) -> None: + """A partial prefix keeps matching: "librar" stems to "librari", which + is longer than what was typed, so the typed run is kept. Using the + shorter of the two widens recall rather than failing closed.""" + assert _matched_ids(backend, query) == {indexed_doc.id} + + def test_pattern_past_the_stem_boundary_is_documented_not_fixed( + self, + backend: TantivyBackend, + indexed_doc: Document, + ) -> None: + """produ*name cannot match a stemmed index ("productname" is indexed as + "productnam"); usage.md must not advertise it. Pinned so the limitation + is deliberate, not accidental.""" + assert _matched_ids(backend, "produ*name") == set() + + +class TestPatternNormalizer: + @pytest.mark.parametrize( + ("text", "expected"), + [ + ("Invoice", "invoic"), + ("companies", "compani"), + # y -> i: same length as typed, and the index only holds the stem + ("library", "librari"), + ("invoic", "invoic"), + ("Universit", "universit"), + ("Café", "cafe"), + ], + ) + def test_shorter_of_the_typed_run_and_its_stem( + self, + text: str, + expected: str, + ) -> None: + assert _make_pattern_normalizer("en")(text) == expected + + def test_run_that_yields_no_token_falls_back_to_the_typed_run(self) -> None: + """A run past the remove_long limit analyzes to zero tokens, so there is + no stem to substitute and the folded run is used as typed.""" + over_long = "invoices" * 20 + assert _make_pattern_normalizer("en")(over_long) == over_long + + @pytest.mark.parametrize("language", [None, "klingon"]) + def test_unstemmed_language_folds_only(self, language: str | None) -> None: + """With no stemmer configured, or one this build has no stemmer for, the + index holds surface forms and the pattern must keep them too.""" + assert _make_pattern_normalizer(language)("Invoices") == "invoices" diff --git a/src/documents/tests/search/test_registry.py b/src/documents/tests/search/test_registry.py index 6de41a00f..f748c6360 100644 --- a/src/documents/tests/search/test_registry.py +++ b/src/documents/tests/search/test_registry.py @@ -91,18 +91,20 @@ class TestFieldRegistry: assert resolved.spec.analyzer is not None assert resolved.spec.analyzer("ABC-123") == ["ABC-123"] - def test_pattern_normalizer_is_ascii_fold_only_no_stemming( + def test_pattern_normalizer_follows_the_registry_language( self, registry: FieldRegistry, ) -> None: + # Index terms are stemmed, so patterns are too, using the registry's + # own language: "Running" has to reach the indexed "run". Without a + # language the index holds surface forms, so it only case/accent-folds. resolved = _resolve(registry, "title") assert resolved.spec.pattern_normalizer is not None - # "running" must NOT be stemmed to "run" by the pattern normalizer, - # only case/accent-folded — even with English stemming configured. - registry_en = get_field_registry("en") - resolved_en = _resolve(registry_en, "title") + assert resolved.spec.pattern_normalizer("Running") == "running" + + resolved_en = _resolve(get_field_registry("en"), "title") assert resolved_en.spec.pattern_normalizer is not None - assert resolved_en.spec.pattern_normalizer("Running") == "running" + assert resolved_en.spec.pattern_normalizer("Running") == "run" def test_registry_is_cached_per_language(self) -> None: a = get_field_registry("en")