fix(search): stem wildcard patterns so prefix searches match again

Index terms are stemmed but query patterns were not, so invoice* matched nothing
while invoic* worked. v2's index was unstemmed (whoosh TEXT() defaults to
StandardAnalyzer), so this regressed against both baselines, not just dev. Uses
the typed run's stem unless the stem is longer than the run, since a stem can be
longer than a partial prefix and a shorter prefix only widens recall. Patterns
spanning the stem boundary (produ*name) still cannot match a stemmed index, so
usage.md loses that example rather than advertising a broken one.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

docs/usage.md

@@ -922,9 +922,16 @@ original_filename:invoice.pdf
 Matching inexact words:
 
 ```
-produ*name
+invoice*
+title:Invoice*
 ```
 
+Wildcards are matched against the _stemmed_ terms stored in the index, not
+against the words as they appear in the document. A trailing `*` therefore
+works as expected (`invoice*` finds "invoice", "invoices" and "invoiced"), but
+a pattern whose text continues past where stemming cuts a word off cannot
+match: `productname` is indexed as `productnam`, so `produ*name` finds nothing.
+
 Matching natural date keywords:
 
 Multi-word date keywords work quoted or unquoted after a date field

src/documents/search/_registry.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import dataclasses
+from typing import TYPE_CHECKING
 
 from whoosh_compat import FieldKind
 from whoosh_compat import FieldRegistry
@@ -8,6 +9,10 @@ from whoosh_compat import FieldRegistry
 from documents.search._fields import PUBLIC_FIELDS
 from documents.search._tokenizer import ascii_fold
 from documents.search._tokenizer import paperless_text_analyzer
+from documents.search._tokenizer import stem_pattern_text
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
 
 _registry_cache: dict[str | None, FieldRegistry] = {}
 
@@ -17,15 +22,26 @@ def _identity_analyzer(text: str) -> list[str]:
     return [text]
 
 
-def _pattern_normalizer(text: str) -> str:
-    """Normalize wildcard/regex query patterns: lowercase -> ascii_fold.
+def _make_pattern_normalizer(language: str | None) -> Callable[[str], str]:
+    """Build the wildcard/regex literal-run normalizer for a search language."""
 
-    Mirrors the lowercase -> ascii_fold steps of the index-time analyzers
-    (paperless_text) without stemming, so pattern queries (e.g. "run*")
-    match tokens that were folded the same way at index time but are not
-    run through a stemmer, which would corrupt wildcard/regex semantics.
-    """
-    return ascii_fold(text.lower())
+    def _pattern_normalizer(text: str) -> str:
+        """Normalize a literal run so it can match indexed terms.
+
+        Index terms go through lowercase -> ascii_fold -> stem, so a pattern
+        that skips stemming can never match one: "invoice*" would look for a
+        term starting with "invoice" while the index holds "invoic". The run is
+        therefore stemmed here too.
+
+        A stem can be longer than the fragment the user typed, though, and a
+        longer prefix matches nothing while a shorter one only widens recall,
+        so the stem is used only when it is no longer than the typed run.
+        """
+        folded = ascii_fold(text.lower())
+        stemmed = stem_pattern_text(folded, language)
+        return folded if len(stemmed) > len(folded) else stemmed
+
+    return _pattern_normalizer
 
 
 def get_field_registry(language: str | None) -> FieldRegistry:
@@ -39,6 +55,7 @@ def get_field_registry(language: str | None) -> FieldRegistry:
         return _registry_cache[language]
 
     text_analyzer = paperless_text_analyzer(language).analyze
+    pattern_normalizer = _make_pattern_normalizer(language)
 
     specs = [
         dataclasses.replace(
@@ -46,7 +63,7 @@ def get_field_registry(language: str | None) -> FieldRegistry:
             analyzer=_identity_analyzer
             if field.kind is FieldKind.KEYWORD
             else text_analyzer,
-            pattern_normalizer=_pattern_normalizer,
+            pattern_normalizer=pattern_normalizer,
         )
         for field in PUBLIC_FIELDS
     ]

src/documents/search/_tokenizer.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import logging
+from functools import cache
 from typing import Final
 
 import tantivy
@@ -100,6 +101,51 @@ def paperless_text_analyzer(language: str | None) -> tantivy.TextAnalyzer:
     return builder.build()
 
 
+@cache
+def _pattern_stemmer(language: str | None) -> tantivy.TextAnalyzer | None:
+    """The stemming tail of paperless_text_analyzer, over a whole literal run.
+
+    Same language gate and same Snowball stemmer paperless_text_analyzer
+    applies at index time, so query patterns follow SEARCH_LANGUAGE. Returns
+    None when that gate disables stemming; paperless_text_analyzer already
+    warns about an unsupported language, so this stays quiet.
+
+    The raw tokenizer keeps the run whole (a wildcard literal is a fragment,
+    not necessarily a word), and remove_long is kept so an over-long run is
+    treated the same way the index treats it.
+    """
+    if not language:
+        return None
+    tantivy_lang = _LANGUAGE_MAP.get(language.lower())
+    if tantivy_lang is None:
+        return None
+    return (
+        tantivy.TextAnalyzerBuilder(tantivy.Tokenizer.raw())
+        .filter(tantivy.Filter.remove_long(_TOKEN_REMOVE_LONG_LIMIT))
+        .filter(tantivy.Filter.stemmer(tantivy_lang))
+        .build()
+    )
+
+
+def stem_pattern_text(text: str, language: str | None) -> str:
+    """Stem an already lowercased/ascii-folded run the way index terms are.
+
+    Returns text unchanged when stemming is disabled for language, and also
+    when the stem step does not yield exactly one token: remove_long drops an
+    over-long run entirely, and there is no single stem to substitute for a run
+    that analyzes to several. Falling back to the text as typed is the safe
+    direction for a pattern prefix, since it can only be as narrow as it was
+    before stemming was considered.
+    """
+    analyzer = _pattern_stemmer(language)
+    if analyzer is None:
+        return text
+    tokens = analyzer.analyze(text)
+    if len(tokens) != 1:
+        return text
+    return tokens[0]
+
+
 def _simple_analyzer() -> tantivy.TextAnalyzer:
     """Tokenizer for shadow sort fields (title_sort, correspondent_sort, type_sort): simple -> lowercase -> ascii_fold."""
     return (

src/documents/tests/search/test_pattern_stemming.py

@@ -0,0 +1,128 @@
+"""Wildcard patterns must match a stemmed index.
+
+Query patterns are normalized but were not stemmed, while index terms are
+stemmed, so the natural spelling of a prefix search matched nothing:
+``invoice*`` found no document although ``invoic*`` did. v2's index was
+UNSTEMMED (whoosh ``TEXT()`` defaults to ``StandardAnalyzer``), so this
+regressed against both baselines.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+
+from documents.models import Document
+from documents.search._registry import _make_pattern_normalizer
+
+if TYPE_CHECKING:
+    from documents.search._backend import TantivyBackend
+
+pytestmark = [pytest.mark.search, pytest.mark.django_db]
+
+CONTENT = (
+    "invoice total due for electricity from both companies, "
+    "payments made to the university library"
+)
+
+
+def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
+    return set(backend.search_ids(query, user=None))
+
+
+@pytest.fixture
+def indexed_doc(backend: TantivyBackend) -> Document:
+    doc = Document.objects.create(
+        title="Invoice 2020 productname",
+        content=CONTENT,
+        checksum="pattern-stemming-1",
+        archive_serial_number=900,
+    )
+    backend.add_or_update(doc)
+    return doc
+
+
+class TestPrefixStemming:
+    @pytest.mark.parametrize(
+        "query",
+        [
+            "invoice*",
+            "electricity*",
+            "companies*",
+            "payments*",
+            "library*",
+            "title:Invoice*",
+        ],
+    )
+    def test_full_word_prefix_matches_its_stem(
+        self,
+        backend: TantivyBackend,
+        indexed_doc: Document,
+        query: str,
+    ) -> None:
+        assert _matched_ids(backend, query) == {indexed_doc.id}
+
+    @pytest.mark.parametrize("query", ["invoic*", "electr*", "payment*"])
+    def test_already_stemmed_prefix_still_matches(
+        self,
+        backend: TantivyBackend,
+        indexed_doc: Document,
+        query: str,
+    ) -> None:
+        assert _matched_ids(backend, query) == {indexed_doc.id}
+
+    @pytest.mark.parametrize("query", ["univers*", "librar*"])
+    def test_partial_prefix_is_not_lengthened_by_its_stem(
+        self,
+        backend: TantivyBackend,
+        indexed_doc: Document,
+        query: str,
+    ) -> None:
+        """A partial prefix keeps matching: "librar" stems to "librari", which
+        is longer than what was typed, so the typed run is kept. Using the
+        shorter of the two widens recall rather than failing closed."""
+        assert _matched_ids(backend, query) == {indexed_doc.id}
+
+    def test_pattern_past_the_stem_boundary_is_documented_not_fixed(
+        self,
+        backend: TantivyBackend,
+        indexed_doc: Document,
+    ) -> None:
+        """produ*name cannot match a stemmed index ("productname" is indexed as
+        "productnam"); usage.md must not advertise it. Pinned so the limitation
+        is deliberate, not accidental."""
+        assert _matched_ids(backend, "produ*name") == set()
+
+
+class TestPatternNormalizer:
+    @pytest.mark.parametrize(
+        ("text", "expected"),
+        [
+            ("Invoice", "invoic"),
+            ("companies", "compani"),
+            # y -> i: same length as typed, and the index only holds the stem
+            ("library", "librari"),
+            ("invoic", "invoic"),
+            ("Universit", "universit"),
+            ("Café", "cafe"),
+        ],
+    )
+    def test_shorter_of_the_typed_run_and_its_stem(
+        self,
+        text: str,
+        expected: str,
+    ) -> None:
+        assert _make_pattern_normalizer("en")(text) == expected
+
+    def test_run_that_yields_no_token_falls_back_to_the_typed_run(self) -> None:
+        """A run past the remove_long limit analyzes to zero tokens, so there is
+        no stem to substitute and the folded run is used as typed."""
+        over_long = "invoices" * 20
+        assert _make_pattern_normalizer("en")(over_long) == over_long
+
+    @pytest.mark.parametrize("language", [None, "klingon"])
+    def test_unstemmed_language_folds_only(self, language: str | None) -> None:
+        """With no stemmer configured, or one this build has no stemmer for, the
+        index holds surface forms and the pattern must keep them too."""
+        assert _make_pattern_normalizer(language)("Invoices") == "invoices"

src/documents/tests/search/test_registry.py

@@ -91,18 +91,20 @@ class TestFieldRegistry:
         assert resolved.spec.analyzer is not None
         assert resolved.spec.analyzer("ABC-123") == ["ABC-123"]
 
-    def test_pattern_normalizer_is_ascii_fold_only_no_stemming(
+    def test_pattern_normalizer_follows_the_registry_language(
         self,
         registry: FieldRegistry,
     ) -> None:
+        # Index terms are stemmed, so patterns are too, using the registry's
+        # own language: "Running" has to reach the indexed "run". Without a
+        # language the index holds surface forms, so it only case/accent-folds.
         resolved = _resolve(registry, "title")
         assert resolved.spec.pattern_normalizer is not None
-        # "running" must NOT be stemmed to "run" by the pattern normalizer,
-        # only case/accent-folded — even with English stemming configured.
-        registry_en = get_field_registry("en")
-        resolved_en = _resolve(registry_en, "title")
+        assert resolved.spec.pattern_normalizer("Running") == "running"
+
+        resolved_en = _resolve(get_field_registry("en"), "title")
         assert resolved_en.spec.pattern_normalizer is not None
-        assert resolved_en.spec.pattern_normalizer("Running") == "running"
+        assert resolved_en.spec.pattern_normalizer("Running") == "run"
 
     def test_registry_is_cached_per_language(self) -> None:
         a = get_field_registry("en")