fix(search): stem wildcard patterns so prefix searches match again

Index terms are stemmed but query patterns were not, so invoice* matched nothing
while invoic* worked. v2's index was unstemmed (whoosh TEXT() defaults to
StandardAnalyzer), so this regressed against both baselines, not just dev. Uses
the typed run's stem unless the stem is longer than the run, since a stem can be
longer than a partial prefix and a shorter prefix only widens recall. Patterns
spanning the stem boundary (produ*name) still cannot match a stemmed index, so
usage.md loses that example rather than advertising a broken one.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

src/documents/tests/search/test_pattern_stemming.py

@@ -0,0 +1,128 @@
+"""Wildcard patterns must match a stemmed index.
+
+Query patterns are normalized but were not stemmed, while index terms are
+stemmed, so the natural spelling of a prefix search matched nothing:
+``invoice*`` found no document although ``invoic*`` did. v2's index was
+UNSTEMMED (whoosh ``TEXT()`` defaults to ``StandardAnalyzer``), so this
+regressed against both baselines.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+
+from documents.models import Document
+from documents.search._registry import _make_pattern_normalizer
+
+if TYPE_CHECKING:
+    from documents.search._backend import TantivyBackend
+
+pytestmark = [pytest.mark.search, pytest.mark.django_db]
+
+CONTENT = (
+    "invoice total due for electricity from both companies, "
+    "payments made to the university library"
+)
+
+
+def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
+    return set(backend.search_ids(query, user=None))
+
+
+@pytest.fixture
+def indexed_doc(backend: TantivyBackend) -> Document:
+    doc = Document.objects.create(
+        title="Invoice 2020 productname",
+        content=CONTENT,
+        checksum="pattern-stemming-1",
+        archive_serial_number=900,
+    )
+    backend.add_or_update(doc)
+    return doc
+
+
+class TestPrefixStemming:
+    @pytest.mark.parametrize(
+        "query",
+        [
+            "invoice*",
+            "electricity*",
+            "companies*",
+            "payments*",
+            "library*",
+            "title:Invoice*",
+        ],
+    )
+    def test_full_word_prefix_matches_its_stem(
+        self,
+        backend: TantivyBackend,
+        indexed_doc: Document,
+        query: str,
+    ) -> None:
+        assert _matched_ids(backend, query) == {indexed_doc.id}
+
+    @pytest.mark.parametrize("query", ["invoic*", "electr*", "payment*"])
+    def test_already_stemmed_prefix_still_matches(
+        self,
+        backend: TantivyBackend,
+        indexed_doc: Document,
+        query: str,
+    ) -> None:
+        assert _matched_ids(backend, query) == {indexed_doc.id}
+
+    @pytest.mark.parametrize("query", ["univers*", "librar*"])
+    def test_partial_prefix_is_not_lengthened_by_its_stem(
+        self,
+        backend: TantivyBackend,
+        indexed_doc: Document,
+        query: str,
+    ) -> None:
+        """A partial prefix keeps matching: "librar" stems to "librari", which
+        is longer than what was typed, so the typed run is kept. Using the
+        shorter of the two widens recall rather than failing closed."""
+        assert _matched_ids(backend, query) == {indexed_doc.id}
+
+    def test_pattern_past_the_stem_boundary_is_documented_not_fixed(
+        self,
+        backend: TantivyBackend,
+        indexed_doc: Document,
+    ) -> None:
+        """produ*name cannot match a stemmed index ("productname" is indexed as
+        "productnam"); usage.md must not advertise it. Pinned so the limitation
+        is deliberate, not accidental."""
+        assert _matched_ids(backend, "produ*name") == set()
+
+
+class TestPatternNormalizer:
+    @pytest.mark.parametrize(
+        ("text", "expected"),
+        [
+            ("Invoice", "invoic"),
+            ("companies", "compani"),
+            # y -> i: same length as typed, and the index only holds the stem
+            ("library", "librari"),
+            ("invoic", "invoic"),
+            ("Universit", "universit"),
+            ("Café", "cafe"),
+        ],
+    )
+    def test_shorter_of_the_typed_run_and_its_stem(
+        self,
+        text: str,
+        expected: str,
+    ) -> None:
+        assert _make_pattern_normalizer("en")(text) == expected
+
+    def test_run_that_yields_no_token_falls_back_to_the_typed_run(self) -> None:
+        """A run past the remove_long limit analyzes to zero tokens, so there is
+        no stem to substitute and the folded run is used as typed."""
+        over_long = "invoices" * 20
+        assert _make_pattern_normalizer("en")(over_long) == over_long
+
+    @pytest.mark.parametrize("language", [None, "klingon"])
+    def test_unstemmed_language_folds_only(self, language: str | None) -> None:
+        """With no stemmer configured, or one this build has no stemmer for, the
+        index holds surface forms and the pattern must keep them too."""
+        assert _make_pattern_normalizer(language)("Invoices") == "invoices"

src/documents/tests/search/test_registry.py

@@ -91,18 +91,20 @@ class TestFieldRegistry:
         assert resolved.spec.analyzer is not None
         assert resolved.spec.analyzer("ABC-123") == ["ABC-123"]
 
-    def test_pattern_normalizer_is_ascii_fold_only_no_stemming(
+    def test_pattern_normalizer_follows_the_registry_language(
         self,
         registry: FieldRegistry,
     ) -> None:
+        # Index terms are stemmed, so patterns are too, using the registry's
+        # own language: "Running" has to reach the indexed "run". Without a
+        # language the index holds surface forms, so it only case/accent-folds.
         resolved = _resolve(registry, "title")
         assert resolved.spec.pattern_normalizer is not None
-        # "running" must NOT be stemmed to "run" by the pattern normalizer,
-        # only case/accent-folded — even with English stemming configured.
-        registry_en = get_field_registry("en")
-        resolved_en = _resolve(registry_en, "title")
+        assert resolved.spec.pattern_normalizer("Running") == "running"
+
+        resolved_en = _resolve(get_field_registry("en"), "title")
         assert resolved_en.spec.pattern_normalizer is not None
-        assert resolved_en.spec.pattern_normalizer("Running") == "running"
+        assert resolved_en.spec.pattern_normalizer("Running") == "run"
 
     def test_registry_is_cached_per_language(self) -> None:
         a = get_field_registry("en")