fix(search): keep wildcard patterns literal on unstemmed KEYWORD fields

get_field_registry() branched the analyzer by field kind but gave every field
the same stemming pattern normalizer. checksum is indexed with the raw
tokenizer, so its terms are neither folded nor stemmed, yet its wildcard
patterns were: "checksum:ceded*" normalized to "cede*" and matched a document
whose checksum starts with "cedef00d". About 2.8% of random hex prefixes were
rewritten this way. Always over-matching rather than missing, but for a field
whose whole purpose is exact identification, returning a different checksum is
a wrong answer.

KEYWORD fields now get a fold-and-lower normalizer, which is what every field
used before pattern stemming was added; TEXT fields keep the stemming one so
"invoice*" still reaches the indexed "invoic".

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

docs/usage.md

@@ -933,7 +933,7 @@ original_filename:invoice.pdf
 - `asn` matches a document's Archive Serial Number.
 - `page_count` matches a document's page count.
 - `num_notes` matches how many notes a document has.
-- `checksum` matches the checksum of the original document file (not the archived/processed version). Unlike the text fields, this one is stored verbatim rather than tokenized, so only a complete, lowercase checksum matches. To search by the first few characters instead, use a wildcard: `checksum:9f86d081*`. Because the field is not stemmed, that prefix is matched literally.
+- `checksum` matches the checksum of the original document file (not the archived/processed version). Unlike the text fields, this one is stored verbatim rather than tokenized, so only a complete, lowercase checksum matches. To search by the first few characters instead, use a wildcard: `checksum:9f86d081*`. Wildcard patterns on the text fields are stemmed to line up with the stemmed index, but `checksum` is indexed without stemming, so its patterns are not stemmed either and the prefix is matched exactly as typed.
 - `original_filename` matches the filename of the document as originally consumed.
 
 `asn`, `page_count` and `num_notes` are numeric and also accept ranges, for example `asn:[50 to 150]`.

src/documents/search/_registry.py

@@ -22,16 +22,22 @@ def _identity_analyzer(text: str) -> list[str]:
     return [text]
 
 
+def _fold_normalizer(text: str) -> str:
+    """Wildcard/regex literal-run normalizer for fields indexed without stemming."""
+    return ascii_fold(text.lower())
+
+
 def _make_pattern_normalizer(language: str | None) -> Callable[[str], str]:
     """Build the wildcard/regex literal-run normalizer for a search language."""
 
     def _pattern_normalizer(text: str) -> str:
         """Normalize a literal run so it can match indexed terms.
 
-        Index terms go through lowercase -> ascii_fold -> stem, so a pattern
-        that skips stemming can never match one: "invoice*" would look for a
-        term starting with "invoice" while the index holds "invoic". The run is
-        therefore stemmed here too.
+        TEXT index terms go through lowercase -> ascii_fold -> stem, so a
+        pattern that skips stemming can never match one: "invoice*" would look
+        for a term starting with "invoice" while the index holds "invoic". The
+        run is therefore stemmed here too. KEYWORD fields are indexed raw and
+        get _fold_normalizer instead, so their patterns stay literal.
 
         A stem can be longer than the fragment the user typed, though, and a
         longer prefix matches nothing, so the stem is used only when it is no
@@ -67,7 +73,9 @@ def get_field_registry(language: str | None) -> FieldRegistry:
             analyzer=_identity_analyzer
             if field.kind is FieldKind.KEYWORD
             else text_analyzer,
-            pattern_normalizer=pattern_normalizer,
+            pattern_normalizer=_fold_normalizer
+            if field.kind is FieldKind.KEYWORD
+            else pattern_normalizer,
         )
         for field in PUBLIC_FIELDS
     ]

src/documents/tests/search/test_keyword_pattern_literal.py

@@ -0,0 +1,88 @@
+"""Wildcard patterns on KEYWORD fields must stay literal.
+
+``checksum`` is the only KEYWORD field: it is indexed with the raw tokenizer,
+so its terms are never lowercased, folded or stemmed. Running its wildcard
+patterns through the stemming normalizer rewrote hex prefixes ("ceded" ->
+"cede") and returned documents whose checksum did not start with what the user
+typed, which for an identity field is a wrong answer.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+
+from documents.models import Document
+from documents.search._registry import get_field_registry
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from whoosh_compat import FieldRegistry
+
+    from documents.search._backend import TantivyBackend
+
+pytestmark = [pytest.mark.search, pytest.mark.django_db]
+
+CEDEF00D = "cedef00ddeadbeef0123456789abcdef01234567"
+CEDEDEAD = "cededeadbeef567801234567" + "89abcdef01234567"
+
+
+def _normalizer(registry: FieldRegistry, name: str) -> Callable[[str], str]:
+    ref = registry.make_ref(name)
+    assert ref is not None
+    resolved = registry.resolve(ref)
+    assert resolved is not None
+    assert resolved.spec.pattern_normalizer is not None
+    return resolved.spec.pattern_normalizer
+
+
+class TestKeywordPatternNormalizer:
+    @pytest.mark.parametrize(
+        "run",
+        [
+            pytest.param("ceded", id="stems_to_cede"),
+            pytest.param("added", id="stems_to_ad"),
+            pytest.param("cafed", id="stems_to_cafe"),
+        ],
+    )
+    def test_keyword_runs_are_folded_not_stemmed(self, run: str) -> None:
+        normalize = _normalizer(get_field_registry("en"), "checksum")
+        assert normalize(run) == run
+
+    def test_text_runs_are_still_stemmed(self) -> None:
+        normalize = _normalizer(get_field_registry("en"), "title")
+        assert normalize("Running") == "run"
+
+
+class TestChecksumPrefixQueries:
+    @pytest.fixture
+    def indexed(self, backend: TantivyBackend) -> None:
+        for i, checksum in enumerate((CEDEF00D, CEDEDEAD)):
+            doc = Document.objects.create(
+                title=f"Checksum doc {i}",
+                content="invoices for the quarter",
+                checksum=checksum,
+                archive_serial_number=940 + i,
+            )
+            backend.add_or_update(doc)
+
+    def _ids(self, backend: TantivyBackend, query: str) -> set[int]:
+        return set(backend.search_ids(query, user=None))
+
+    def test_prefix_matches_only_the_document_that_starts_with_it(
+        self,
+        backend: TantivyBackend,
+        indexed: None,
+    ) -> None:
+        matched = self._ids(backend, "checksum:ceded*")
+        expected = Document.objects.get(checksum=CEDEDEAD).pk
+        assert matched == {expected}
+
+    def test_text_prefix_still_reaches_the_stemmed_index(
+        self,
+        backend: TantivyBackend,
+        indexed: None,
+    ) -> None:
+        assert len(self._ids(backend, "invoice*")) == 2