From 82546f13b3a7be5332d00f43c170ddce7d014a5c Mon Sep 17 00:00:00 2001 From: stumpylog <797416+stumpylog@users.noreply.github.com> Date: Thu, 20 Aug 2026 12:08:47 -0700 Subject: [PATCH] fix(search): keep wildcard patterns literal on unstemmed KEYWORD fields get_field_registry() branched the analyzer by field kind but gave every field the same stemming pattern normalizer. checksum is indexed with the raw tokenizer, so its terms are neither folded nor stemmed, yet its wildcard patterns were: "checksum:ceded*" normalized to "cede*" and matched a document whose checksum starts with "cedef00d". About 2.8% of random hex prefixes were rewritten this way. Always over-matching rather than missing, but for a field whose whole purpose is exact identification, returning a different checksum is a wrong answer. KEYWORD fields now get a fold-and-lower normalizer, which is what every field used before pattern stemming was added; TEXT fields keep the stemming one so "invoice*" still reaches the indexed "invoic". Co-Authored-By: Claude Opus 5 --- docs/usage.md | 2 +- src/documents/search/_registry.py | 18 ++-- .../search/test_keyword_pattern_literal.py | 88 +++++++++++++++++++ 3 files changed, 102 insertions(+), 6 deletions(-) create mode 100644 src/documents/tests/search/test_keyword_pattern_literal.py diff --git a/docs/usage.md b/docs/usage.md index 63412bb49..2683d63a2 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -933,7 +933,7 @@ original_filename:invoice.pdf - `asn` matches a document's Archive Serial Number. - `page_count` matches a document's page count. - `num_notes` matches how many notes a document has. -- `checksum` matches the checksum of the original document file (not the archived/processed version). Unlike the text fields, this one is stored verbatim rather than tokenized, so only a complete, lowercase checksum matches. To search by the first few characters instead, use a wildcard: `checksum:9f86d081*`. Because the field is not stemmed, that prefix is matched literally. +- `checksum` matches the checksum of the original document file (not the archived/processed version). Unlike the text fields, this one is stored verbatim rather than tokenized, so only a complete, lowercase checksum matches. To search by the first few characters instead, use a wildcard: `checksum:9f86d081*`. Wildcard patterns on the text fields are stemmed to line up with the stemmed index, but `checksum` is indexed without stemming, so its patterns are not stemmed either and the prefix is matched exactly as typed. - `original_filename` matches the filename of the document as originally consumed. `asn`, `page_count` and `num_notes` are numeric and also accept ranges, for example `asn:[50 to 150]`. diff --git a/src/documents/search/_registry.py b/src/documents/search/_registry.py index 9877781ba..c813f8736 100644 --- a/src/documents/search/_registry.py +++ b/src/documents/search/_registry.py @@ -22,16 +22,22 @@ def _identity_analyzer(text: str) -> list[str]: return [text] +def _fold_normalizer(text: str) -> str: + """Wildcard/regex literal-run normalizer for fields indexed without stemming.""" + return ascii_fold(text.lower()) + + def _make_pattern_normalizer(language: str | None) -> Callable[[str], str]: """Build the wildcard/regex literal-run normalizer for a search language.""" def _pattern_normalizer(text: str) -> str: """Normalize a literal run so it can match indexed terms. - Index terms go through lowercase -> ascii_fold -> stem, so a pattern - that skips stemming can never match one: "invoice*" would look for a - term starting with "invoice" while the index holds "invoic". The run is - therefore stemmed here too. + TEXT index terms go through lowercase -> ascii_fold -> stem, so a + pattern that skips stemming can never match one: "invoice*" would look + for a term starting with "invoice" while the index holds "invoic". The + run is therefore stemmed here too. KEYWORD fields are indexed raw and + get _fold_normalizer instead, so their patterns stay literal. A stem can be longer than the fragment the user typed, though, and a longer prefix matches nothing, so the stem is used only when it is no @@ -67,7 +73,9 @@ def get_field_registry(language: str | None) -> FieldRegistry: analyzer=_identity_analyzer if field.kind is FieldKind.KEYWORD else text_analyzer, - pattern_normalizer=pattern_normalizer, + pattern_normalizer=_fold_normalizer + if field.kind is FieldKind.KEYWORD + else pattern_normalizer, ) for field in PUBLIC_FIELDS ] diff --git a/src/documents/tests/search/test_keyword_pattern_literal.py b/src/documents/tests/search/test_keyword_pattern_literal.py new file mode 100644 index 000000000..92fee11bb --- /dev/null +++ b/src/documents/tests/search/test_keyword_pattern_literal.py @@ -0,0 +1,88 @@ +"""Wildcard patterns on KEYWORD fields must stay literal. + +``checksum`` is the only KEYWORD field: it is indexed with the raw tokenizer, +so its terms are never lowercased, folded or stemmed. Running its wildcard +patterns through the stemming normalizer rewrote hex prefixes ("ceded" -> +"cede") and returned documents whose checksum did not start with what the user +typed, which for an identity field is a wrong answer. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from documents.models import Document +from documents.search._registry import get_field_registry + +if TYPE_CHECKING: + from collections.abc import Callable + + from whoosh_compat import FieldRegistry + + from documents.search._backend import TantivyBackend + +pytestmark = [pytest.mark.search, pytest.mark.django_db] + +CEDEF00D = "cedef00ddeadbeef0123456789abcdef01234567" +CEDEDEAD = "cededeadbeef567801234567" + "89abcdef01234567" + + +def _normalizer(registry: FieldRegistry, name: str) -> Callable[[str], str]: + ref = registry.make_ref(name) + assert ref is not None + resolved = registry.resolve(ref) + assert resolved is not None + assert resolved.spec.pattern_normalizer is not None + return resolved.spec.pattern_normalizer + + +class TestKeywordPatternNormalizer: + @pytest.mark.parametrize( + "run", + [ + pytest.param("ceded", id="stems_to_cede"), + pytest.param("added", id="stems_to_ad"), + pytest.param("cafed", id="stems_to_cafe"), + ], + ) + def test_keyword_runs_are_folded_not_stemmed(self, run: str) -> None: + normalize = _normalizer(get_field_registry("en"), "checksum") + assert normalize(run) == run + + def test_text_runs_are_still_stemmed(self) -> None: + normalize = _normalizer(get_field_registry("en"), "title") + assert normalize("Running") == "run" + + +class TestChecksumPrefixQueries: + @pytest.fixture + def indexed(self, backend: TantivyBackend) -> None: + for i, checksum in enumerate((CEDEF00D, CEDEDEAD)): + doc = Document.objects.create( + title=f"Checksum doc {i}", + content="invoices for the quarter", + checksum=checksum, + archive_serial_number=940 + i, + ) + backend.add_or_update(doc) + + def _ids(self, backend: TantivyBackend, query: str) -> set[int]: + return set(backend.search_ids(query, user=None)) + + def test_prefix_matches_only_the_document_that_starts_with_it( + self, + backend: TantivyBackend, + indexed: None, + ) -> None: + matched = self._ids(backend, "checksum:ceded*") + expected = Document.objects.get(checksum=CEDEDEAD).pk + assert matched == {expected} + + def test_text_prefix_still_reaches_the_stemmed_index( + self, + backend: TantivyBackend, + indexed: None, + ) -> None: + assert len(self._ids(backend, "invoice*")) == 2