diff --git a/docs/usage.md b/docs/usage.md index 63412bb49..2683d63a2 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -933,7 +933,7 @@ original_filename:invoice.pdf - `asn` matches a document's Archive Serial Number. - `page_count` matches a document's page count. - `num_notes` matches how many notes a document has. -- `checksum` matches the checksum of the original document file (not the archived/processed version). Unlike the text fields, this one is stored verbatim rather than tokenized, so only a complete, lowercase checksum matches. To search by the first few characters instead, use a wildcard: `checksum:9f86d081*`. Because the field is not stemmed, that prefix is matched literally. +- `checksum` matches the checksum of the original document file (not the archived/processed version). Unlike the text fields, this one is stored verbatim rather than tokenized, so only a complete, lowercase checksum matches. To search by the first few characters instead, use a wildcard: `checksum:9f86d081*`. Wildcard patterns on the text fields are stemmed to line up with the stemmed index, but `checksum` is indexed without stemming, so its patterns are not stemmed either and the prefix is matched exactly as typed. - `original_filename` matches the filename of the document as originally consumed. `asn`, `page_count` and `num_notes` are numeric and also accept ranges, for example `asn:[50 to 150]`. diff --git a/src/documents/search/_registry.py b/src/documents/search/_registry.py index 9877781ba..c813f8736 100644 --- a/src/documents/search/_registry.py +++ b/src/documents/search/_registry.py @@ -22,16 +22,22 @@ def _identity_analyzer(text: str) -> list[str]: return [text] +def _fold_normalizer(text: str) -> str: + """Wildcard/regex literal-run normalizer for fields indexed without stemming.""" + return ascii_fold(text.lower()) + + def _make_pattern_normalizer(language: str | None) -> Callable[[str], str]: """Build the wildcard/regex literal-run normalizer for a search language.""" def _pattern_normalizer(text: str) -> str: """Normalize a literal run so it can match indexed terms. - Index terms go through lowercase -> ascii_fold -> stem, so a pattern - that skips stemming can never match one: "invoice*" would look for a - term starting with "invoice" while the index holds "invoic". The run is - therefore stemmed here too. + TEXT index terms go through lowercase -> ascii_fold -> stem, so a + pattern that skips stemming can never match one: "invoice*" would look + for a term starting with "invoice" while the index holds "invoic". The + run is therefore stemmed here too. KEYWORD fields are indexed raw and + get _fold_normalizer instead, so their patterns stay literal. A stem can be longer than the fragment the user typed, though, and a longer prefix matches nothing, so the stem is used only when it is no @@ -67,7 +73,9 @@ def get_field_registry(language: str | None) -> FieldRegistry: analyzer=_identity_analyzer if field.kind is FieldKind.KEYWORD else text_analyzer, - pattern_normalizer=pattern_normalizer, + pattern_normalizer=_fold_normalizer + if field.kind is FieldKind.KEYWORD + else pattern_normalizer, ) for field in PUBLIC_FIELDS ] diff --git a/src/documents/tests/search/test_keyword_pattern_literal.py b/src/documents/tests/search/test_keyword_pattern_literal.py new file mode 100644 index 000000000..92fee11bb --- /dev/null +++ b/src/documents/tests/search/test_keyword_pattern_literal.py @@ -0,0 +1,88 @@ +"""Wildcard patterns on KEYWORD fields must stay literal. + +``checksum`` is the only KEYWORD field: it is indexed with the raw tokenizer, +so its terms are never lowercased, folded or stemmed. Running its wildcard +patterns through the stemming normalizer rewrote hex prefixes ("ceded" -> +"cede") and returned documents whose checksum did not start with what the user +typed, which for an identity field is a wrong answer. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from documents.models import Document +from documents.search._registry import get_field_registry + +if TYPE_CHECKING: + from collections.abc import Callable + + from whoosh_compat import FieldRegistry + + from documents.search._backend import TantivyBackend + +pytestmark = [pytest.mark.search, pytest.mark.django_db] + +CEDEF00D = "cedef00ddeadbeef0123456789abcdef01234567" +CEDEDEAD = "cededeadbeef567801234567" + "89abcdef01234567" + + +def _normalizer(registry: FieldRegistry, name: str) -> Callable[[str], str]: + ref = registry.make_ref(name) + assert ref is not None + resolved = registry.resolve(ref) + assert resolved is not None + assert resolved.spec.pattern_normalizer is not None + return resolved.spec.pattern_normalizer + + +class TestKeywordPatternNormalizer: + @pytest.mark.parametrize( + "run", + [ + pytest.param("ceded", id="stems_to_cede"), + pytest.param("added", id="stems_to_ad"), + pytest.param("cafed", id="stems_to_cafe"), + ], + ) + def test_keyword_runs_are_folded_not_stemmed(self, run: str) -> None: + normalize = _normalizer(get_field_registry("en"), "checksum") + assert normalize(run) == run + + def test_text_runs_are_still_stemmed(self) -> None: + normalize = _normalizer(get_field_registry("en"), "title") + assert normalize("Running") == "run" + + +class TestChecksumPrefixQueries: + @pytest.fixture + def indexed(self, backend: TantivyBackend) -> None: + for i, checksum in enumerate((CEDEF00D, CEDEDEAD)): + doc = Document.objects.create( + title=f"Checksum doc {i}", + content="invoices for the quarter", + checksum=checksum, + archive_serial_number=940 + i, + ) + backend.add_or_update(doc) + + def _ids(self, backend: TantivyBackend, query: str) -> set[int]: + return set(backend.search_ids(query, user=None)) + + def test_prefix_matches_only_the_document_that_starts_with_it( + self, + backend: TantivyBackend, + indexed: None, + ) -> None: + matched = self._ids(backend, "checksum:ceded*") + expected = Document.objects.get(checksum=CEDEDEAD).pk + assert matched == {expected} + + def test_text_prefix_still_reaches_the_stemmed_index( + self, + backend: TantivyBackend, + indexed: None, + ) -> None: + assert len(self._ids(backend, "invoice*")) == 2