diff --git a/docs/usage.md b/docs/usage.md index fd2ff3b99..1611de109 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -927,10 +927,12 @@ title:Invoice* ``` Wildcards are matched against the _stemmed_ terms stored in the index, not -against the words as they appear in the document. A trailing `*` therefore -works as expected (`invoice*` finds "invoice", "invoices" and "invoiced"), but -a pattern whose text continues past where stemming cuts a word off cannot -match: `productname` is indexed as `productnam`, so `produ*name` finds nothing. +against the words as they appear in the document. A trailing `*` matches a word +and its inflections (`invoice*` finds "invoice", "invoices" and "invoiced"), +but not every longer word that starts with the same letters: `copy*` finds +"copy" and "copies", not "copyright". For the same reason, a pattern whose text +continues past where stemming cuts a word off cannot match at all: +`productname` is indexed as `productnam`, so `produ*name` finds nothing. Matching natural date keywords: diff --git a/src/documents/search/_tokenizer.py b/src/documents/search/_tokenizer.py index fddddbc9c..beadc0312 100644 --- a/src/documents/search/_tokenizer.py +++ b/src/documents/search/_tokenizer.py @@ -131,11 +131,14 @@ def stem_pattern_text(text: str, language: str | None) -> str: """Stem an already lowercased/ascii-folded run the way index terms are. Returns text unchanged when stemming is disabled for language, and also - when the stem step does not yield exactly one token: remove_long drops an - over-long run entirely, and there is no single stem to substitute for a run - that analyzes to several. Falling back to the text as typed is the safe - direction for a pattern prefix, since it can only be as narrow as it was - before stemming was considered. + when the stem step does not yield exactly one token: remove_long drops a run + past the length limit, leaving no stem to substitute. Falling back to the + text as typed is the safe direction for a pattern prefix, since it can only + be as narrow as it was before stemming was considered. + + The raw tokenizer emits one token whatever the input and the stemmer is + 1-to-1, so only the zero-token case can fire today; the guard covers both + counts so a tokenizer change cannot turn this into an IndexError. """ analyzer = _pattern_stemmer(language) if analyzer is None: diff --git a/src/documents/tests/search/test_pattern_stemming.py b/src/documents/tests/search/test_pattern_stemming.py index 6fddec710..7442413af 100644 --- a/src/documents/tests/search/test_pattern_stemming.py +++ b/src/documents/tests/search/test_pattern_stemming.py @@ -15,6 +15,9 @@ import pytest from documents.models import Document from documents.search._registry import _make_pattern_normalizer +from documents.search._tokenizer import ascii_fold +from documents.search._tokenizer import paperless_text_analyzer +from documents.search._tokenizer import stem_pattern_text if TYPE_CHECKING: from documents.search._backend import TantivyBackend @@ -23,7 +26,7 @@ pytestmark = [pytest.mark.search, pytest.mark.django_db] CONTENT = ( "invoice total due for electricity from both companies, " - "payments made to the university library" + "payments made to the university library, copies attached" ) @@ -94,6 +97,51 @@ class TestPrefixStemming: is deliberate, not accidental.""" assert _matched_ids(backend, "produ*name") == set() + def test_stem_substitution_loses_compounds_accepted_trade( + self, + backend: TantivyBackend, + indexed_doc: Document, + ) -> None: + """English stemming substitutes as well as truncates: "copy" and + "copies" both index as "copi", while "copyright" keeps its literal "y". + So "copy*" reaches the base word and its inflections but no longer + reaches the compound, which it did before patterns were stemmed. That + trade is deliberate: the same substitution is what makes "company*" and + "library*" work at all, and no rule over one normalized string tells the + two apart. Matching both would need the pattern to be emitted as a + disjunction of the folded and stemmed forms, which belongs in the + emitter, not here. + """ + compound = Document.objects.create( + title="Copyright notice", + content="copyright notice for the work", + checksum="pattern-stemming-2", + archive_serial_number=901, + ) + backend.add_or_update(compound) + + assert _matched_ids(backend, "copy*") == {indexed_doc.id} + assert _matched_ids(backend, "copyright*") == {compound.id} + + +class TestStemsMatchTheIndexAnalyzer: + """stem_pattern_text rebuilds paperless_text_analyzer's stemming tail rather + than sharing it, so a filter added to the index analyzer alone would silently + stop patterns from reaching the terms it produces. + """ + + @pytest.mark.parametrize( + "language", + ["en", "de", "fr", "es", "sv", None, "klingon"], + ) + @pytest.mark.parametrize( + "word", + ["Copies", "copyright", "Companies", "Invoices", "laufen", "casas", "Straße"], + ) + def test_stem_equals_the_index_term(self, word: str, language: str | None) -> None: + indexed = paperless_text_analyzer(language).analyze(word)[0] + assert stem_pattern_text(ascii_fold(word.lower()), language) == indexed + class TestPatternNormalizer: @pytest.mark.parametrize(