diff --git a/docs/usage.md b/docs/usage.md index b38e12fac..3d15b3683 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -933,7 +933,7 @@ original_filename:invoice.pdf - `asn` matches a document's Archive Serial Number. - `page_count` matches a document's page count. - `num_notes` matches how many notes a document has. -- `checksum` matches the checksum of the original document file (not the archived/processed version). Unlike the text fields, this one is stored verbatim rather than tokenized, so only a complete, lowercase checksum matches. To search by the first few characters instead, use a wildcard: `checksum:9f86d081*`. Wildcard patterns on the text fields are also tried stemmed, to line up with the stemmed index, but `checksum` is indexed without stemming, so its patterns are matched exactly as typed and nothing else. +- `checksum` matches the checksum of the original document file (not the archived/processed version). Unlike the text fields, this one is stored verbatim rather than tokenized, so only a complete, lowercase checksum matches. To search by the first few characters instead, use a wildcard: `checksum:9f86d081*`. Wildcard patterns on the text fields are also tried stemmed, to line up with the stemmed index, but `checksum` is indexed without stemming, so its patterns are not stemmed either: a wildcard prefix is matched literally, apart from being lowercased first. `checksum:9F86D081*` therefore does find the document, even though the plain uppercase term does not. - `original_filename` matches the filename of the document as originally consumed. `asn`, `page_count` and `num_notes` are numeric and also accept ranges, for example `asn:[50 to 150]`. @@ -952,10 +952,15 @@ pattern is tried both as you typed it and in its stemmed form, so a trailing and "invoiced") as well as longer words whose stored term still begins with what you typed (`copy*` finds "copyright" alongside "copy" and "copies"). -It is still not a plain prefix search over the original text: where stemming -shortens a word, a pattern that reaches past the point it was cut off matches -nothing. `productname` is stored as `productnam`, so `produ*name` finds -nothing, and "happiness" is stored as `happi`, so `happine*` does not find it. +It is still not a plain prefix search over the original text. A trailing `*` +matches a stored term when either the run you typed or its stemmed form is a +prefix of that term, so a fragment that stops part-way between the two matches +neither: `universities*` finds "university" and "universities", which are both +stored as `univers`, while the shorter `universit*` finds nothing at all. For +the same reason `happine*` does not find "happiness", which is stored as +`happi`. And a pattern that requires letters after the wildcard which stemming +has removed cannot match either: `productname` is stored as `productnam`, so +`produ*name` finds nothing. Matching natural date keywords: @@ -963,7 +968,9 @@ The multi-word date keywords listed below work quoted or unquoted after a date field (`added:"previous month"` and `added:previous month` are equivalent); elsewhere in a query the same words are treated as ordinary search text. Other date expressions the parser accepts (relative offsets -like `-1 week`, or specific dates like `12 december 2019`) must be quoted. +like `-1 week`, or specific dates like `12 december 2019`) must be quoted when +they stand alone as a value; inside a range's brackets they work unquoted, as +in `added:[-1 week to now]`. ``` added:today diff --git a/src/documents/tests/search/test_documented_syntax.py b/src/documents/tests/search/test_documented_syntax.py index 24dd25f45..9c4df4b5a 100644 --- a/src/documents/tests/search/test_documented_syntax.py +++ b/src/documents/tests/search/test_documented_syntax.py @@ -182,6 +182,11 @@ class TestArchiveMetadataFields: "original_filename:invoice.pdf", f"checksum:{DOC_CHECKSUM}", "checksum:9f86d081*", + # A checksum term is stored verbatim, but a checksum *pattern* is + # lowercased before it is matched, which the docs now say outright + # next to the "only a complete, lowercase checksum matches" rule + # that the uppercase term in the negative list below pins. + "checksum:9F86D081*", ], ) def test_documented_metadata_query_matches( diff --git a/src/documents/tests/search/test_pattern_stemming.py b/src/documents/tests/search/test_pattern_stemming.py index 41f926fb3..4c04d4179 100644 --- a/src/documents/tests/search/test_pattern_stemming.py +++ b/src/documents/tests/search/test_pattern_stemming.py @@ -78,18 +78,42 @@ class TestPrefixStemming: assert _matched_ids(backend, query) == {indexed_doc.id} @pytest.mark.parametrize("query", ["univers*", "librar*"]) - def test_partial_prefix_is_not_lengthened_by_its_stem( + def test_partial_prefix_reaches_the_stemmed_term( self, backend: TantivyBackend, indexed_doc: Document, query: str, ) -> None: - """A partial prefix keeps matching. "librar" stems to "librari", which - is longer than what was typed and so matches no term on its own, but - the run is offered as typed alongside its stem and that form reaches - "librari" in the index.""" + """A prefix shorter than a whole word still matches, and neither of + these needs the two-alternative path to do it. + + Measured under "en": the stemmer leaves "librar" alone, so it has one + form, and that form is a prefix of the "librari" the index holds for + "library". "univers" stems to the *shorter* "univ", and the run as + typed and its stem are both prefixes of the "univers" the index holds + for "university". The case where the two forms genuinely diverge, and + only one of them matches, is + test_stem_substitution_reaches_both_the_inflection_and_the_compound. + """ assert _matched_ids(backend, query) == {indexed_doc.id} + def test_full_word_reaches_the_stem_but_a_fragment_of_it_does_not( + self, + backend: TantivyBackend, + indexed_doc: Document, + ) -> None: + """The alternatives widen recall without turning a wildcard into a + prefix search over the original text. + + "university" is stored as "univers". The stem of "universities" is + that same "univers", so the longer word matches; "universit" is a + prefix of neither its own stem nor the stored term, so the *shorter* + fragment matches nothing. usage.md names this pair, so a reader told + that `universit*` fails is also told which spelling works. + """ + assert _matched_ids(backend, "universities*") == {indexed_doc.id} + assert _matched_ids(backend, "universit*") == set() + def test_pattern_past_the_stem_boundary_is_documented_not_fixed( self, backend: TantivyBackend,