diff --git a/src/documents/tests/search/test_compact_date_forms.py b/src/documents/tests/search/test_compact_date_forms.py index 987994c15..48e0f2fd2 100644 --- a/src/documents/tests/search/test_compact_date_forms.py +++ b/src/documents/tests/search/test_compact_date_forms.py @@ -1,14 +1,13 @@ """Whoosh's compact, separator-free date spelling, resolved end to end. -whoosh-compat owns both widths of this spelling and asserts both of each -form's bounds directly: ``test_compact_numeric_datetime`` pins the 8-digit -form as a whole calendar day (lower bound, upper bound and exclusivity), and -``test_compact_numeric_datetime_full_width_is_a_single_second_instant`` pins -the 14-digit form as one instant. The 14-digit form is kept here as the single -representative because it is the one that exercises paperless's ``added`` -DATETIME fast field at full precision: the corpus separates a document at -the named instant from one on the same calendar day at another hour and one -on the next day at the same hour, so a query that degrades into a whole-day +whoosh-compat owns both widths of this spelling and asserts both forms' +bounds directly in its own test suite: the 8-digit form as a whole calendar +day (lower bound, upper bound and exclusivity), and the 14-digit form as a +single instant. The 14-digit form is kept here as the single representative +because it is the one that exercises paperless's ``added`` DATETIME fast +field at full precision: the corpus separates a document at the named +instant from one on the same calendar day at another hour and one on the +next day at the same hour, so a query that degrades into a whole-day window, or drops the time of day, matches the wrong set rather than passing on a corpus that could not tell the difference. """ @@ -70,6 +69,18 @@ def test_fourteen_digits_is_a_single_instant( backend: TantivyBackend, docs: dict[str, int], ) -> None: - # same_day is what tells this apart from the 8-digit day-window form, - # next_day from a form that ignored the time altogether. + """ + GIVEN: + - Three documents indexed on the ``added`` DATETIME fast field: + one at 2005-03-04T15:30:00, one on the same calendar day at a + different hour, and one on the next day at the same hour + WHEN: + - Searching with the 14-digit compact date form + ``added:20050304153000`` + THEN: + - Only the document at that exact instant matches; the same-day + document is what tells this apart from the 8-digit day-window + form, and the next-day document from a form that ignored the + time of day altogether + """ assert _matched_ids(backend, "added:20050304153000") == {docs["instant"]} diff --git a/src/documents/tests/search/test_documented_syntax.py b/src/documents/tests/search/test_documented_syntax.py index 0071a66e0..244f7bb5c 100644 --- a/src/documents/tests/search/test_documented_syntax.py +++ b/src/documents/tests/search/test_documented_syntax.py @@ -77,6 +77,14 @@ class TestLogicalExpressions: backend: TantivyBackend, docs: dict[str, int], ) -> None: + """ + GIVEN: + - Two indexed documents, one containing "secret" and one not + WHEN: + - "invoice NOT secret" is searched, as docs/usage.md documents + THEN: + - Only the document without "secret" matches + """ assert _matched_ids(backend, "invoice NOT secret") == {docs["plain"]} def test_leading_hyphen_requires_the_term_instead_of_excluding_it( @@ -84,8 +92,17 @@ class TestLogicalExpressions: backend: TantivyBackend, docs: dict[str, int], ) -> None: - # The docs warn about exactly this: separators are stripped at index - # time, so "-secret" is the term "secret" and the query is an AND. + """ + GIVEN: + - Two indexed documents, one containing "secret" and one not + WHEN: + - "invoice -secret" is searched (a leading hyphen, not "NOT") + THEN: + - Only the document containing "secret" matches, because + separators are stripped at index time, so "-secret" is + indexed as the plain term "secret" and the query becomes an + AND rather than an exclusion, exactly as the docs warn + """ assert _matched_ids(backend, "invoice -secret") == {docs["secret"]} def test_or_inside_parentheses_matches_either_branch( @@ -93,6 +110,15 @@ class TestLogicalExpressions: backend: TantivyBackend, docs: dict[str, int], ) -> None: + """ + GIVEN: + - Two indexed documents, one containing "secret" and one + containing "ordinary" + WHEN: + - "invoice AND (secret OR ordinary)" is searched + THEN: + - Both documents match + """ matched = _matched_ids(backend, "invoice AND (secret OR ordinary)") assert matched == {docs["secret"], docs["plain"]} @@ -102,6 +128,15 @@ class TestPhraseSearch: self, backend: TantivyBackend, ) -> None: + """ + GIVEN: + - A document whose content contains "the quick brown fox jumps" + WHEN: + - A quoted phrase is searched, in order and out of order + THEN: + - The in-order phrase matches, and the same words reordered do + not + """ doc = _index( backend, title="Phrase", @@ -131,6 +166,16 @@ class TestTagCommaList: self, backend: TantivyBackend, ) -> None: + """ + GIVEN: + - A document carrying both "bills" and "unpaid" tags, and a + second document carrying only "bills" (plus "archived") + WHEN: + - "tag:bills,unpaid" is searched + THEN: + - Only the document carrying every listed tag matches, and a + single-tag "tag:bills" search still matches both documents + """ bills = Tag.objects.create(name="bills") unpaid = Tag.objects.create(name="unpaid") archived = Tag.objects.create(name="archived") @@ -195,6 +240,18 @@ class TestArchiveMetadataFields: doc: Document, query: str, ) -> None: + """ + GIVEN: + - A document with an ASN, page count, a note, an original + filename and a known checksum + WHEN: + - Every documented metadata-field spelling (exact value, + range, and, for checksum, a lowercase prefix pattern + regardless of the case the pattern itself is typed in) is + searched + THEN: + - Each one matches the document + """ assert _matched_ids(backend, query) == {doc.pk} @pytest.mark.parametrize( @@ -211,6 +268,16 @@ class TestArchiveMetadataFields: doc: Document, query: str, ) -> None: + """ + GIVEN: + - A document with a known, complete, lowercase checksum + WHEN: + - An exact-value search is run with a partial or uppercase + spelling of that checksum + THEN: + - Nothing matches, as the docs say only a complete, lowercase + checksum matches as an exact value + """ assert _matched_ids(backend, query) == set() @@ -272,6 +339,21 @@ class TestDocumentedDateForms: query: str, label: str, ) -> None: + """ + GIVEN: + - Documents dated today, yesterday, tomorrow, next/last + Monday, in January, and on an old fixed date, indexed + against a frozen "now" (a Monday) + WHEN: + - Every documented date-form spelling is searched: relative + keywords, quoted multi-word phrases, a bare year-month, an + explicit range, a quoted full timestamp standing alone, an + unquoted full timestamp as a range bound, and a + single-quoted range bound + THEN: + - Each form matches exactly the document dated on its day or + within its month + """ assert _matched_ids(backend, query) == {dated[label]} @pytest.mark.parametrize( @@ -302,6 +384,21 @@ class TestDocumentedDateForms: dated: dict[str, int], query: str, ) -> None: + """ + GIVEN: + - A realistic dated corpus (see the `dated` fixture) + WHEN: + - A zero-width date form ("now", "noon", "midnight", a quoted + "now") or a standalone relative offset ("-1 week") is + searched: each resolves to a single instant rather than a + span, and quoting does not rescue them the way it rescues + other multi-word date expressions, since the problem is the + width of the resulting range, not how the value is + delimited + THEN: + - Nothing matches, exactly as the docs warn, rather than + presenting these as usable spellings + """ assert _matched_ids(backend, query) == set() def test_bare_timestamp_is_rejected_rather_than_matching_nothing( @@ -309,13 +406,20 @@ class TestDocumentedDateForms: backend: TantivyBackend, dated: dict[str, int], ) -> None: - """The bare, unquoted spelling of a full timestamp. The quoted and - range-bound spellings pinned above do work and match this fixture's - document; this one is a user-fixable error rather than an empty - result set, so the docs tell the user to quote it. - - The reported value is the whole contiguous fragment the user typed, - not just the prefix the date grammar's tokenizer first split on. + """ + GIVEN: + - A realistic dated corpus, including a document dated at a + known full timestamp + WHEN: + - The bare, unquoted spelling of that full timestamp is + searched (the quoted and range-bound spellings pinned above + do work and match this fixture's document) + THEN: + - `InvalidDateQuery` is raised rather than the query silently + matching nothing, since this is a user-fixable error the + docs tell the user to quote, and the reported value is the + whole contiguous fragment the user typed, not just the + prefix the date grammar's tokenizer first split on """ with pytest.raises(InvalidDateQuery) as exc_info: _matched_ids(backend, "added:2005-03-04T15:30:00Z") @@ -327,13 +431,20 @@ class TestDocumentedDateForms: backend: TantivyBackend, dated: dict[str, int], ) -> None: - """The same offset that matches nothing on its own spans the last - seven days as a lower bound. The docs say so, next to the warning - about the standalone form, so both readings are pinned together. - - "last_monday" is indexed at 2026-06-08T10:00, two hours before the - window opens, so its exclusion is what shows the bound is the offset - and not a whole-day rounding of it. + """ + GIVEN: + - A realistic dated corpus, including a document dated two + hours before a "last Monday to now" window opens, and + documents dated today and yesterday, inside that window + WHEN: + - "added:['-1 week' to now]" is searched: the same offset + that matches nothing standing alone (see the test above), + used here as a range bound instead + THEN: + - The window matches today and yesterday but excludes the + document two hours before it opens, showing the bound is + the offset itself and not a whole-day rounding of it, as + the docs say next to the warning about the standalone form """ assert _matched_ids(backend, "added:['-1 week' to now]") == { dated["today"], @@ -345,10 +456,18 @@ class TestDocumentedDateForms: backend: TantivyBackend, dated: dict[str, int], ) -> None: - """Quoting a range bound is allowed, but only with single quotes: the - double-quoted spelling reaches the date grammar with its quotes still - attached and is not a recognizable date. The docs say so, so pin which - of the two quote characters is the one that fails. + """ + GIVEN: + - A realistic dated corpus + WHEN: + - A range bound is double-quoted rather than single-quoted + ("added:[\"2005-03-04\" to 2005-03-05]") + THEN: + - `InvalidDateQuery` is raised, pinning which of the two + quote characters fails: quoting a range bound is allowed, + but only with single quotes, since the double-quoted + spelling reaches the date grammar with its quotes still + attached and is not a recognizable date """ with pytest.raises(InvalidDateQuery) as exc_info: _matched_ids(backend, 'added:["2005-03-04" to 2005-03-05]') diff --git a/src/documents/tests/search/test_pattern_stemming.py b/src/documents/tests/search/test_pattern_stemming.py index 4c04d4179..36b5e85dd 100644 --- a/src/documents/tests/search/test_pattern_stemming.py +++ b/src/documents/tests/search/test_pattern_stemming.py @@ -66,6 +66,19 @@ class TestPrefixStemming: indexed_doc: Document, query: str, ) -> None: + """ + GIVEN: + - A document indexed with content containing "invoice", + "electricity", "companies", "payments", "library" and title + "Invoice 2020 productname" + WHEN: + - A prefix wildcard on the full, unstemmed word is queried + (e.g. "invoice*", "title:Invoice*") + THEN: + - The document matches, since the pattern normalizer offers + the word's stem as an alternative alongside the typed run, + reaching the stemmed index term + """ assert _matched_ids(backend, query) == {indexed_doc.id} @pytest.mark.parametrize("query", ["invoic*", "electr*", "payment*"]) @@ -75,6 +88,16 @@ class TestPrefixStemming: indexed_doc: Document, query: str, ) -> None: + """ + GIVEN: + - The same indexed document + WHEN: + - A prefix wildcard is typed already in its stemmed spelling + (e.g. "invoic*") + THEN: + - The document still matches, since the typed-run alternative + is itself a prefix of the stored stemmed term + """ assert _matched_ids(backend, query) == {indexed_doc.id} @pytest.mark.parametrize("query", ["univers*", "librar*"]) @@ -84,16 +107,22 @@ class TestPrefixStemming: indexed_doc: Document, query: str, ) -> None: - """A prefix shorter than a whole word still matches, and neither of - these needs the two-alternative path to do it. - - Measured under "en": the stemmer leaves "librar" alone, so it has one - form, and that form is a prefix of the "librari" the index holds for - "library". "univers" stems to the *shorter* "univ", and the run as - typed and its stem are both prefixes of the "univers" the index holds - for "university". The case where the two forms genuinely diverge, and - only one of them matches, is - test_stem_substitution_reaches_both_the_inflection_and_the_compound. + """ + GIVEN: + - The same indexed document + WHEN: + - A prefix shorter than a whole word is queried ("univers*", + "librar*") + THEN: + - It still matches, and neither case needs the two-alternative + path to do it: measured under "en", the stemmer leaves + "librar" alone, so it has one form, and that form is a + prefix of the "librari" the index holds for "library"; + "univers" stems to the *shorter* "univ", and the run as + typed and its stem are both prefixes of the "univers" the + index holds for "university". The case where the two forms + genuinely diverge, and only one of them matches, is + test_stem_substitution_reaches_both_the_inflection_and_the_compound """ assert _matched_ids(backend, query) == {indexed_doc.id} @@ -102,14 +131,19 @@ class TestPrefixStemming: backend: TantivyBackend, indexed_doc: Document, ) -> None: - """The alternatives widen recall without turning a wildcard into a - prefix search over the original text. - - "university" is stored as "univers". The stem of "universities" is - that same "univers", so the longer word matches; "universit" is a - prefix of neither its own stem nor the stored term, so the *shorter* - fragment matches nothing. usage.md names this pair, so a reader told - that `universit*` fails is also told which spelling works. + """ + GIVEN: + - The same indexed document, storing "university" as "univers" + WHEN: + - "universities*" and "universit*" are each queried + THEN: + - "universities*" matches, since the stem of "universities" is + that same "univers"; "universit*" matches nothing, since + "universit" is a prefix of neither its own stem nor the + stored term. The alternatives widen recall without turning + a wildcard into a prefix search over the original text, and + usage.md names this exact pair so a reader told that + `universit*` fails is also told which spelling works """ assert _matched_ids(backend, "universities*") == {indexed_doc.id} assert _matched_ids(backend, "universit*") == set() @@ -119,9 +153,18 @@ class TestPrefixStemming: backend: TantivyBackend, indexed_doc: Document, ) -> None: - """produ*name cannot match a stemmed index ("productname" is indexed as - "productnam"); usage.md must not advertise it. Pinned so the limitation - is deliberate, not accidental.""" + """ + GIVEN: + - The same indexed document, with "productname" indexed as + "productnam" + WHEN: + - "produ*name" (a pattern straddling the stem boundary) is + queried + THEN: + - It matches nothing; produ*name cannot match a stemmed + index, and usage.md must not advertise it. Pinned so the + limitation is deliberate, not accidental + """ assert _matched_ids(backend, "produ*name") == set() def test_stem_substitution_reaches_both_the_inflection_and_the_compound( @@ -129,12 +172,22 @@ class TestPrefixStemming: backend: TantivyBackend, indexed_doc: Document, ) -> None: - """English stemming substitutes as well as truncates: "copy" and - "copies" both index as "copi", while "copyright" keeps its literal "y". - Neither form is a prefix of the other, so no single normalized string - reaches both. The run is therefore emitted as a disjunction of the - folded and stemmed forms, and "copy*" reaches the base word, its - inflections and the compound alike. + """ + GIVEN: + - The indexed document (containing "copies") plus a second + document titled "Copyright notice" with content "copyright + notice for the work" + WHEN: + - "copy*" and "copyright*" are each queried + THEN: + - "copy*" matches both documents, and "copyright*" matches + only the compound one. English stemming substitutes as well + as truncates: "copy" and "copies" both index as "copi", + while "copyright" keeps its literal "y". Neither form is a + prefix of the other, so no single normalized string reaches + both; the run is therefore emitted as a disjunction of the + folded and stemmed forms, and "copy*" reaches the base + word, its inflections and the compound alike """ compound = Document.objects.create( title="Copyright notice", @@ -163,6 +216,22 @@ class TestStemsMatchTheIndexAnalyzer: ["Copies", "copyright", "Companies", "Invoices", "laufen", "casas", "Straße"], ) def test_stem_equals_the_index_term(self, word: str, language: str | None) -> None: + """ + GIVEN: + - A word, across several representative index languages + ("en", "de", "fr", "es", "sv"), no language, and an + unsupported language ("klingon") + WHEN: + - `stem_pattern_text` (the pattern-side stemmer) processes the + folded word, and `paperless_text_analyzer` (the index-side + analyzer) independently processes the same word + THEN: + - The two produce the identical term. `stem_pattern_text` + rebuilds `paperless_text_analyzer`'s stemming tail rather + than sharing it, so a filter added to the index analyzer + alone would silently stop patterns from reaching the terms + it produces; this pins the two staying in sync + """ indexed = paperless_text_analyzer(language).analyze(word)[0] assert stem_pattern_text(ascii_fold(word.lower()), language) == indexed @@ -197,26 +266,64 @@ class TestPatternNormalizer: text: str, expected: tuple[str, ...], ) -> None: + """ + GIVEN: + - The "en" pattern normalizer + WHEN: + - It processes a literal run (e.g. "Invoice", "library", + "Café") + THEN: + - It returns the folded run and, where it differs, the + stemmed form, as distinct alternatives; a run the stemmer + leaves alone (e.g. "invoic") collapses back to the single + folded form. "library" needs both forms since y -> i is a + substitution: the index holds "librari" for "library" and + "library" for "librarian" + """ assert _forms(_make_pattern_normalizer("en"), text) == expected def test_run_that_yields_no_token_falls_back_to_the_typed_run(self) -> None: - """A run past the remove_long limit analyzes to zero tokens, so there is - no stem to offer and only the folded run remains.""" + """ + GIVEN: + - The "en" pattern normalizer + WHEN: + - It processes a run past the analyzer's remove_long limit + THEN: + - The run analyzes to zero tokens, so there is no stem to + offer, and only the folded run remains + """ over_long = "invoices" * 20 assert _forms(_make_pattern_normalizer("en"), over_long) == (over_long,) @pytest.mark.parametrize("language", [None, "klingon"]) def test_unstemmed_language_folds_only(self, language: str | None) -> None: - """With no stemmer configured, or one this build has no stemmer for, the - index holds surface forms and the pattern must keep them too.""" + """ + GIVEN: + - A pattern normalizer with no language configured, or one + this build has no stemmer for ("klingon") + WHEN: + - It processes "Invoices" + THEN: + - Only the folded form ("invoices") is offered, since with no + stemmer configured the index holds surface forms and the + pattern must keep them too + """ assert _forms(_make_pattern_normalizer(language), "Invoices") == ("invoices",) @pytest.mark.parametrize("char", ["a", "Z", "é"]) def test_a_single_character_collapses_to_one_folded_form(self, char: str) -> None: - """A bracket class body is normalized one character at a time and the - answer is used only when it is a single one-character form, so a - stemmer that changed a lone character would silently disable folding - inside classes.""" + """ + GIVEN: + - The "en" pattern normalizer + WHEN: + - It processes a single character + THEN: + - Exactly one, one-character form is returned. A bracket + class body is normalized one character at a time and the + answer is used only when it is a single one-character + form, so a stemmer that changed a lone character would + silently disable folding inside classes + """ forms = _forms(_make_pattern_normalizer("en"), char) assert len(forms) == 1 assert len(forms[0]) == 1 @@ -228,6 +335,15 @@ class TestBracketClassStillFolds: backend: TantivyBackend, indexed_doc: Document, ) -> None: - """The class body is folded per character, which the alternatives - contract preserves only because a lone character stems to itself.""" + """ + GIVEN: + - The indexed document, titled "Invoice 2020 productname" + WHEN: + - A bracket-class pattern mixing case is queried + ("title:[IP]nvoice*") + THEN: + - It matches: the class body is folded per character, which + the alternatives contract preserves only because a lone + character stems to itself + """ assert _matched_ids(backend, "title:[IP]nvoice*") == {indexed_doc.id}