diff --git a/src/documents/tests/search/test_compact_date_forms.py b/src/documents/tests/search/test_compact_date_forms.py new file mode 100644 index 000000000..987994c15 --- /dev/null +++ b/src/documents/tests/search/test_compact_date_forms.py @@ -0,0 +1,75 @@ +"""Whoosh's compact, separator-free date spelling, resolved end to end. + +whoosh-compat owns both widths of this spelling and asserts both of each +form's bounds directly: ``test_compact_numeric_datetime`` pins the 8-digit +form as a whole calendar day (lower bound, upper bound and exclusivity), and +``test_compact_numeric_datetime_full_width_is_a_single_second_instant`` pins +the 14-digit form as one instant. The 14-digit form is kept here as the single +representative because it is the one that exercises paperless's ``added`` +DATETIME fast field at full precision: the corpus separates a document at +the named instant from one on the same calendar day at another hour and one +on the next day at the same hour, so a query that degrades into a whole-day +window, or drops the time of day, matches the wrong set rather than passing +on a corpus that could not tell the difference. +""" + +from __future__ import annotations + +from datetime import UTC +from datetime import datetime +from typing import TYPE_CHECKING + +import pytest + +from documents.models import Document + +if TYPE_CHECKING: + from documents.search._backend import TantivyBackend + +pytestmark = [pytest.mark.search, pytest.mark.django_db] + + +def _matched_ids(backend: TantivyBackend, query: str) -> set[int]: + return set(backend.search_ids(query, user=None)) + + +def _index(backend: TantivyBackend, **kwargs: object) -> Document: + doc = Document.objects.create(**kwargs) + backend.add_or_update(doc) + return doc + + +@pytest.fixture +def docs(backend: TantivyBackend) -> dict[str, int]: + return { + "instant": _index( + backend, + title="On the instant", + content="x", + checksum="compact-date-instant", + added=datetime(2005, 3, 4, 15, 30, tzinfo=UTC), + ).pk, + "same_day": _index( + backend, + title="Same day, other hour", + content="x", + checksum="compact-date-same-day", + added=datetime(2005, 3, 4, 9, 0, tzinfo=UTC), + ).pk, + "next_day": _index( + backend, + title="Next day, same hour", + content="x", + checksum="compact-date-next-day", + added=datetime(2005, 3, 5, 15, 30, tzinfo=UTC), + ).pk, + } + + +def test_fourteen_digits_is_a_single_instant( + backend: TantivyBackend, + docs: dict[str, int], +) -> None: + # same_day is what tells this apart from the 8-digit day-window form, + # next_day from a form that ignored the time altogether. + assert _matched_ids(backend, "added:20050304153000") == {docs["instant"]} diff --git a/src/documents/tests/search/test_documented_syntax.py b/src/documents/tests/search/test_documented_syntax.py new file mode 100644 index 000000000..0071a66e0 --- /dev/null +++ b/src/documents/tests/search/test_documented_syntax.py @@ -0,0 +1,355 @@ +"""Pins the search syntax that ``docs/usage.md`` promises users. + +Every query here appears verbatim, or as a direct paraphrase, in the +"Document searches" section of ``docs/usage.md``. Each case indexes real +documents and asserts on matched document IDs rather than on the parsed +query, because a query that parses cleanly is not necessarily a query that +means what the documentation says it means: ``added:now`` parses without a +single diagnostic and then matches nothing, because it resolves to an +instant rather than to a span. + +The negative cases matter as much as the positive ones. They pin the +behaviours the docs explicitly warn about, so that if any of them ever +starts working the warning can be removed deliberately rather than being +left standing as a lie. +""" + +from __future__ import annotations + +from datetime import UTC +from datetime import datetime +from typing import TYPE_CHECKING + +import pytest +import time_machine + +from documents.models import Document +from documents.models import Note +from documents.models import Tag +from documents.search._errors import InvalidDateQuery + +if TYPE_CHECKING: + from collections.abc import Generator + + from django.contrib.auth.models import User + + from documents.search._backend import TantivyBackend + +pytestmark = [pytest.mark.search, pytest.mark.django_db] + +# A Monday, so that "next monday"/"last monday" land a clean week either side. +FROZEN_NOW = datetime(2026, 6, 15, 12, 0, tzinfo=UTC) + +# The checksum used in the docs' `checksum:` example. +DOC_CHECKSUM = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08" + + +def _matched_ids(backend: TantivyBackend, query: str) -> set[int]: + return set(backend.search_ids(query, user=None)) + + +def _index(backend: TantivyBackend, **kwargs: object) -> Document: + doc = Document.objects.create(**kwargs) + backend.add_or_update(doc) + return doc + + +class TestLogicalExpressions: + @pytest.fixture + def docs(self, backend: TantivyBackend) -> dict[str, int]: + return { + "secret": _index( + backend, + title="Invoice one", + content="invoice secret contents", + checksum="doc-syntax-secret", + ).pk, + "plain": _index( + backend, + title="Invoice two", + content="invoice ordinary contents", + checksum="doc-syntax-plain", + ).pk, + } + + def test_not_excludes_a_term( + self, + backend: TantivyBackend, + docs: dict[str, int], + ) -> None: + assert _matched_ids(backend, "invoice NOT secret") == {docs["plain"]} + + def test_leading_hyphen_requires_the_term_instead_of_excluding_it( + self, + backend: TantivyBackend, + docs: dict[str, int], + ) -> None: + # The docs warn about exactly this: separators are stripped at index + # time, so "-secret" is the term "secret" and the query is an AND. + assert _matched_ids(backend, "invoice -secret") == {docs["secret"]} + + def test_or_inside_parentheses_matches_either_branch( + self, + backend: TantivyBackend, + docs: dict[str, int], + ) -> None: + matched = _matched_ids(backend, "invoice AND (secret OR ordinary)") + assert matched == {docs["secret"], docs["plain"]} + + +class TestPhraseSearch: + def test_quoted_phrase_requires_the_words_in_order( + self, + backend: TantivyBackend, + ) -> None: + doc = _index( + backend, + title="Phrase", + content="the quick brown fox jumps", + checksum="doc-syntax-phrase", + ) + assert _matched_ids(backend, '"quick brown fox"') == {doc.pk} + assert _matched_ids(backend, '"brown quick fox"') == set() + + +class TestTagCommaList: + """``tag:bills,unpaid`` is published syntax (docs/usage.md), so this checks + that the documented spelling still returns what the docs promise: only the + document carrying every listed tag. + + It is deliberately not proof of paperless's field configuration, and must + not be read as such. Removing ``comma_values`` from the ``tag`` FieldSpec + leaves this test passing, because paperless's analyzer splits the literal + value "bills,unpaid" into the same two tokens the value-list reading + produces, so the two readings select the same documents. The registry fact + -- that ``tag`` opts in and no other field does -- is observable only at + the registry, and is owned by test_registry.py's + ``test_tag_is_comma_values``/``test_correspondent_is_not_comma_values``. + """ + + def test_comma_list_requires_every_listed_tag( + self, + backend: TantivyBackend, + ) -> None: + bills = Tag.objects.create(name="bills") + unpaid = Tag.objects.create(name="unpaid") + archived = Tag.objects.create(name="archived") + + both = Document.objects.create( + title="Both tags", + content="body", + checksum="doc-syntax-tag-both", + ) + both.tags.add(bills, unpaid) + backend.add_or_update(both) + + one = Document.objects.create( + title="One tag", + content="body", + checksum="doc-syntax-tag-one", + ) + one.tags.add(bills, archived) + backend.add_or_update(one) + + assert _matched_ids(backend, "tag:bills,unpaid") == {both.pk} + assert _matched_ids(backend, "tag:bills") == {both.pk, one.pk} + + +class TestArchiveMetadataFields: + @pytest.fixture + def doc(self, backend: TantivyBackend, admin_user: User) -> Document: + doc = Document.objects.create( + title="Metadata", + content="body", + checksum=DOC_CHECKSUM, + archive_serial_number=100, + page_count=12, + original_filename="invoice.pdf", + ) + Note.objects.create(document=doc, user=admin_user, note="a note") + backend.add_or_update(doc) + return doc + + @pytest.mark.parametrize( + "query", + [ + "asn:100", + "asn:[50 to 150]", + "page_count:12", + "page_count:[10 to 20]", + "num_notes:1", + "num_notes:[1 to 5]", + "original_filename:invoice.pdf", + f"checksum:{DOC_CHECKSUM}", + "checksum:9f86d081*", + # A checksum term is stored verbatim, but a checksum *pattern* is + # lowercased before it is matched, which the docs now say outright + # next to the "only a complete, lowercase checksum matches" rule + # that the uppercase term in the negative list below pins. + "checksum:9F86D081*", + ], + ) + def test_documented_metadata_query_matches( + self, + backend: TantivyBackend, + doc: Document, + query: str, + ) -> None: + assert _matched_ids(backend, query) == {doc.pk} + + @pytest.mark.parametrize( + "query", + [ + # The docs say only a complete, lowercase checksum matches. + "checksum:9f86d081", + f"checksum:{DOC_CHECKSUM.upper()}", + ], + ) + def test_partial_or_uppercase_checksum_matches_nothing( + self, + backend: TantivyBackend, + doc: Document, + query: str, + ) -> None: + assert _matched_ids(backend, query) == set() + + +class TestDocumentedDateForms: + @pytest.fixture(autouse=True) + def frozen_now(self) -> Generator[None, None, None]: + with time_machine.travel(FROZEN_NOW, tick=False): + yield + + @pytest.fixture + def dated(self, backend: TantivyBackend) -> dict[str, int]: + stamps = { + "today": datetime(2026, 6, 15, 9, 0, tzinfo=UTC), + "yesterday": datetime(2026, 6, 14, 9, 0, tzinfo=UTC), + "tomorrow": datetime(2026, 6, 16, 9, 0, tzinfo=UTC), + "next_monday": datetime(2026, 6, 22, 10, 0, tzinfo=UTC), + "last_monday": datetime(2026, 6, 8, 10, 0, tzinfo=UTC), + "january": datetime(2026, 1, 10, 10, 0, tzinfo=UTC), + "old": datetime(2005, 3, 4, 15, 30, tzinfo=UTC), + } + return { + label: _index( + backend, + title=label, + content="dated body", + checksum=f"doc-syntax-date-{label}", + added=stamp, + ).pk + for label, stamp in stamps.items() + } + + @pytest.mark.parametrize( + ("query", "label"), + [ + ("added:today", "today"), + ("added:yesterday", "yesterday"), + ("added:tomorrow", "tomorrow"), + ('added:"next monday"', "next_monday"), + ('added:"last monday"', "last_monday"), + ("added:january", "january"), + ("added:2005-03-04", "old"), + ("added:2005-03", "old"), + ("added:[2005-01-01 to 2005-12-31]", "old"), + ("added:[2005 to 2009]", "old"), + # A full timestamp works, but only quoted when it stands alone, + # and only unquoted when it is a range bound. The bare standalone + # spelling is pinned as a non-match below. + ('added:"2005-03-04T15:30:00Z"', "old"), + ("added:[2005-03-04T09:00:00Z to 2005-03-04T17:00:00Z]", "old"), + # A quoted range bound works when the quotes are single ones; the + # double-quoted spelling is pinned as an error below. + ("added:['2005-03-04' to 2005-03-05]", "old"), + ], + ) + def test_documented_date_form_matches_its_day_or_month( + self, + backend: TantivyBackend, + dated: dict[str, int], + query: str, + label: str, + ) -> None: + assert _matched_ids(backend, query) == {dated[label]} + + @pytest.mark.parametrize( + "query", + [ + # Zero-width: these resolve to a single instant, not a span, so + # nothing in a realistic corpus lands on them. The docs warn + # about them rather than presenting them as usable. + "added:now", + "added:noon", + "added:midnight", + # Quoting is what rescues the other multi-word date expressions, + # so pin that it does not rescue these: the problem is the width + # of the resulting range, not the way the value is delimited. + # One quoted spelling is enough for that; which keyword sits + # inside the quotes is grammar whoosh-compat owns. + 'added:"now"', + # A relative offset, which the warning in the docs names by this + # exact spelling. Standing alone it is an instant like the rest of + # this list; the same offset used as a range bound is a real + # window, pinned by the test below. + 'added:"-1 week"', + ], + ) + def test_forms_the_docs_warn_about_match_nothing( + self, + backend: TantivyBackend, + dated: dict[str, int], + query: str, + ) -> None: + assert _matched_ids(backend, query) == set() + + def test_bare_timestamp_is_rejected_rather_than_matching_nothing( + self, + backend: TantivyBackend, + dated: dict[str, int], + ) -> None: + """The bare, unquoted spelling of a full timestamp. The quoted and + range-bound spellings pinned above do work and match this fixture's + document; this one is a user-fixable error rather than an empty + result set, so the docs tell the user to quote it. + + The reported value is the whole contiguous fragment the user typed, + not just the prefix the date grammar's tokenizer first split on. + """ + with pytest.raises(InvalidDateQuery) as exc_info: + _matched_ids(backend, "added:2005-03-04T15:30:00Z") + assert exc_info.value.field == "added" + assert exc_info.value.value == "2005-03-04T15:30:00Z" + + def test_relative_offset_as_a_range_bound_is_a_real_window( + self, + backend: TantivyBackend, + dated: dict[str, int], + ) -> None: + """The same offset that matches nothing on its own spans the last + seven days as a lower bound. The docs say so, next to the warning + about the standalone form, so both readings are pinned together. + + "last_monday" is indexed at 2026-06-08T10:00, two hours before the + window opens, so its exclusion is what shows the bound is the offset + and not a whole-day rounding of it. + """ + assert _matched_ids(backend, "added:['-1 week' to now]") == { + dated["today"], + dated["yesterday"], + } + + def test_double_quoted_range_bound_is_rejected( + self, + backend: TantivyBackend, + dated: dict[str, int], + ) -> None: + """Quoting a range bound is allowed, but only with single quotes: the + double-quoted spelling reaches the date grammar with its quotes still + attached and is not a recognizable date. The docs say so, so pin which + of the two quote characters is the one that fails. + """ + with pytest.raises(InvalidDateQuery) as exc_info: + _matched_ids(backend, 'added:["2005-03-04" to 2005-03-05]') + assert exc_info.value.value == '"2005-03-04"' diff --git a/src/documents/tests/search/test_pattern_stemming.py b/src/documents/tests/search/test_pattern_stemming.py new file mode 100644 index 000000000..4c04d4179 --- /dev/null +++ b/src/documents/tests/search/test_pattern_stemming.py @@ -0,0 +1,233 @@ +"""Wildcard patterns must match a stemmed index. + +Query patterns are normalized but were not stemmed, while index terms are +stemmed, so the natural spelling of a prefix search matched nothing: +``invoice*`` found no document although ``invoic*`` did. v2's index was +UNSTEMMED (whoosh ``TEXT()`` defaults to ``StandardAnalyzer``), so this +regressed against both baselines. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from documents.models import Document +from documents.search._registry import _make_pattern_normalizer +from documents.search._tokenizer import ascii_fold +from documents.search._tokenizer import paperless_text_analyzer +from documents.search._tokenizer import stem_pattern_text + +if TYPE_CHECKING: + from whoosh_compat import PatternNormalizer + + from documents.search._backend import TantivyBackend + +pytestmark = [pytest.mark.search, pytest.mark.django_db] + +CONTENT = ( + "invoice total due for electricity from both companies, " + "payments made to the university library, copies attached" +) + + +def _matched_ids(backend: TantivyBackend, query: str) -> set[int]: + return set(backend.search_ids(query, user=None)) + + +@pytest.fixture +def indexed_doc(backend: TantivyBackend) -> Document: + doc = Document.objects.create( + title="Invoice 2020 productname", + content=CONTENT, + checksum="pattern-stemming-1", + archive_serial_number=900, + ) + backend.add_or_update(doc) + return doc + + +class TestPrefixStemming: + @pytest.mark.parametrize( + "query", + [ + "invoice*", + "electricity*", + "companies*", + "payments*", + "library*", + "title:Invoice*", + ], + ) + def test_full_word_prefix_matches_its_stem( + self, + backend: TantivyBackend, + indexed_doc: Document, + query: str, + ) -> None: + assert _matched_ids(backend, query) == {indexed_doc.id} + + @pytest.mark.parametrize("query", ["invoic*", "electr*", "payment*"]) + def test_already_stemmed_prefix_still_matches( + self, + backend: TantivyBackend, + indexed_doc: Document, + query: str, + ) -> None: + assert _matched_ids(backend, query) == {indexed_doc.id} + + @pytest.mark.parametrize("query", ["univers*", "librar*"]) + def test_partial_prefix_reaches_the_stemmed_term( + self, + backend: TantivyBackend, + indexed_doc: Document, + query: str, + ) -> None: + """A prefix shorter than a whole word still matches, and neither of + these needs the two-alternative path to do it. + + Measured under "en": the stemmer leaves "librar" alone, so it has one + form, and that form is a prefix of the "librari" the index holds for + "library". "univers" stems to the *shorter* "univ", and the run as + typed and its stem are both prefixes of the "univers" the index holds + for "university". The case where the two forms genuinely diverge, and + only one of them matches, is + test_stem_substitution_reaches_both_the_inflection_and_the_compound. + """ + assert _matched_ids(backend, query) == {indexed_doc.id} + + def test_full_word_reaches_the_stem_but_a_fragment_of_it_does_not( + self, + backend: TantivyBackend, + indexed_doc: Document, + ) -> None: + """The alternatives widen recall without turning a wildcard into a + prefix search over the original text. + + "university" is stored as "univers". The stem of "universities" is + that same "univers", so the longer word matches; "universit" is a + prefix of neither its own stem nor the stored term, so the *shorter* + fragment matches nothing. usage.md names this pair, so a reader told + that `universit*` fails is also told which spelling works. + """ + assert _matched_ids(backend, "universities*") == {indexed_doc.id} + assert _matched_ids(backend, "universit*") == set() + + def test_pattern_past_the_stem_boundary_is_documented_not_fixed( + self, + backend: TantivyBackend, + indexed_doc: Document, + ) -> None: + """produ*name cannot match a stemmed index ("productname" is indexed as + "productnam"); usage.md must not advertise it. Pinned so the limitation + is deliberate, not accidental.""" + assert _matched_ids(backend, "produ*name") == set() + + def test_stem_substitution_reaches_both_the_inflection_and_the_compound( + self, + backend: TantivyBackend, + indexed_doc: Document, + ) -> None: + """English stemming substitutes as well as truncates: "copy" and + "copies" both index as "copi", while "copyright" keeps its literal "y". + Neither form is a prefix of the other, so no single normalized string + reaches both. The run is therefore emitted as a disjunction of the + folded and stemmed forms, and "copy*" reaches the base word, its + inflections and the compound alike. + """ + compound = Document.objects.create( + title="Copyright notice", + content="copyright notice for the work", + checksum="pattern-stemming-2", + archive_serial_number=901, + ) + backend.add_or_update(compound) + + assert _matched_ids(backend, "copy*") == {indexed_doc.id, compound.id} + assert _matched_ids(backend, "copyright*") == {compound.id} + + +class TestStemsMatchTheIndexAnalyzer: + """stem_pattern_text rebuilds paperless_text_analyzer's stemming tail rather + than sharing it, so a filter added to the index analyzer alone would silently + stop patterns from reaching the terms it produces. + """ + + @pytest.mark.parametrize( + "language", + ["en", "de", "fr", "es", "sv", None, "klingon"], + ) + @pytest.mark.parametrize( + "word", + ["Copies", "copyright", "Companies", "Invoices", "laufen", "casas", "Straße"], + ) + def test_stem_equals_the_index_term(self, word: str, language: str | None) -> None: + indexed = paperless_text_analyzer(language).analyze(word)[0] + assert stem_pattern_text(ascii_fold(word.lower()), language) == indexed + + +def _forms(normalize: PatternNormalizer, text: str) -> tuple[str, ...]: + """The distinct forms a term may match, in order, the way the emitter reads + the normalizer's answer (see whoosh_compat.PatternNormalizer).""" + result = normalize(text) + if isinstance(result, str): + return (result,) + return tuple(dict.fromkeys(result)) + + +class TestPatternNormalizer: + @pytest.mark.parametrize( + ("text", "expected"), + [ + ("Invoice", ("invoice", "invoic")), + ("companies", ("companies", "compani")), + # y -> i is a substitution, so both forms are needed: the index + # holds "librari" for "library" and "library" for "librarian". + ("library", ("library", "librari")), + # A run the stemmer leaves alone collapses back to one form, so it + # costs exactly the one regex branch it did before. + ("invoic", ("invoic",)), + ("Universit", ("universit",)), + ("Café", ("cafe",)), + ], + ) + def test_offers_the_typed_run_and_its_stem( + self, + text: str, + expected: tuple[str, ...], + ) -> None: + assert _forms(_make_pattern_normalizer("en"), text) == expected + + def test_run_that_yields_no_token_falls_back_to_the_typed_run(self) -> None: + """A run past the remove_long limit analyzes to zero tokens, so there is + no stem to offer and only the folded run remains.""" + over_long = "invoices" * 20 + assert _forms(_make_pattern_normalizer("en"), over_long) == (over_long,) + + @pytest.mark.parametrize("language", [None, "klingon"]) + def test_unstemmed_language_folds_only(self, language: str | None) -> None: + """With no stemmer configured, or one this build has no stemmer for, the + index holds surface forms and the pattern must keep them too.""" + assert _forms(_make_pattern_normalizer(language), "Invoices") == ("invoices",) + + @pytest.mark.parametrize("char", ["a", "Z", "é"]) + def test_a_single_character_collapses_to_one_folded_form(self, char: str) -> None: + """A bracket class body is normalized one character at a time and the + answer is used only when it is a single one-character form, so a + stemmer that changed a lone character would silently disable folding + inside classes.""" + forms = _forms(_make_pattern_normalizer("en"), char) + assert len(forms) == 1 + assert len(forms[0]) == 1 + + +class TestBracketClassStillFolds: + def test_class_body_matches_case_insensitively( + self, + backend: TantivyBackend, + indexed_doc: Document, + ) -> None: + """The class body is folded per character, which the alternatives + contract preserves only because a lone character stems to itself.""" + assert _matched_ids(backend, "title:[IP]nvoice*") == {indexed_doc.id}