feat(search): let a wildcard match the typed run or its stem

src/documents/tests/search/test_compact_date_forms.py

@@ -0,0 +1,75 @@
+"""Whoosh's compact, separator-free date spelling, resolved end to end.
+
+whoosh-compat owns both widths of this spelling and asserts both of each
+form's bounds directly: ``test_compact_numeric_datetime`` pins the 8-digit
+form as a whole calendar day (lower bound, upper bound and exclusivity), and
+``test_compact_numeric_datetime_full_width_is_a_single_second_instant`` pins
+the 14-digit form as one instant. The 14-digit form is kept here as the single
+representative because it is the one that exercises paperless's ``added``
+DATETIME fast field at full precision: the corpus separates a document at
+the named instant from one on the same calendar day at another hour and one
+on the next day at the same hour, so a query that degrades into a whole-day
+window, or drops the time of day, matches the wrong set rather than passing
+on a corpus that could not tell the difference.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC
+from datetime import datetime
+from typing import TYPE_CHECKING
+
+import pytest
+
+from documents.models import Document
+
+if TYPE_CHECKING:
+    from documents.search._backend import TantivyBackend
+
+pytestmark = [pytest.mark.search, pytest.mark.django_db]
+
+
+def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
+    return set(backend.search_ids(query, user=None))
+
+
+def _index(backend: TantivyBackend, **kwargs: object) -> Document:
+    doc = Document.objects.create(**kwargs)
+    backend.add_or_update(doc)
+    return doc
+
+
+@pytest.fixture
+def docs(backend: TantivyBackend) -> dict[str, int]:
+    return {
+        "instant": _index(
+            backend,
+            title="On the instant",
+            content="x",
+            checksum="compact-date-instant",
+            added=datetime(2005, 3, 4, 15, 30, tzinfo=UTC),
+        ).pk,
+        "same_day": _index(
+            backend,
+            title="Same day, other hour",
+            content="x",
+            checksum="compact-date-same-day",
+            added=datetime(2005, 3, 4, 9, 0, tzinfo=UTC),
+        ).pk,
+        "next_day": _index(
+            backend,
+            title="Next day, same hour",
+            content="x",
+            checksum="compact-date-next-day",
+            added=datetime(2005, 3, 5, 15, 30, tzinfo=UTC),
+        ).pk,
+    }
+
+
+def test_fourteen_digits_is_a_single_instant(
+    backend: TantivyBackend,
+    docs: dict[str, int],
+) -> None:
+    # same_day is what tells this apart from the 8-digit day-window form,
+    # next_day from a form that ignored the time altogether.
+    assert _matched_ids(backend, "added:20050304153000") == {docs["instant"]}

src/documents/tests/search/test_documented_syntax.py

@@ -0,0 +1,355 @@
+"""Pins the search syntax that ``docs/usage.md`` promises users.
+
+Every query here appears verbatim, or as a direct paraphrase, in the
+"Document searches" section of ``docs/usage.md``. Each case indexes real
+documents and asserts on matched document IDs rather than on the parsed
+query, because a query that parses cleanly is not necessarily a query that
+means what the documentation says it means: ``added:now`` parses without a
+single diagnostic and then matches nothing, because it resolves to an
+instant rather than to a span.
+
+The negative cases matter as much as the positive ones. They pin the
+behaviours the docs explicitly warn about, so that if any of them ever
+starts working the warning can be removed deliberately rather than being
+left standing as a lie.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC
+from datetime import datetime
+from typing import TYPE_CHECKING
+
+import pytest
+import time_machine
+
+from documents.models import Document
+from documents.models import Note
+from documents.models import Tag
+from documents.search._errors import InvalidDateQuery
+
+if TYPE_CHECKING:
+    from collections.abc import Generator
+
+    from django.contrib.auth.models import User
+
+    from documents.search._backend import TantivyBackend
+
+pytestmark = [pytest.mark.search, pytest.mark.django_db]
+
+# A Monday, so that "next monday"/"last monday" land a clean week either side.
+FROZEN_NOW = datetime(2026, 6, 15, 12, 0, tzinfo=UTC)
+
+# The checksum used in the docs' `checksum:` example.
+DOC_CHECKSUM = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"
+
+
+def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
+    return set(backend.search_ids(query, user=None))
+
+
+def _index(backend: TantivyBackend, **kwargs: object) -> Document:
+    doc = Document.objects.create(**kwargs)
+    backend.add_or_update(doc)
+    return doc
+
+
+class TestLogicalExpressions:
+    @pytest.fixture
+    def docs(self, backend: TantivyBackend) -> dict[str, int]:
+        return {
+            "secret": _index(
+                backend,
+                title="Invoice one",
+                content="invoice secret contents",
+                checksum="doc-syntax-secret",
+            ).pk,
+            "plain": _index(
+                backend,
+                title="Invoice two",
+                content="invoice ordinary contents",
+                checksum="doc-syntax-plain",
+            ).pk,
+        }
+
+    def test_not_excludes_a_term(
+        self,
+        backend: TantivyBackend,
+        docs: dict[str, int],
+    ) -> None:
+        assert _matched_ids(backend, "invoice NOT secret") == {docs["plain"]}
+
+    def test_leading_hyphen_requires_the_term_instead_of_excluding_it(
+        self,
+        backend: TantivyBackend,
+        docs: dict[str, int],
+    ) -> None:
+        # The docs warn about exactly this: separators are stripped at index
+        # time, so "-secret" is the term "secret" and the query is an AND.
+        assert _matched_ids(backend, "invoice -secret") == {docs["secret"]}
+
+    def test_or_inside_parentheses_matches_either_branch(
+        self,
+        backend: TantivyBackend,
+        docs: dict[str, int],
+    ) -> None:
+        matched = _matched_ids(backend, "invoice AND (secret OR ordinary)")
+        assert matched == {docs["secret"], docs["plain"]}
+
+
+class TestPhraseSearch:
+    def test_quoted_phrase_requires_the_words_in_order(
+        self,
+        backend: TantivyBackend,
+    ) -> None:
+        doc = _index(
+            backend,
+            title="Phrase",
+            content="the quick brown fox jumps",
+            checksum="doc-syntax-phrase",
+        )
+        assert _matched_ids(backend, '"quick brown fox"') == {doc.pk}
+        assert _matched_ids(backend, '"brown quick fox"') == set()
+
+
+class TestTagCommaList:
+    """``tag:bills,unpaid`` is published syntax (docs/usage.md), so this checks
+    that the documented spelling still returns what the docs promise: only the
+    document carrying every listed tag.
+
+    It is deliberately not proof of paperless's field configuration, and must
+    not be read as such. Removing ``comma_values`` from the ``tag`` FieldSpec
+    leaves this test passing, because paperless's analyzer splits the literal
+    value "bills,unpaid" into the same two tokens the value-list reading
+    produces, so the two readings select the same documents. The registry fact
+    -- that ``tag`` opts in and no other field does -- is observable only at
+    the registry, and is owned by test_registry.py's
+    ``test_tag_is_comma_values``/``test_correspondent_is_not_comma_values``.
+    """
+
+    def test_comma_list_requires_every_listed_tag(
+        self,
+        backend: TantivyBackend,
+    ) -> None:
+        bills = Tag.objects.create(name="bills")
+        unpaid = Tag.objects.create(name="unpaid")
+        archived = Tag.objects.create(name="archived")
+
+        both = Document.objects.create(
+            title="Both tags",
+            content="body",
+            checksum="doc-syntax-tag-both",
+        )
+        both.tags.add(bills, unpaid)
+        backend.add_or_update(both)
+
+        one = Document.objects.create(
+            title="One tag",
+            content="body",
+            checksum="doc-syntax-tag-one",
+        )
+        one.tags.add(bills, archived)
+        backend.add_or_update(one)
+
+        assert _matched_ids(backend, "tag:bills,unpaid") == {both.pk}
+        assert _matched_ids(backend, "tag:bills") == {both.pk, one.pk}
+
+
+class TestArchiveMetadataFields:
+    @pytest.fixture
+    def doc(self, backend: TantivyBackend, admin_user: User) -> Document:
+        doc = Document.objects.create(
+            title="Metadata",
+            content="body",
+            checksum=DOC_CHECKSUM,
+            archive_serial_number=100,
+            page_count=12,
+            original_filename="invoice.pdf",
+        )
+        Note.objects.create(document=doc, user=admin_user, note="a note")
+        backend.add_or_update(doc)
+        return doc
+
+    @pytest.mark.parametrize(
+        "query",
+        [
+            "asn:100",
+            "asn:[50 to 150]",
+            "page_count:12",
+            "page_count:[10 to 20]",
+            "num_notes:1",
+            "num_notes:[1 to 5]",
+            "original_filename:invoice.pdf",
+            f"checksum:{DOC_CHECKSUM}",
+            "checksum:9f86d081*",
+            # A checksum term is stored verbatim, but a checksum *pattern* is
+            # lowercased before it is matched, which the docs now say outright
+            # next to the "only a complete, lowercase checksum matches" rule
+            # that the uppercase term in the negative list below pins.
+            "checksum:9F86D081*",
+        ],
+    )
+    def test_documented_metadata_query_matches(
+        self,
+        backend: TantivyBackend,
+        doc: Document,
+        query: str,
+    ) -> None:
+        assert _matched_ids(backend, query) == {doc.pk}
+
+    @pytest.mark.parametrize(
+        "query",
+        [
+            # The docs say only a complete, lowercase checksum matches.
+            "checksum:9f86d081",
+            f"checksum:{DOC_CHECKSUM.upper()}",
+        ],
+    )
+    def test_partial_or_uppercase_checksum_matches_nothing(
+        self,
+        backend: TantivyBackend,
+        doc: Document,
+        query: str,
+    ) -> None:
+        assert _matched_ids(backend, query) == set()
+
+
+class TestDocumentedDateForms:
+    @pytest.fixture(autouse=True)
+    def frozen_now(self) -> Generator[None, None, None]:
+        with time_machine.travel(FROZEN_NOW, tick=False):
+            yield
+
+    @pytest.fixture
+    def dated(self, backend: TantivyBackend) -> dict[str, int]:
+        stamps = {
+            "today": datetime(2026, 6, 15, 9, 0, tzinfo=UTC),
+            "yesterday": datetime(2026, 6, 14, 9, 0, tzinfo=UTC),
+            "tomorrow": datetime(2026, 6, 16, 9, 0, tzinfo=UTC),
+            "next_monday": datetime(2026, 6, 22, 10, 0, tzinfo=UTC),
+            "last_monday": datetime(2026, 6, 8, 10, 0, tzinfo=UTC),
+            "january": datetime(2026, 1, 10, 10, 0, tzinfo=UTC),
+            "old": datetime(2005, 3, 4, 15, 30, tzinfo=UTC),
+        }
+        return {
+            label: _index(
+                backend,
+                title=label,
+                content="dated body",
+                checksum=f"doc-syntax-date-{label}",
+                added=stamp,
+            ).pk
+            for label, stamp in stamps.items()
+        }
+
+    @pytest.mark.parametrize(
+        ("query", "label"),
+        [
+            ("added:today", "today"),
+            ("added:yesterday", "yesterday"),
+            ("added:tomorrow", "tomorrow"),
+            ('added:"next monday"', "next_monday"),
+            ('added:"last monday"', "last_monday"),
+            ("added:january", "january"),
+            ("added:2005-03-04", "old"),
+            ("added:2005-03", "old"),
+            ("added:[2005-01-01 to 2005-12-31]", "old"),
+            ("added:[2005 to 2009]", "old"),
+            # A full timestamp works, but only quoted when it stands alone,
+            # and only unquoted when it is a range bound. The bare standalone
+            # spelling is pinned as a non-match below.
+            ('added:"2005-03-04T15:30:00Z"', "old"),
+            ("added:[2005-03-04T09:00:00Z to 2005-03-04T17:00:00Z]", "old"),
+            # A quoted range bound works when the quotes are single ones; the
+            # double-quoted spelling is pinned as an error below.
+            ("added:['2005-03-04' to 2005-03-05]", "old"),
+        ],
+    )
+    def test_documented_date_form_matches_its_day_or_month(
+        self,
+        backend: TantivyBackend,
+        dated: dict[str, int],
+        query: str,
+        label: str,
+    ) -> None:
+        assert _matched_ids(backend, query) == {dated[label]}
+
+    @pytest.mark.parametrize(
+        "query",
+        [
+            # Zero-width: these resolve to a single instant, not a span, so
+            # nothing in a realistic corpus lands on them. The docs warn
+            # about them rather than presenting them as usable.
+            "added:now",
+            "added:noon",
+            "added:midnight",
+            # Quoting is what rescues the other multi-word date expressions,
+            # so pin that it does not rescue these: the problem is the width
+            # of the resulting range, not the way the value is delimited.
+            # One quoted spelling is enough for that; which keyword sits
+            # inside the quotes is grammar whoosh-compat owns.
+            'added:"now"',
+            # A relative offset, which the warning in the docs names by this
+            # exact spelling. Standing alone it is an instant like the rest of
+            # this list; the same offset used as a range bound is a real
+            # window, pinned by the test below.
+            'added:"-1 week"',
+        ],
+    )
+    def test_forms_the_docs_warn_about_match_nothing(
+        self,
+        backend: TantivyBackend,
+        dated: dict[str, int],
+        query: str,
+    ) -> None:
+        assert _matched_ids(backend, query) == set()
+
+    def test_bare_timestamp_is_rejected_rather_than_matching_nothing(
+        self,
+        backend: TantivyBackend,
+        dated: dict[str, int],
+    ) -> None:
+        """The bare, unquoted spelling of a full timestamp. The quoted and
+        range-bound spellings pinned above do work and match this fixture's
+        document; this one is a user-fixable error rather than an empty
+        result set, so the docs tell the user to quote it.
+
+        The reported value is the whole contiguous fragment the user typed,
+        not just the prefix the date grammar's tokenizer first split on.
+        """
+        with pytest.raises(InvalidDateQuery) as exc_info:
+            _matched_ids(backend, "added:2005-03-04T15:30:00Z")
+        assert exc_info.value.field == "added"
+        assert exc_info.value.value == "2005-03-04T15:30:00Z"
+
+    def test_relative_offset_as_a_range_bound_is_a_real_window(
+        self,
+        backend: TantivyBackend,
+        dated: dict[str, int],
+    ) -> None:
+        """The same offset that matches nothing on its own spans the last
+        seven days as a lower bound. The docs say so, next to the warning
+        about the standalone form, so both readings are pinned together.
+
+        "last_monday" is indexed at 2026-06-08T10:00, two hours before the
+        window opens, so its exclusion is what shows the bound is the offset
+        and not a whole-day rounding of it.
+        """
+        assert _matched_ids(backend, "added:['-1 week' to now]") == {
+            dated["today"],
+            dated["yesterday"],
+        }
+
+    def test_double_quoted_range_bound_is_rejected(
+        self,
+        backend: TantivyBackend,
+        dated: dict[str, int],
+    ) -> None:
+        """Quoting a range bound is allowed, but only with single quotes: the
+        double-quoted spelling reaches the date grammar with its quotes still
+        attached and is not a recognizable date. The docs say so, so pin which
+        of the two quote characters is the one that fails.
+        """
+        with pytest.raises(InvalidDateQuery) as exc_info:
+            _matched_ids(backend, 'added:["2005-03-04" to 2005-03-05]')
+        assert exc_info.value.value == '"2005-03-04"'

src/documents/tests/search/test_pattern_stemming.py

@@ -0,0 +1,233 @@
+"""Wildcard patterns must match a stemmed index.
+
+Query patterns are normalized but were not stemmed, while index terms are
+stemmed, so the natural spelling of a prefix search matched nothing:
+``invoice*`` found no document although ``invoic*`` did. v2's index was
+UNSTEMMED (whoosh ``TEXT()`` defaults to ``StandardAnalyzer``), so this
+regressed against both baselines.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+
+from documents.models import Document
+from documents.search._registry import _make_pattern_normalizer
+from documents.search._tokenizer import ascii_fold
+from documents.search._tokenizer import paperless_text_analyzer
+from documents.search._tokenizer import stem_pattern_text
+
+if TYPE_CHECKING:
+    from whoosh_compat import PatternNormalizer
+
+    from documents.search._backend import TantivyBackend
+
+pytestmark = [pytest.mark.search, pytest.mark.django_db]
+
+CONTENT = (
+    "invoice total due for electricity from both companies, "
+    "payments made to the university library, copies attached"
+)
+
+
+def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
+    return set(backend.search_ids(query, user=None))
+
+
+@pytest.fixture
+def indexed_doc(backend: TantivyBackend) -> Document:
+    doc = Document.objects.create(
+        title="Invoice 2020 productname",
+        content=CONTENT,
+        checksum="pattern-stemming-1",
+        archive_serial_number=900,
+    )
+    backend.add_or_update(doc)
+    return doc
+
+
+class TestPrefixStemming:
+    @pytest.mark.parametrize(
+        "query",
+        [
+            "invoice*",
+            "electricity*",
+            "companies*",
+            "payments*",
+            "library*",
+            "title:Invoice*",
+        ],
+    )
+    def test_full_word_prefix_matches_its_stem(
+        self,
+        backend: TantivyBackend,
+        indexed_doc: Document,
+        query: str,
+    ) -> None:
+        assert _matched_ids(backend, query) == {indexed_doc.id}
+
+    @pytest.mark.parametrize("query", ["invoic*", "electr*", "payment*"])
+    def test_already_stemmed_prefix_still_matches(
+        self,
+        backend: TantivyBackend,
+        indexed_doc: Document,
+        query: str,
+    ) -> None:
+        assert _matched_ids(backend, query) == {indexed_doc.id}
+
+    @pytest.mark.parametrize("query", ["univers*", "librar*"])
+    def test_partial_prefix_reaches_the_stemmed_term(
+        self,
+        backend: TantivyBackend,
+        indexed_doc: Document,
+        query: str,
+    ) -> None:
+        """A prefix shorter than a whole word still matches, and neither of
+        these needs the two-alternative path to do it.
+
+        Measured under "en": the stemmer leaves "librar" alone, so it has one
+        form, and that form is a prefix of the "librari" the index holds for
+        "library". "univers" stems to the *shorter* "univ", and the run as
+        typed and its stem are both prefixes of the "univers" the index holds
+        for "university". The case where the two forms genuinely diverge, and
+        only one of them matches, is
+        test_stem_substitution_reaches_both_the_inflection_and_the_compound.
+        """
+        assert _matched_ids(backend, query) == {indexed_doc.id}
+
+    def test_full_word_reaches_the_stem_but_a_fragment_of_it_does_not(
+        self,
+        backend: TantivyBackend,
+        indexed_doc: Document,
+    ) -> None:
+        """The alternatives widen recall without turning a wildcard into a
+        prefix search over the original text.
+
+        "university" is stored as "univers". The stem of "universities" is
+        that same "univers", so the longer word matches; "universit" is a
+        prefix of neither its own stem nor the stored term, so the *shorter*
+        fragment matches nothing. usage.md names this pair, so a reader told
+        that `universit*` fails is also told which spelling works.
+        """
+        assert _matched_ids(backend, "universities*") == {indexed_doc.id}
+        assert _matched_ids(backend, "universit*") == set()
+
+    def test_pattern_past_the_stem_boundary_is_documented_not_fixed(
+        self,
+        backend: TantivyBackend,
+        indexed_doc: Document,
+    ) -> None:
+        """produ*name cannot match a stemmed index ("productname" is indexed as
+        "productnam"); usage.md must not advertise it. Pinned so the limitation
+        is deliberate, not accidental."""
+        assert _matched_ids(backend, "produ*name") == set()
+
+    def test_stem_substitution_reaches_both_the_inflection_and_the_compound(
+        self,
+        backend: TantivyBackend,
+        indexed_doc: Document,
+    ) -> None:
+        """English stemming substitutes as well as truncates: "copy" and
+        "copies" both index as "copi", while "copyright" keeps its literal "y".
+        Neither form is a prefix of the other, so no single normalized string
+        reaches both. The run is therefore emitted as a disjunction of the
+        folded and stemmed forms, and "copy*" reaches the base word, its
+        inflections and the compound alike.
+        """
+        compound = Document.objects.create(
+            title="Copyright notice",
+            content="copyright notice for the work",
+            checksum="pattern-stemming-2",
+            archive_serial_number=901,
+        )
+        backend.add_or_update(compound)
+
+        assert _matched_ids(backend, "copy*") == {indexed_doc.id, compound.id}
+        assert _matched_ids(backend, "copyright*") == {compound.id}
+
+
+class TestStemsMatchTheIndexAnalyzer:
+    """stem_pattern_text rebuilds paperless_text_analyzer's stemming tail rather
+    than sharing it, so a filter added to the index analyzer alone would silently
+    stop patterns from reaching the terms it produces.
+    """
+
+    @pytest.mark.parametrize(
+        "language",
+        ["en", "de", "fr", "es", "sv", None, "klingon"],
+    )
+    @pytest.mark.parametrize(
+        "word",
+        ["Copies", "copyright", "Companies", "Invoices", "laufen", "casas", "Straße"],
+    )
+    def test_stem_equals_the_index_term(self, word: str, language: str | None) -> None:
+        indexed = paperless_text_analyzer(language).analyze(word)[0]
+        assert stem_pattern_text(ascii_fold(word.lower()), language) == indexed
+
+
+def _forms(normalize: PatternNormalizer, text: str) -> tuple[str, ...]:
+    """The distinct forms a term may match, in order, the way the emitter reads
+    the normalizer's answer (see whoosh_compat.PatternNormalizer)."""
+    result = normalize(text)
+    if isinstance(result, str):
+        return (result,)
+    return tuple(dict.fromkeys(result))
+
+
+class TestPatternNormalizer:
+    @pytest.mark.parametrize(
+        ("text", "expected"),
+        [
+            ("Invoice", ("invoice", "invoic")),
+            ("companies", ("companies", "compani")),
+            # y -> i is a substitution, so both forms are needed: the index
+            # holds "librari" for "library" and "library" for "librarian".
+            ("library", ("library", "librari")),
+            # A run the stemmer leaves alone collapses back to one form, so it
+            # costs exactly the one regex branch it did before.
+            ("invoic", ("invoic",)),
+            ("Universit", ("universit",)),
+            ("Café", ("cafe",)),
+        ],
+    )
+    def test_offers_the_typed_run_and_its_stem(
+        self,
+        text: str,
+        expected: tuple[str, ...],
+    ) -> None:
+        assert _forms(_make_pattern_normalizer("en"), text) == expected
+
+    def test_run_that_yields_no_token_falls_back_to_the_typed_run(self) -> None:
+        """A run past the remove_long limit analyzes to zero tokens, so there is
+        no stem to offer and only the folded run remains."""
+        over_long = "invoices" * 20
+        assert _forms(_make_pattern_normalizer("en"), over_long) == (over_long,)
+
+    @pytest.mark.parametrize("language", [None, "klingon"])
+    def test_unstemmed_language_folds_only(self, language: str | None) -> None:
+        """With no stemmer configured, or one this build has no stemmer for, the
+        index holds surface forms and the pattern must keep them too."""
+        assert _forms(_make_pattern_normalizer(language), "Invoices") == ("invoices",)
+
+    @pytest.mark.parametrize("char", ["a", "Z", "é"])
+    def test_a_single_character_collapses_to_one_folded_form(self, char: str) -> None:
+        """A bracket class body is normalized one character at a time and the
+        answer is used only when it is a single one-character form, so a
+        stemmer that changed a lone character would silently disable folding
+        inside classes."""
+        forms = _forms(_make_pattern_normalizer("en"), char)
+        assert len(forms) == 1
+        assert len(forms[0]) == 1
+
+
+class TestBracketClassStillFolds:
+    def test_class_body_matches_case_insensitively(
+        self,
+        backend: TantivyBackend,
+        indexed_doc: Document,
+    ) -> None:
+        """The class body is folded per character, which the alternatives
+        contract preserves only because a lone character stems to itself."""
+        assert _matched_ids(backend, "title:[IP]nvoice*") == {indexed_doc.id}