amorfo77/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2026-08-28 21:47:34 +00:00
Code Issues Projects Releases Wiki Activity

feat(search): let a wildcard match the typed run or its stem

Browse Source
This commit is contained in:
stumpylog
2026-08-28 09:48:21 -07:00
parent f1ea2007ec
commit d0239d37bd
3 changed files with 663 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/documents/tests/search/test_compact_date_forms.py
+75
View File
@@ -0,0 +1,75 @@
"""Whoosh's compact, separator-free date spelling, resolved end to end.
whoosh-compat owns both widths of this spelling and asserts both of each
form's bounds directly: ``test_compact_numeric_datetime`` pins the 8-digit
form as a whole calendar day (lower bound, upper bound and exclusivity), and
``test_compact_numeric_datetime_full_width_is_a_single_second_instant`` pins
the 14-digit form as one instant. The 14-digit form is kept here as the single
representative because it is the one that exercises paperless's ``added``
DATETIME fast field at full precision: the corpus separates a document at
the named instant from one on the same calendar day at another hour and one
on the next day at the same hour, so a query that degrades into a whole-day
window, or drops the time of day, matches the wrong set rather than passing
on a corpus that could not tell the difference.
"""
from __future__ import annotations
from datetime import UTC
from datetime import datetime
from typing import TYPE_CHECKING
import pytest
from documents.models import Document
if TYPE_CHECKING:
from documents.search._backend import TantivyBackend
pytestmark = [pytest.mark.search, pytest.mark.django_db]
def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
return set(backend.search_ids(query, user=None))
def _index(backend: TantivyBackend, **kwargs: object) -> Document:
doc = Document.objects.create(**kwargs)
backend.add_or_update(doc)
return doc
@pytest.fixture
def docs(backend: TantivyBackend) -> dict[str, int]:
return {
"instant": _index(
backend,
title="On the instant",
content="x",
checksum="compact-date-instant",
added=datetime(2005, 3, 4, 15, 30, tzinfo=UTC),
).pk,
"same_day": _index(
backend,
title="Same day, other hour",
content="x",
checksum="compact-date-same-day",
added=datetime(2005, 3, 4, 9, 0, tzinfo=UTC),
).pk,
"next_day": _index(
backend,
title="Next day, same hour",
content="x",
checksum="compact-date-next-day",
added=datetime(2005, 3, 5, 15, 30, tzinfo=UTC),
).pk,
}
def test_fourteen_digits_is_a_single_instant(
backend: TantivyBackend,
docs: dict[str, int],
) -> None:
# same_day is what tells this apart from the 8-digit day-window form,
# next_day from a form that ignored the time altogether.
assert _matched_ids(backend, "added:20050304153000") == {docs["instant"]}
src/documents/tests/search/test_documented_syntax.py
+355
View File
@@ -0,0 +1,355 @@
"""Pins the search syntax that ``docs/usage.md`` promises users.
Every query here appears verbatim, or as a direct paraphrase, in the
"Document searches" section of ``docs/usage.md``. Each case indexes real
documents and asserts on matched document IDs rather than on the parsed
query, because a query that parses cleanly is not necessarily a query that
means what the documentation says it means: ``added:now`` parses without a
single diagnostic and then matches nothing, because it resolves to an
instant rather than to a span.
The negative cases matter as much as the positive ones. They pin the
behaviours the docs explicitly warn about, so that if any of them ever
starts working the warning can be removed deliberately rather than being
left standing as a lie.
"""
from __future__ import annotations
from datetime import UTC
from datetime import datetime
from typing import TYPE_CHECKING
import pytest
import time_machine
from documents.models import Document
from documents.models import Note
from documents.models import Tag
from documents.search._errors import InvalidDateQuery
if TYPE_CHECKING:
from collections.abc import Generator
from django.contrib.auth.models import User
from documents.search._backend import TantivyBackend
pytestmark = [pytest.mark.search, pytest.mark.django_db]
# A Monday, so that "next monday"/"last monday" land a clean week either side.
FROZEN_NOW = datetime(2026, 6, 15, 12, 0, tzinfo=UTC)
# The checksum used in the docs' `checksum:` example.
DOC_CHECKSUM = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"
def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
return set(backend.search_ids(query, user=None))
def _index(backend: TantivyBackend, **kwargs: object) -> Document:
doc = Document.objects.create(**kwargs)
backend.add_or_update(doc)
return doc
class TestLogicalExpressions:
@pytest.fixture
def docs(self, backend: TantivyBackend) -> dict[str, int]:
return {
"secret": _index(
backend,
title="Invoice one",
content="invoice secret contents",
checksum="doc-syntax-secret",
).pk,
"plain": _index(
backend,
title="Invoice two",
content="invoice ordinary contents",
checksum="doc-syntax-plain",
).pk,
}
def test_not_excludes_a_term(
self,
backend: TantivyBackend,
docs: dict[str, int],
) -> None:
assert _matched_ids(backend, "invoice NOT secret") == {docs["plain"]}
def test_leading_hyphen_requires_the_term_instead_of_excluding_it(
self,
backend: TantivyBackend,
docs: dict[str, int],
) -> None:
# The docs warn about exactly this: separators are stripped at index
# time, so "-secret" is the term "secret" and the query is an AND.
assert _matched_ids(backend, "invoice -secret") == {docs["secret"]}
def test_or_inside_parentheses_matches_either_branch(
self,
backend: TantivyBackend,
docs: dict[str, int],
) -> None:
matched = _matched_ids(backend, "invoice AND (secret OR ordinary)")
assert matched == {docs["secret"], docs["plain"]}
class TestPhraseSearch:
def test_quoted_phrase_requires_the_words_in_order(
self,
backend: TantivyBackend,
) -> None:
doc = _index(
backend,
title="Phrase",
content="the quick brown fox jumps",
checksum="doc-syntax-phrase",
)
assert _matched_ids(backend, '"quick brown fox"') == {doc.pk}
assert _matched_ids(backend, '"brown quick fox"') == set()
class TestTagCommaList:
"""``tag:bills,unpaid`` is published syntax (docs/usage.md), so this checks
that the documented spelling still returns what the docs promise: only the
document carrying every listed tag.
It is deliberately not proof of paperless's field configuration, and must
not be read as such. Removing ``comma_values`` from the ``tag`` FieldSpec
leaves this test passing, because paperless's analyzer splits the literal
value "bills,unpaid" into the same two tokens the value-list reading
produces, so the two readings select the same documents. The registry fact
-- that ``tag`` opts in and no other field does -- is observable only at
the registry, and is owned by test_registry.py's
``test_tag_is_comma_values``/``test_correspondent_is_not_comma_values``.
"""
def test_comma_list_requires_every_listed_tag(
self,
backend: TantivyBackend,
) -> None:
bills = Tag.objects.create(name="bills")
unpaid = Tag.objects.create(name="unpaid")
archived = Tag.objects.create(name="archived")
both = Document.objects.create(
title="Both tags",
content="body",
checksum="doc-syntax-tag-both",
)
both.tags.add(bills, unpaid)
backend.add_or_update(both)
one = Document.objects.create(
title="One tag",
content="body",
checksum="doc-syntax-tag-one",
)
one.tags.add(bills, archived)
backend.add_or_update(one)
assert _matched_ids(backend, "tag:bills,unpaid") == {both.pk}
assert _matched_ids(backend, "tag:bills") == {both.pk, one.pk}
class TestArchiveMetadataFields:
@pytest.fixture
def doc(self, backend: TantivyBackend, admin_user: User) -> Document:
doc = Document.objects.create(
title="Metadata",
content="body",
checksum=DOC_CHECKSUM,
archive_serial_number=100,
page_count=12,
original_filename="invoice.pdf",
)
Note.objects.create(document=doc, user=admin_user, note="a note")
backend.add_or_update(doc)
return doc
@pytest.mark.parametrize(
"query",
[
"asn:100",
"asn:[50 to 150]",
"page_count:12",
"page_count:[10 to 20]",
"num_notes:1",
"num_notes:[1 to 5]",
"original_filename:invoice.pdf",
f"checksum:{DOC_CHECKSUM}",
"checksum:9f86d081*",
# A checksum term is stored verbatim, but a checksum *pattern* is
# lowercased before it is matched, which the docs now say outright
# next to the "only a complete, lowercase checksum matches" rule
# that the uppercase term in the negative list below pins.
"checksum:9F86D081*",
],
)
def test_documented_metadata_query_matches(
self,
backend: TantivyBackend,
doc: Document,
query: str,
) -> None:
assert _matched_ids(backend, query) == {doc.pk}
@pytest.mark.parametrize(
"query",
[
# The docs say only a complete, lowercase checksum matches.
"checksum:9f86d081",
f"checksum:{DOC_CHECKSUM.upper()}",
],
)
def test_partial_or_uppercase_checksum_matches_nothing(
self,
backend: TantivyBackend,
doc: Document,
query: str,
) -> None:
assert _matched_ids(backend, query) == set()
class TestDocumentedDateForms:
@pytest.fixture(autouse=True)
def frozen_now(self) -> Generator[None, None, None]:
with time_machine.travel(FROZEN_NOW, tick=False):
yield
@pytest.fixture
def dated(self, backend: TantivyBackend) -> dict[str, int]:
stamps = {
"today": datetime(2026, 6, 15, 9, 0, tzinfo=UTC),
"yesterday": datetime(2026, 6, 14, 9, 0, tzinfo=UTC),
"tomorrow": datetime(2026, 6, 16, 9, 0, tzinfo=UTC),
"next_monday": datetime(2026, 6, 22, 10, 0, tzinfo=UTC),
"last_monday": datetime(2026, 6, 8, 10, 0, tzinfo=UTC),
"january": datetime(2026, 1, 10, 10, 0, tzinfo=UTC),
"old": datetime(2005, 3, 4, 15, 30, tzinfo=UTC),
}
return {
label: _index(
backend,
title=label,
content="dated body",
checksum=f"doc-syntax-date-{label}",
added=stamp,
).pk
for label, stamp in stamps.items()
}
@pytest.mark.parametrize(
("query", "label"),
[
("added:today", "today"),
("added:yesterday", "yesterday"),
("added:tomorrow", "tomorrow"),
('added:"next monday"', "next_monday"),
('added:"last monday"', "last_monday"),
("added:january", "january"),
("added:2005-03-04", "old"),
("added:2005-03", "old"),
("added:[2005-01-01 to 2005-12-31]", "old"),
("added:[2005 to 2009]", "old"),
# A full timestamp works, but only quoted when it stands alone,
# and only unquoted when it is a range bound. The bare standalone
# spelling is pinned as a non-match below.
('added:"2005-03-04T15:30:00Z"', "old"),
("added:[2005-03-04T09:00:00Z to 2005-03-04T17:00:00Z]", "old"),
# A quoted range bound works when the quotes are single ones; the
# double-quoted spelling is pinned as an error below.
("added:['2005-03-04' to 2005-03-05]", "old"),
],
)
def test_documented_date_form_matches_its_day_or_month(
self,
backend: TantivyBackend,
dated: dict[str, int],
query: str,
label: str,
) -> None:
assert _matched_ids(backend, query) == {dated[label]}
@pytest.mark.parametrize(
"query",
[
# Zero-width: these resolve to a single instant, not a span, so
# nothing in a realistic corpus lands on them. The docs warn
# about them rather than presenting them as usable.
"added:now",
"added:noon",
"added:midnight",
# Quoting is what rescues the other multi-word date expressions,
# so pin that it does not rescue these: the problem is the width
# of the resulting range, not the way the value is delimited.
# One quoted spelling is enough for that; which keyword sits
# inside the quotes is grammar whoosh-compat owns.
'added:"now"',
# A relative offset, which the warning in the docs names by this
# exact spelling. Standing alone it is an instant like the rest of
# this list; the same offset used as a range bound is a real
# window, pinned by the test below.
'added:"-1 week"',
],
)
def test_forms_the_docs_warn_about_match_nothing(
self,
backend: TantivyBackend,
dated: dict[str, int],
query: str,
) -> None:
assert _matched_ids(backend, query) == set()
def test_bare_timestamp_is_rejected_rather_than_matching_nothing(
self,
backend: TantivyBackend,
dated: dict[str, int],
) -> None:
"""The bare, unquoted spelling of a full timestamp. The quoted and
range-bound spellings pinned above do work and match this fixture's
document; this one is a user-fixable error rather than an empty
result set, so the docs tell the user to quote it.
The reported value is the whole contiguous fragment the user typed,
not just the prefix the date grammar's tokenizer first split on.
"""
with pytest.raises(InvalidDateQuery) as exc_info:
_matched_ids(backend, "added:2005-03-04T15:30:00Z")
assert exc_info.value.field == "added"
assert exc_info.value.value == "2005-03-04T15:30:00Z"
def test_relative_offset_as_a_range_bound_is_a_real_window(
self,
backend: TantivyBackend,
dated: dict[str, int],
) -> None:
"""The same offset that matches nothing on its own spans the last
seven days as a lower bound. The docs say so, next to the warning
about the standalone form, so both readings are pinned together.
"last_monday" is indexed at 2026-06-08T10:00, two hours before the
window opens, so its exclusion is what shows the bound is the offset
and not a whole-day rounding of it.
"""
assert _matched_ids(backend, "added:['-1 week' to now]") == {
dated["today"],
dated["yesterday"],
}
def test_double_quoted_range_bound_is_rejected(
self,
backend: TantivyBackend,
dated: dict[str, int],
) -> None:
"""Quoting a range bound is allowed, but only with single quotes: the
double-quoted spelling reaches the date grammar with its quotes still
attached and is not a recognizable date. The docs say so, so pin which
of the two quote characters is the one that fails.
"""
with pytest.raises(InvalidDateQuery) as exc_info:
_matched_ids(backend, 'added:["2005-03-04" to 2005-03-05]')
assert exc_info.value.value == '"2005-03-04"'
src/documents/tests/search/test_pattern_stemming.py
+233
View File
@@ -0,0 +1,233 @@
"""Wildcard patterns must match a stemmed index.
Query patterns are normalized but were not stemmed, while index terms are
stemmed, so the natural spelling of a prefix search matched nothing:
``invoice*`` found no document although ``invoic*`` did. v2's index was
UNSTEMMED (whoosh ``TEXT()`` defaults to ``StandardAnalyzer``), so this
regressed against both baselines.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import pytest
from documents.models import Document
from documents.search._registry import _make_pattern_normalizer
from documents.search._tokenizer import ascii_fold
from documents.search._tokenizer import paperless_text_analyzer
from documents.search._tokenizer import stem_pattern_text
if TYPE_CHECKING:
from whoosh_compat import PatternNormalizer
from documents.search._backend import TantivyBackend
pytestmark = [pytest.mark.search, pytest.mark.django_db]
CONTENT = (
"invoice total due for electricity from both companies, "
"payments made to the university library, copies attached"
)
def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
return set(backend.search_ids(query, user=None))
@pytest.fixture
def indexed_doc(backend: TantivyBackend) -> Document:
doc = Document.objects.create(
title="Invoice 2020 productname",
content=CONTENT,
checksum="pattern-stemming-1",
archive_serial_number=900,
)
backend.add_or_update(doc)
return doc
class TestPrefixStemming:
@pytest.mark.parametrize(
"query",
[
"invoice*",
"electricity*",
"companies*",
"payments*",
"library*",
"title:Invoice*",
],
)
def test_full_word_prefix_matches_its_stem(
self,
backend: TantivyBackend,
indexed_doc: Document,
query: str,
) -> None:
assert _matched_ids(backend, query) == {indexed_doc.id}
@pytest.mark.parametrize("query", ["invoic*", "electr*", "payment*"])
def test_already_stemmed_prefix_still_matches(
self,
backend: TantivyBackend,
indexed_doc: Document,
query: str,
) -> None:
assert _matched_ids(backend, query) == {indexed_doc.id}
@pytest.mark.parametrize("query", ["univers*", "librar*"])
def test_partial_prefix_reaches_the_stemmed_term(
self,
backend: TantivyBackend,
indexed_doc: Document,
query: str,
) -> None:
"""A prefix shorter than a whole word still matches, and neither of
these needs the two-alternative path to do it.
Measured under "en": the stemmer leaves "librar" alone, so it has one
form, and that form is a prefix of the "librari" the index holds for
"library". "univers" stems to the *shorter* "univ", and the run as
typed and its stem are both prefixes of the "univers" the index holds
for "university". The case where the two forms genuinely diverge, and
only one of them matches, is
test_stem_substitution_reaches_both_the_inflection_and_the_compound.
"""
assert _matched_ids(backend, query) == {indexed_doc.id}
def test_full_word_reaches_the_stem_but_a_fragment_of_it_does_not(
self,
backend: TantivyBackend,
indexed_doc: Document,
) -> None:
"""The alternatives widen recall without turning a wildcard into a
prefix search over the original text.
"university" is stored as "univers". The stem of "universities" is
that same "univers", so the longer word matches; "universit" is a
prefix of neither its own stem nor the stored term, so the *shorter*
fragment matches nothing. usage.md names this pair, so a reader told
that `universit*` fails is also told which spelling works.
"""
assert _matched_ids(backend, "universities*") == {indexed_doc.id}
assert _matched_ids(backend, "universit*") == set()
def test_pattern_past_the_stem_boundary_is_documented_not_fixed(
self,
backend: TantivyBackend,
indexed_doc: Document,
) -> None:
"""produ*name cannot match a stemmed index ("productname" is indexed as
"productnam"); usage.md must not advertise it. Pinned so the limitation
is deliberate, not accidental."""
assert _matched_ids(backend, "produ*name") == set()
def test_stem_substitution_reaches_both_the_inflection_and_the_compound(
self,
backend: TantivyBackend,
indexed_doc: Document,
) -> None:
"""English stemming substitutes as well as truncates: "copy" and
"copies" both index as "copi", while "copyright" keeps its literal "y".
Neither form is a prefix of the other, so no single normalized string
reaches both. The run is therefore emitted as a disjunction of the
folded and stemmed forms, and "copy*" reaches the base word, its
inflections and the compound alike.
"""
compound = Document.objects.create(
title="Copyright notice",
content="copyright notice for the work",
checksum="pattern-stemming-2",
archive_serial_number=901,
)
backend.add_or_update(compound)
assert _matched_ids(backend, "copy*") == {indexed_doc.id, compound.id}
assert _matched_ids(backend, "copyright*") == {compound.id}
class TestStemsMatchTheIndexAnalyzer:
"""stem_pattern_text rebuilds paperless_text_analyzer's stemming tail rather
than sharing it, so a filter added to the index analyzer alone would silently
stop patterns from reaching the terms it produces.
"""
@pytest.mark.parametrize(
"language",
["en", "de", "fr", "es", "sv", None, "klingon"],
)
@pytest.mark.parametrize(
"word",
["Copies", "copyright", "Companies", "Invoices", "laufen", "casas", "Straße"],
)
def test_stem_equals_the_index_term(self, word: str, language: str | None) -> None:
indexed = paperless_text_analyzer(language).analyze(word)[0]
assert stem_pattern_text(ascii_fold(word.lower()), language) == indexed
def _forms(normalize: PatternNormalizer, text: str) -> tuple[str, ...]:
"""The distinct forms a term may match, in order, the way the emitter reads
the normalizer's answer (see whoosh_compat.PatternNormalizer)."""
result = normalize(text)
if isinstance(result, str):
return (result,)
return tuple(dict.fromkeys(result))
class TestPatternNormalizer:
@pytest.mark.parametrize(
("text", "expected"),
[
("Invoice", ("invoice", "invoic")),
("companies", ("companies", "compani")),
# y -> i is a substitution, so both forms are needed: the index
# holds "librari" for "library" and "library" for "librarian".
("library", ("library", "librari")),
# A run the stemmer leaves alone collapses back to one form, so it
# costs exactly the one regex branch it did before.
("invoic", ("invoic",)),
("Universit", ("universit",)),
("Café", ("cafe",)),
],
)
def test_offers_the_typed_run_and_its_stem(
self,
text: str,
expected: tuple[str, ...],
) -> None:
assert _forms(_make_pattern_normalizer("en"), text) == expected
def test_run_that_yields_no_token_falls_back_to_the_typed_run(self) -> None:
"""A run past the remove_long limit analyzes to zero tokens, so there is
no stem to offer and only the folded run remains."""
over_long = "invoices" * 20
assert _forms(_make_pattern_normalizer("en"), over_long) == (over_long,)
@pytest.mark.parametrize("language", [None, "klingon"])
def test_unstemmed_language_folds_only(self, language: str | None) -> None:
"""With no stemmer configured, or one this build has no stemmer for, the
index holds surface forms and the pattern must keep them too."""
assert _forms(_make_pattern_normalizer(language), "Invoices") == ("invoices",)
@pytest.mark.parametrize("char", ["a", "Z", "é"])
def test_a_single_character_collapses_to_one_folded_form(self, char: str) -> None:
"""A bracket class body is normalized one character at a time and the
answer is used only when it is a single one-character form, so a
stemmer that changed a lone character would silently disable folding
inside classes."""
forms = _forms(_make_pattern_normalizer("en"), char)
assert len(forms) == 1
assert len(forms[0]) == 1
class TestBracketClassStillFolds:
def test_class_body_matches_case_insensitively(
self,
backend: TantivyBackend,
indexed_doc: Document,
) -> None:
"""The class body is folded per character, which the alternatives
contract preserves only because a lone character stems to itself."""
assert _matched_ids(backend, "title:[IP]nvoice*") == {indexed_doc.id}