amorfo77/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2026-09-12 12:47:58 +00:00
Code Issues Projects Releases Wiki Activity

Adds the test doc strings too

Browse Source
This commit is contained in:
stumpylog
2026-09-11 11:05:45 -07:00
parent 981dfe7bb1
commit 55209dedb5
3 changed files with 314 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/documents/tests/search/test_compact_date_forms.py
+22 -11
View File
@@ -1,14 +1,13 @@
"""Whoosh's compact, separator-free date spelling, resolved end to end.
whoosh-compat owns both widths of this spelling and asserts both of each
form's bounds directly: ``test_compact_numeric_datetime`` pins the 8-digit
form as a whole calendar day (lower bound, upper bound and exclusivity), and
``test_compact_numeric_datetime_full_width_is_a_single_second_instant`` pins
the 14-digit form as one instant. The 14-digit form is kept here as the single
representative because it is the one that exercises paperless's ``added``
DATETIME fast field at full precision: the corpus separates a document at
the named instant from one on the same calendar day at another hour and one
on the next day at the same hour, so a query that degrades into a whole-day
whoosh-compat owns both widths of this spelling and asserts both forms'
bounds directly in its own test suite: the 8-digit form as a whole calendar
day (lower bound, upper bound and exclusivity), and the 14-digit form as a
single instant. The 14-digit form is kept here as the single representative
because it is the one that exercises paperless's ``added`` DATETIME fast
field at full precision: the corpus separates a document at the named
instant from one on the same calendar day at another hour and one on the
next day at the same hour, so a query that degrades into a whole-day
window, or drops the time of day, matches the wrong set rather than passing
on a corpus that could not tell the difference.
"""
@@ -70,6 +69,18 @@ def test_fourteen_digits_is_a_single_instant(
backend: TantivyBackend,
docs: dict[str, int],
) -> None:
# same_day is what tells this apart from the 8-digit day-window form,
# next_day from a form that ignored the time altogether.
"""
GIVEN:
- Three documents indexed on the ``added`` DATETIME fast field:
one at 2005-03-04T15:30:00, one on the same calendar day at a
different hour, and one on the next day at the same hour
WHEN:
- Searching with the 14-digit compact date form
``added:20050304153000``
THEN:
- Only the document at that exact instant matches; the same-day
document is what tells this apart from the 8-digit day-window
form, and the next-day document from a form that ignored the
time of day altogether
"""
assert _matched_ids(backend, "added:20050304153000") == {docs["instant"]}
src/documents/tests/search/test_documented_syntax.py
+139 -20
View File
@@ -77,6 +77,14 @@ class TestLogicalExpressions:
backend: TantivyBackend,
docs: dict[str, int],
) -> None:
"""
GIVEN:
- Two indexed documents, one containing "secret" and one not
WHEN:
- "invoice NOT secret" is searched, as docs/usage.md documents
THEN:
- Only the document without "secret" matches
"""
assert _matched_ids(backend, "invoice NOT secret") == {docs["plain"]}
def test_leading_hyphen_requires_the_term_instead_of_excluding_it(
@@ -84,8 +92,17 @@ class TestLogicalExpressions:
backend: TantivyBackend,
docs: dict[str, int],
) -> None:
# The docs warn about exactly this: separators are stripped at index
# time, so "-secret" is the term "secret" and the query is an AND.
"""
GIVEN:
- Two indexed documents, one containing "secret" and one not
WHEN:
- "invoice -secret" is searched (a leading hyphen, not "NOT")
THEN:
- Only the document containing "secret" matches, because
separators are stripped at index time, so "-secret" is
indexed as the plain term "secret" and the query becomes an
AND rather than an exclusion, exactly as the docs warn
"""
assert _matched_ids(backend, "invoice -secret") == {docs["secret"]}
def test_or_inside_parentheses_matches_either_branch(
@@ -93,6 +110,15 @@ class TestLogicalExpressions:
backend: TantivyBackend,
docs: dict[str, int],
) -> None:
"""
GIVEN:
- Two indexed documents, one containing "secret" and one
containing "ordinary"
WHEN:
- "invoice AND (secret OR ordinary)" is searched
THEN:
- Both documents match
"""
matched = _matched_ids(backend, "invoice AND (secret OR ordinary)")
assert matched == {docs["secret"], docs["plain"]}
@@ -102,6 +128,15 @@ class TestPhraseSearch:
self,
backend: TantivyBackend,
) -> None:
"""
GIVEN:
- A document whose content contains "the quick brown fox jumps"
WHEN:
- A quoted phrase is searched, in order and out of order
THEN:
- The in-order phrase matches, and the same words reordered do
not
"""
doc = _index(
backend,
title="Phrase",
@@ -131,6 +166,16 @@ class TestTagCommaList:
self,
backend: TantivyBackend,
) -> None:
"""
GIVEN:
- A document carrying both "bills" and "unpaid" tags, and a
second document carrying only "bills" (plus "archived")
WHEN:
- "tag:bills,unpaid" is searched
THEN:
- Only the document carrying every listed tag matches, and a
single-tag "tag:bills" search still matches both documents
"""
bills = Tag.objects.create(name="bills")
unpaid = Tag.objects.create(name="unpaid")
archived = Tag.objects.create(name="archived")
@@ -195,6 +240,18 @@ class TestArchiveMetadataFields:
doc: Document,
query: str,
) -> None:
"""
GIVEN:
- A document with an ASN, page count, a note, an original
filename and a known checksum
WHEN:
- Every documented metadata-field spelling (exact value,
range, and, for checksum, a lowercase prefix pattern
regardless of the case the pattern itself is typed in) is
searched
THEN:
- Each one matches the document
"""
assert _matched_ids(backend, query) == {doc.pk}
@pytest.mark.parametrize(
@@ -211,6 +268,16 @@ class TestArchiveMetadataFields:
doc: Document,
query: str,
) -> None:
"""
GIVEN:
- A document with a known, complete, lowercase checksum
WHEN:
- An exact-value search is run with a partial or uppercase
spelling of that checksum
THEN:
- Nothing matches, as the docs say only a complete, lowercase
checksum matches as an exact value
"""
assert _matched_ids(backend, query) == set()
@@ -272,6 +339,21 @@ class TestDocumentedDateForms:
query: str,
label: str,
) -> None:
"""
GIVEN:
- Documents dated today, yesterday, tomorrow, next/last
Monday, in January, and on an old fixed date, indexed
against a frozen "now" (a Monday)
WHEN:
- Every documented date-form spelling is searched: relative
keywords, quoted multi-word phrases, a bare year-month, an
explicit range, a quoted full timestamp standing alone, an
unquoted full timestamp as a range bound, and a
single-quoted range bound
THEN:
- Each form matches exactly the document dated on its day or
within its month
"""
assert _matched_ids(backend, query) == {dated[label]}
@pytest.mark.parametrize(
@@ -302,6 +384,21 @@ class TestDocumentedDateForms:
dated: dict[str, int],
query: str,
) -> None:
"""
GIVEN:
- A realistic dated corpus (see the `dated` fixture)
WHEN:
- A zero-width date form ("now", "noon", "midnight", a quoted
"now") or a standalone relative offset ("-1 week") is
searched: each resolves to a single instant rather than a
span, and quoting does not rescue them the way it rescues
other multi-word date expressions, since the problem is the
width of the resulting range, not how the value is
delimited
THEN:
- Nothing matches, exactly as the docs warn, rather than
presenting these as usable spellings
"""
assert _matched_ids(backend, query) == set()
def test_bare_timestamp_is_rejected_rather_than_matching_nothing(
@@ -309,13 +406,20 @@ class TestDocumentedDateForms:
backend: TantivyBackend,
dated: dict[str, int],
) -> None:
"""The bare, unquoted spelling of a full timestamp. The quoted and
range-bound spellings pinned above do work and match this fixture's
document; this one is a user-fixable error rather than an empty
result set, so the docs tell the user to quote it.
The reported value is the whole contiguous fragment the user typed,
not just the prefix the date grammar's tokenizer first split on.
"""
GIVEN:
- A realistic dated corpus, including a document dated at a
known full timestamp
WHEN:
- The bare, unquoted spelling of that full timestamp is
searched (the quoted and range-bound spellings pinned above
do work and match this fixture's document)
THEN:
- `InvalidDateQuery` is raised rather than the query silently
matching nothing, since this is a user-fixable error the
docs tell the user to quote, and the reported value is the
whole contiguous fragment the user typed, not just the
prefix the date grammar's tokenizer first split on
"""
with pytest.raises(InvalidDateQuery) as exc_info:
_matched_ids(backend, "added:2005-03-04T15:30:00Z")
@@ -327,13 +431,20 @@ class TestDocumentedDateForms:
backend: TantivyBackend,
dated: dict[str, int],
) -> None:
"""The same offset that matches nothing on its own spans the last
seven days as a lower bound. The docs say so, next to the warning
about the standalone form, so both readings are pinned together.
"last_monday" is indexed at 2026-06-08T10:00, two hours before the
window opens, so its exclusion is what shows the bound is the offset
and not a whole-day rounding of it.
"""
GIVEN:
- A realistic dated corpus, including a document dated two
hours before a "last Monday to now" window opens, and
documents dated today and yesterday, inside that window
WHEN:
- "added:['-1 week' to now]" is searched: the same offset
that matches nothing standing alone (see the test above),
used here as a range bound instead
THEN:
- The window matches today and yesterday but excludes the
document two hours before it opens, showing the bound is
the offset itself and not a whole-day rounding of it, as
the docs say next to the warning about the standalone form
"""
assert _matched_ids(backend, "added:['-1 week' to now]") == {
dated["today"],
@@ -345,10 +456,18 @@ class TestDocumentedDateForms:
backend: TantivyBackend,
dated: dict[str, int],
) -> None:
"""Quoting a range bound is allowed, but only with single quotes: the
double-quoted spelling reaches the date grammar with its quotes still
attached and is not a recognizable date. The docs say so, so pin which
of the two quote characters is the one that fails.
"""
GIVEN:
- A realistic dated corpus
WHEN:
- A range bound is double-quoted rather than single-quoted
("added:[\"2005-03-04\" to 2005-03-05]")
THEN:
- `InvalidDateQuery` is raised, pinning which of the two
quote characters fails: quoting a range bound is allowed,
but only with single quotes, since the double-quoted
spelling reaches the date grammar with its quotes still
attached and is not a recognizable date
"""
with pytest.raises(InvalidDateQuery) as exc_info:
_matched_ids(backend, 'added:["2005-03-04" to 2005-03-05]')
src/documents/tests/search/test_pattern_stemming.py
+153 -37
View File
@@ -66,6 +66,19 @@ class TestPrefixStemming:
indexed_doc: Document,
query: str,
) -> None:
"""
GIVEN:
- A document indexed with content containing "invoice",
"electricity", "companies", "payments", "library" and title
"Invoice 2020 productname"
WHEN:
- A prefix wildcard on the full, unstemmed word is queried
(e.g. "invoice*", "title:Invoice*")
THEN:
- The document matches, since the pattern normalizer offers
the word's stem as an alternative alongside the typed run,
reaching the stemmed index term
"""
assert _matched_ids(backend, query) == {indexed_doc.id}
@pytest.mark.parametrize("query", ["invoic*", "electr*", "payment*"])
@@ -75,6 +88,16 @@ class TestPrefixStemming:
indexed_doc: Document,
query: str,
) -> None:
"""
GIVEN:
- The same indexed document
WHEN:
- A prefix wildcard is typed already in its stemmed spelling
(e.g. "invoic*")
THEN:
- The document still matches, since the typed-run alternative
is itself a prefix of the stored stemmed term
"""
assert _matched_ids(backend, query) == {indexed_doc.id}
@pytest.mark.parametrize("query", ["univers*", "librar*"])
@@ -84,16 +107,22 @@ class TestPrefixStemming:
indexed_doc: Document,
query: str,
) -> None:
"""A prefix shorter than a whole word still matches, and neither of
these needs the two-alternative path to do it.
Measured under "en": the stemmer leaves "librar" alone, so it has one
form, and that form is a prefix of the "librari" the index holds for
"library". "univers" stems to the *shorter* "univ", and the run as
typed and its stem are both prefixes of the "univers" the index holds
for "university". The case where the two forms genuinely diverge, and
only one of them matches, is
test_stem_substitution_reaches_both_the_inflection_and_the_compound.
"""
GIVEN:
- The same indexed document
WHEN:
- A prefix shorter than a whole word is queried ("univers*",
"librar*")
THEN:
- It still matches, and neither case needs the two-alternative
path to do it: measured under "en", the stemmer leaves
"librar" alone, so it has one form, and that form is a
prefix of the "librari" the index holds for "library";
"univers" stems to the *shorter* "univ", and the run as
typed and its stem are both prefixes of the "univers" the
index holds for "university". The case where the two forms
genuinely diverge, and only one of them matches, is
test_stem_substitution_reaches_both_the_inflection_and_the_compound
"""
assert _matched_ids(backend, query) == {indexed_doc.id}
@@ -102,14 +131,19 @@ class TestPrefixStemming:
backend: TantivyBackend,
indexed_doc: Document,
) -> None:
"""The alternatives widen recall without turning a wildcard into a
prefix search over the original text.
"university" is stored as "univers". The stem of "universities" is
that same "univers", so the longer word matches; "universit" is a
prefix of neither its own stem nor the stored term, so the *shorter*
fragment matches nothing. usage.md names this pair, so a reader told
that `universit*` fails is also told which spelling works.
"""
GIVEN:
- The same indexed document, storing "university" as "univers"
WHEN:
- "universities*" and "universit*" are each queried
THEN:
- "universities*" matches, since the stem of "universities" is
that same "univers"; "universit*" matches nothing, since
"universit" is a prefix of neither its own stem nor the
stored term. The alternatives widen recall without turning
a wildcard into a prefix search over the original text, and
usage.md names this exact pair so a reader told that
`universit*` fails is also told which spelling works
"""
assert _matched_ids(backend, "universities*") == {indexed_doc.id}
assert _matched_ids(backend, "universit*") == set()
@@ -119,9 +153,18 @@ class TestPrefixStemming:
backend: TantivyBackend,
indexed_doc: Document,
) -> None:
"""produ*name cannot match a stemmed index ("productname" is indexed as
"productnam"); usage.md must not advertise it. Pinned so the limitation
is deliberate, not accidental."""
"""
GIVEN:
- The same indexed document, with "productname" indexed as
"productnam"
WHEN:
- "produ*name" (a pattern straddling the stem boundary) is
queried
THEN:
- It matches nothing; produ*name cannot match a stemmed
index, and usage.md must not advertise it. Pinned so the
limitation is deliberate, not accidental
"""
assert _matched_ids(backend, "produ*name") == set()
def test_stem_substitution_reaches_both_the_inflection_and_the_compound(
@@ -129,12 +172,22 @@ class TestPrefixStemming:
backend: TantivyBackend,
indexed_doc: Document,
) -> None:
"""English stemming substitutes as well as truncates: "copy" and
"copies" both index as "copi", while "copyright" keeps its literal "y".
Neither form is a prefix of the other, so no single normalized string
reaches both. The run is therefore emitted as a disjunction of the
folded and stemmed forms, and "copy*" reaches the base word, its
inflections and the compound alike.
"""
GIVEN:
- The indexed document (containing "copies") plus a second
document titled "Copyright notice" with content "copyright
notice for the work"
WHEN:
- "copy*" and "copyright*" are each queried
THEN:
- "copy*" matches both documents, and "copyright*" matches
only the compound one. English stemming substitutes as well
as truncates: "copy" and "copies" both index as "copi",
while "copyright" keeps its literal "y". Neither form is a
prefix of the other, so no single normalized string reaches
both; the run is therefore emitted as a disjunction of the
folded and stemmed forms, and "copy*" reaches the base
word, its inflections and the compound alike
"""
compound = Document.objects.create(
title="Copyright notice",
@@ -163,6 +216,22 @@ class TestStemsMatchTheIndexAnalyzer:
["Copies", "copyright", "Companies", "Invoices", "laufen", "casas", "Straße"],
)
def test_stem_equals_the_index_term(self, word: str, language: str | None) -> None:
"""
GIVEN:
- A word, across several representative index languages
("en", "de", "fr", "es", "sv"), no language, and an
unsupported language ("klingon")
WHEN:
- `stem_pattern_text` (the pattern-side stemmer) processes the
folded word, and `paperless_text_analyzer` (the index-side
analyzer) independently processes the same word
THEN:
- The two produce the identical term. `stem_pattern_text`
rebuilds `paperless_text_analyzer`'s stemming tail rather
than sharing it, so a filter added to the index analyzer
alone would silently stop patterns from reaching the terms
it produces; this pins the two staying in sync
"""
indexed = paperless_text_analyzer(language).analyze(word)[0]
assert stem_pattern_text(ascii_fold(word.lower()), language) == indexed
@@ -197,26 +266,64 @@ class TestPatternNormalizer:
text: str,
expected: tuple[str, ...],
) -> None:
"""
GIVEN:
- The "en" pattern normalizer
WHEN:
- It processes a literal run (e.g. "Invoice", "library",
"Café")
THEN:
- It returns the folded run and, where it differs, the
stemmed form, as distinct alternatives; a run the stemmer
leaves alone (e.g. "invoic") collapses back to the single
folded form. "library" needs both forms since y -> i is a
substitution: the index holds "librari" for "library" and
"library" for "librarian"
"""
assert _forms(_make_pattern_normalizer("en"), text) == expected
def test_run_that_yields_no_token_falls_back_to_the_typed_run(self) -> None:
"""A run past the remove_long limit analyzes to zero tokens, so there is
no stem to offer and only the folded run remains."""
"""
GIVEN:
- The "en" pattern normalizer
WHEN:
- It processes a run past the analyzer's remove_long limit
THEN:
- The run analyzes to zero tokens, so there is no stem to
offer, and only the folded run remains
"""
over_long = "invoices" * 20
assert _forms(_make_pattern_normalizer("en"), over_long) == (over_long,)
@pytest.mark.parametrize("language", [None, "klingon"])
def test_unstemmed_language_folds_only(self, language: str | None) -> None:
"""With no stemmer configured, or one this build has no stemmer for, the
index holds surface forms and the pattern must keep them too."""
"""
GIVEN:
- A pattern normalizer with no language configured, or one
this build has no stemmer for ("klingon")
WHEN:
- It processes "Invoices"
THEN:
- Only the folded form ("invoices") is offered, since with no
stemmer configured the index holds surface forms and the
pattern must keep them too
"""
assert _forms(_make_pattern_normalizer(language), "Invoices") == ("invoices",)
@pytest.mark.parametrize("char", ["a", "Z", "é"])
def test_a_single_character_collapses_to_one_folded_form(self, char: str) -> None:
"""A bracket class body is normalized one character at a time and the
answer is used only when it is a single one-character form, so a
stemmer that changed a lone character would silently disable folding
inside classes."""
"""
GIVEN:
- The "en" pattern normalizer
WHEN:
- It processes a single character
THEN:
- Exactly one, one-character form is returned. A bracket
class body is normalized one character at a time and the
answer is used only when it is a single one-character
form, so a stemmer that changed a lone character would
silently disable folding inside classes
"""
forms = _forms(_make_pattern_normalizer("en"), char)
assert len(forms) == 1
assert len(forms[0]) == 1
@@ -228,6 +335,15 @@ class TestBracketClassStillFolds:
backend: TantivyBackend,
indexed_doc: Document,
) -> None:
"""The class body is folded per character, which the alternatives
contract preserves only because a lone character stems to itself."""
"""
GIVEN:
- The indexed document, titled "Invoice 2020 productname"
WHEN:
- A bracket-class pattern mixing case is queried
("title:[IP]nvoice*")
THEN:
- It matches: the class body is folded per character, which
the alternatives contract preserves only because a lone
character stems to itself
"""
assert _matched_ids(backend, "title:[IP]nvoice*") == {indexed_doc.id}