mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-08-20 17:53:20 +00:00
Trenton HolmesandClaude Fable 5
8789ff55f5
The fuzzy blend clause handed the raw query string to tantivy's own parser, which rejects whoosh-only grammar (date keywords, whoosh ranges, aliases needing resolution), so any mixed query silently lost its fuzzy clause: a typo'd word beside "added:today" stopped matching the moment the date keyword appeared, while the same typo without it still matched. Before the whoosh-compat migration the parser received the translated string, so fuzzy survived mixed queries. The clause is now built from whoosh_compat.free_text_tokens over the already-parsed AST: the query's free-text words, analyzed, deduplicated, with negated terms excluded so a NOT'd word cannot resurface through the fuzzy clause. The joined word string is always plain tokens, so tantivy always parses it; a defensive word-character filter guards any future field whose analyzer passes punctuation through, and the ValueError skip remains as insurance. One chosen trade-off is documented in the docstring: a term fielded on a default search field contributes its text unfielded, widening fuzzy recall on the 0.1-boosted secondary clause. Two result-level acceptance tests pin the behavior: the mixed typo-plus-date-keyword query matches its document again, and a NOT'd word does not fuzzy-resurface (shaped so the assertion genuinely fails under a naive all-words implementation: the excluded word's document is the only candidate hit, so score normalization cannot mask it). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01WMsn6DgzbvSqh1pwy66VVF
411 lines
16 KiB
Python
411 lines
16 KiB
Python
"""Result-level acceptance corpus: real documents indexed via build_schema(),
|
|
real queries run through parse_user_query(), matched-document-ID sets
|
|
asserted — not intermediate ASTs or query strings. This is paperless-ngx's
|
|
analogue of whoosh-compat's own tests/emitter/test_acceptance_e2e.py.
|
|
|
|
Supersedes test_query.py's TestParseUserQuery result-level cases and the
|
|
now-deleted test_date_grammar_parity.py.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from datetime import UTC
|
|
from datetime import date
|
|
from datetime import datetime
|
|
from typing import TYPE_CHECKING
|
|
|
|
import pytest
|
|
import time_machine
|
|
|
|
from documents.models import CustomField
|
|
from documents.models import CustomFieldInstance
|
|
from documents.models import Document
|
|
from documents.models import Note
|
|
from documents.models import Tag
|
|
from documents.search._query import parse_user_query
|
|
|
|
if TYPE_CHECKING:
|
|
from documents.search._backend import TantivyBackend
|
|
|
|
pytestmark = [pytest.mark.search, pytest.mark.django_db]
|
|
|
|
FROZEN_NOW = datetime(2026, 6, 15, 12, 0, tzinfo=UTC)
|
|
|
|
|
|
def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
|
|
return set(backend.search_ids(query, user=None))
|
|
|
|
|
|
@pytest.fixture
|
|
def indexed_documents(backend: TantivyBackend) -> dict[str, int]:
|
|
"""Index a small fixture set, return {label: doc_id} for corpus queries."""
|
|
docs = {
|
|
"invoice_2020": Document.objects.create(
|
|
title="Invoice 2020",
|
|
content="invoice total due",
|
|
checksum="acc-invoice-2020",
|
|
archive_serial_number=100,
|
|
),
|
|
"invoice_2021": Document.objects.create(
|
|
title="Invoice 2021",
|
|
content="invoice total due",
|
|
checksum="acc-invoice-2021",
|
|
archive_serial_number=101,
|
|
),
|
|
"invoice_2023": Document.objects.create(
|
|
title="Invoice 2023",
|
|
content="invoice total due",
|
|
checksum="acc-invoice-2023",
|
|
archive_serial_number=102,
|
|
),
|
|
"receipt_2022": Document.objects.create(
|
|
title="Receipt 2022",
|
|
content="receipt total due",
|
|
checksum="acc-receipt-2022",
|
|
archive_serial_number=103,
|
|
),
|
|
}
|
|
for doc in docs.values():
|
|
backend.add_or_update(doc)
|
|
return {label: doc.pk for label, doc in docs.items()}
|
|
|
|
|
|
class TestIssue13568BracketWildcard:
|
|
"""paperless-ngx#13568: title:202[0-3]* must keep its character class,
|
|
not fold to a prefix query that silently drops it (whoosh-compat
|
|
DIVERGENCES.md entry 13)."""
|
|
|
|
def test_bracket_class_wildcard_matches_only_in_range_years(
|
|
self,
|
|
backend: TantivyBackend,
|
|
indexed_documents: dict[str, int],
|
|
) -> None:
|
|
# [0-1] (not [0-3]) is deliberate: the fixture's four years are
|
|
# 2020/2021/2022/2023, i.e. their trailing digit is 0/1/2/3
|
|
# respectively - a [0-3] class would match all four and the test
|
|
# would pass even if the character class were silently dropped and
|
|
# folded to an unconstrained "202*" prefix. [0-1] partitions the
|
|
# fixture into a genuine in-range/out-of-range split.
|
|
matched = _matched_ids(backend, "title:202[0-1]*")
|
|
expected = {
|
|
indexed_documents["invoice_2020"],
|
|
indexed_documents["invoice_2021"],
|
|
}
|
|
assert matched == expected, (
|
|
"title:202[0-1]* must match 2020/2021 titles and exclude 2022/2023 "
|
|
"- if this matches everything, the wildcard's character class was "
|
|
"silently dropped (issue #13568's original bug)"
|
|
)
|
|
|
|
|
|
class TestCommaValueLists:
|
|
"""whoosh-compat's CommaValuesPlugin splits `tag:foo,bar` into
|
|
`tag:foo AND tag:bar` (DIVERGENCES.md entries 17/36), matching real
|
|
Whoosh's KEYWORD(commas=True) analyzer-time comma splitting - not an OR
|
|
across the listed values. A document must carry every listed tag to
|
|
match."""
|
|
|
|
def test_tag_comma_list_matches_only_documents_with_both_tags(
|
|
self,
|
|
backend: TantivyBackend,
|
|
) -> None:
|
|
tag_foo = Tag.objects.create(name="foo")
|
|
tag_bar = Tag.objects.create(name="bar")
|
|
tag_baz = Tag.objects.create(name="baz")
|
|
|
|
doc_both = Document.objects.create(
|
|
title="Both",
|
|
content="x",
|
|
checksum="acc-comma-both",
|
|
)
|
|
doc_both.tags.add(tag_foo, tag_bar)
|
|
doc_foo_only = Document.objects.create(
|
|
title="FooOnly",
|
|
content="x",
|
|
checksum="acc-comma-foo",
|
|
)
|
|
doc_foo_only.tags.add(tag_foo)
|
|
doc_other = Document.objects.create(
|
|
title="Other",
|
|
content="x",
|
|
checksum="acc-comma-other",
|
|
)
|
|
doc_other.tags.add(tag_baz)
|
|
for doc in (doc_both, doc_foo_only, doc_other):
|
|
backend.add_or_update(doc)
|
|
matched = _matched_ids(backend, "tag:foo,bar")
|
|
assert matched == {doc_both.pk}
|
|
|
|
|
|
class TestFieldBoosts:
|
|
def test_title_boost_ranks_title_match_above_content_only_match(
|
|
self,
|
|
backend: TantivyBackend,
|
|
) -> None:
|
|
title_match = Document.objects.create(
|
|
title="urgent",
|
|
content="nothing else relevant",
|
|
checksum="acc-boost-title",
|
|
)
|
|
content_match = Document.objects.create(
|
|
title="nothing",
|
|
content="urgent matter here",
|
|
checksum="acc-boost-content",
|
|
)
|
|
backend.add_or_update(title_match)
|
|
backend.add_or_update(content_match)
|
|
query = parse_user_query(backend._index, "urgent", UTC)
|
|
searcher = backend._index.searcher()
|
|
results = searcher.search(query, limit=10)
|
|
ranked_ids = [
|
|
searcher.doc(addr).to_dict()["id"][0] for _score, addr in results.hits
|
|
]
|
|
assert ranked_ids[0] == title_match.pk
|
|
|
|
|
|
class TestJsonSubpaths:
|
|
def test_notes_user_matches_document_with_that_note_author(
|
|
self,
|
|
backend: TantivyBackend,
|
|
) -> None:
|
|
from django.contrib.auth.models import User
|
|
|
|
alice = User.objects.create_user(username="alice")
|
|
doc_with_note = Document.objects.create(
|
|
title="Has note",
|
|
content="x",
|
|
checksum="acc-note-with",
|
|
)
|
|
Note.objects.create(document=doc_with_note, user=alice, note="reminder")
|
|
doc_without = Document.objects.create(
|
|
title="No note",
|
|
content="x",
|
|
checksum="acc-note-without",
|
|
)
|
|
backend.add_or_update(doc_with_note)
|
|
backend.add_or_update(doc_without)
|
|
matched = _matched_ids(backend, "notes.user:alice")
|
|
assert matched == {doc_with_note.pk}
|
|
|
|
def test_custom_fields_name_and_value_combine(
|
|
self,
|
|
backend: TantivyBackend,
|
|
) -> None:
|
|
field = CustomField.objects.create(
|
|
name="Contract Number",
|
|
data_type=CustomField.FieldDataType.STRING,
|
|
)
|
|
other_field = CustomField.objects.create(
|
|
name="Other Field",
|
|
data_type=CustomField.FieldDataType.STRING,
|
|
)
|
|
matching = Document.objects.create(
|
|
title="Matching",
|
|
content="x",
|
|
checksum="acc-cf-matching",
|
|
)
|
|
CustomFieldInstance.objects.create(
|
|
document=matching,
|
|
field=field,
|
|
value_text="policy",
|
|
)
|
|
non_matching = Document.objects.create(
|
|
title="Non-matching",
|
|
content="x",
|
|
checksum="acc-cf-nonmatching",
|
|
)
|
|
CustomFieldInstance.objects.create(
|
|
document=non_matching,
|
|
field=other_field,
|
|
value_text="policy",
|
|
)
|
|
backend.add_or_update(matching)
|
|
backend.add_or_update(non_matching)
|
|
matched = _matched_ids(
|
|
backend,
|
|
'custom_fields.name:"Contract Number" custom_fields.value:policy',
|
|
)
|
|
assert matched == {matching.pk}
|
|
|
|
|
|
class TestMultitokenInNestedOr:
|
|
"""whoosh-compat DIVERGENCES.md entry 15: Multitoken.DEFAULT resolves by
|
|
syntactic enclosing group, not the parser's fixed default group. Prove
|
|
it doesn't matter for paperless's actual data/fields."""
|
|
|
|
def test_multitoken_tag_value_inside_top_level_or_matches_either_branch(
|
|
self,
|
|
backend: TantivyBackend,
|
|
) -> None:
|
|
# "multi word tag" is a multitoken field value; nested inside a
|
|
# top-level OR with an unrelated clause.
|
|
doc_a = Document.objects.create(title="A", content="x", checksum="acc-mt-a")
|
|
doc_a.tags.create(name="multi word tag")
|
|
doc_b = Document.objects.create(title="B", content="x", checksum="acc-mt-b")
|
|
doc_b.tags.create(name="unrelated")
|
|
backend.add_or_update(doc_a)
|
|
backend.add_or_update(doc_b)
|
|
matched = _matched_ids(backend, 'tag:"multi word tag" OR title:B')
|
|
assert matched == {doc_a.pk, doc_b.pk}
|
|
|
|
|
|
class TestRfc3339TZDateRange:
|
|
"""paperless-ngx#13010: created/added bracket ranges using RFC3339 T/Z
|
|
datetime separators (e.g. `[2026-01-01T00:00:00Z TO ...]`) must keep
|
|
working - this is v2/Whoosh saved-search backward compatibility, not a
|
|
generic ISO-format nicety. whoosh-compat's own grammar previously had no
|
|
support for `T`/`Z` at all (fixed upstream, whoosh-compat commit
|
|
f936143); this proves the fix holds end-to-end against real indexed
|
|
documents and real timezone-sensitive matching, not just that the
|
|
library's date_from() parses the text."""
|
|
|
|
def test_t_z_range_matches_only_documents_within_bounds(
|
|
self,
|
|
backend: TantivyBackend,
|
|
) -> None:
|
|
in_range = Document.objects.create(
|
|
title="In range",
|
|
content="x",
|
|
checksum="acc-rfc3339-in-range",
|
|
added=datetime(2026, 3, 15, 10, 0, tzinfo=UTC),
|
|
)
|
|
out_of_range = Document.objects.create(
|
|
title="Out of range",
|
|
content="x",
|
|
checksum="acc-rfc3339-out-of-range",
|
|
added=datetime(2026, 8, 1, 10, 0, tzinfo=UTC),
|
|
)
|
|
for doc in (in_range, out_of_range):
|
|
backend.add_or_update(doc)
|
|
matched = _matched_ids(
|
|
backend,
|
|
"added:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z]",
|
|
)
|
|
assert matched == {in_range.pk}
|
|
|
|
def test_comma_combined_t_z_ranges_across_two_fields(
|
|
self,
|
|
backend: TantivyBackend,
|
|
) -> None:
|
|
# The Whoosh v2 comma syntax: two field:[range] expressions joined
|
|
# by a comma must be ANDed together (not passed to Tantivy as a
|
|
# literal comma, which it cannot parse).
|
|
matching = Document.objects.create(
|
|
title="Matches both ranges",
|
|
content="x",
|
|
checksum="acc-rfc3339-comma-match",
|
|
created=date(2026, 3, 15),
|
|
added=datetime(2026, 5, 15, 10, 0, tzinfo=UTC),
|
|
)
|
|
wrong_added = Document.objects.create(
|
|
title="created in range, added out of range",
|
|
content="x",
|
|
checksum="acc-rfc3339-comma-wrong-added",
|
|
created=date(2026, 3, 15),
|
|
added=datetime(2026, 8, 1, 10, 0, tzinfo=UTC),
|
|
)
|
|
for doc in (matching, wrong_added):
|
|
backend.add_or_update(doc)
|
|
matched = _matched_ids(
|
|
backend,
|
|
"created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z],"
|
|
"added:[2026-05-01T00:00:00Z TO 2026-06-01T00:00:00Z]",
|
|
)
|
|
assert matched == {matching.pk}
|
|
|
|
def test_z_suffixed_bound_is_absolute_utc_not_local_shifted(
|
|
self,
|
|
backend: TantivyBackend,
|
|
) -> None:
|
|
# A document timestamped exactly at a Z-suffixed range boundary must
|
|
# match under UTC - if Z were (incorrectly) reinterpreted as local
|
|
# wall-clock time and shifted again, this exact-boundary match would
|
|
# silently fail or succeed for the wrong reason.
|
|
at_boundary = Document.objects.create(
|
|
title="At Z boundary",
|
|
content="x",
|
|
checksum="acc-rfc3339-z-boundary",
|
|
added=datetime(2026, 1, 1, 0, 0, 0, tzinfo=UTC),
|
|
)
|
|
backend.add_or_update(at_boundary)
|
|
matched = _matched_ids(
|
|
backend,
|
|
"added:[2026-01-01T00:00:00Z TO 2026-01-01T00:00:01Z]",
|
|
)
|
|
assert matched == {at_boundary.pk}
|
|
|
|
|
|
class TestUnregisteredIdFieldFoldsToLiteralText:
|
|
"""tag_id, owner_id, etc. are intentionally excluded from the
|
|
FieldRegistry - whoosh-compat parity leniency folds them into a literal
|
|
text search rather than raising a diagnostic/400 (see docs/usage.md's
|
|
advanced-search section). Prove the fold is inert against real data, not
|
|
just that parsing doesn't raise."""
|
|
|
|
def test_tag_id_query_matches_nothing(
|
|
self,
|
|
backend: TantivyBackend,
|
|
indexed_documents: dict[str, int],
|
|
) -> None:
|
|
matched = _matched_ids(backend, "tag_id:5")
|
|
assert matched == set()
|
|
|
|
|
|
class TestFuzzyBlendSurvivesWhooshGrammar:
|
|
"""A query mixing whoosh-only grammar (a date keyword) with a typo'd
|
|
free-text word must still fuzzy-match the intended document when
|
|
ADVANCED_FUZZY_SEARCH_THRESHOLD is enabled. The fuzzy clause is built
|
|
from the parsed query's free-text tokens (whoosh_compat's
|
|
free_text_tokens), never from the raw query string, so whoosh grammar
|
|
that tantivy's own parser rejects cannot knock the fuzzy clause out."""
|
|
|
|
def test_typo_fuzzy_matches_alongside_date_keyword(
|
|
self,
|
|
backend: TantivyBackend,
|
|
settings,
|
|
) -> None:
|
|
settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.5
|
|
with time_machine.travel(FROZEN_NOW, tick=False):
|
|
doc = Document.objects.create(
|
|
title="Receipt March",
|
|
content="receipt total due",
|
|
checksum="fuzzy-blend-1",
|
|
archive_serial_number=900,
|
|
)
|
|
backend.add_or_update(doc)
|
|
# Sanity: the exact spelling matches through the exact clause.
|
|
assert doc.pk in _matched_ids(backend, "added:today receipt")
|
|
# The regression: the misspelling (one transposition) only
|
|
# matches via the fuzzy clause, and "added:today" is
|
|
# whoosh-only grammar tantivy's parser rejects, so raw-string
|
|
# fuzzy parsing skips the clause entirely and this returns
|
|
# nothing. The typo is deliberate; keep codespell away from it.
|
|
typo_query = "added:today reciept" # codespell:ignore reciept
|
|
assert doc.pk in _matched_ids(backend, typo_query)
|
|
|
|
def test_negated_words_do_not_fuzzy_match(
|
|
self,
|
|
backend: TantivyBackend,
|
|
settings,
|
|
) -> None:
|
|
# A term the user excluded must not resurface through the fuzzy
|
|
# clause. The shape is chosen so this genuinely discriminates: the
|
|
# indexed document contains the NOT'd word but NOT the positive
|
|
# word, so nothing matches the exact clause, and a fuzzy string
|
|
# naively built from ALL words (including the NOT'd one) would
|
|
# make this document the sole hit, normalize its score to 1.0,
|
|
# and survive any threshold. (A shape with an exact-matching
|
|
# sibling document does NOT discriminate: normalization ranks the
|
|
# resurfaced doc far below the exact match and the threshold cuts
|
|
# it even for a naive implementation.)
|
|
settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.5
|
|
with time_machine.travel(FROZEN_NOW, tick=False):
|
|
receipt_only = Document.objects.create(
|
|
title="Receipt Archive",
|
|
content="receipt archived stack",
|
|
checksum="fuzzy-blend-2",
|
|
archive_serial_number=901,
|
|
)
|
|
backend.add_or_update(receipt_only)
|
|
assert _matched_ids(backend, "added:today total NOT receipt") == set()
|