mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-08-19 09:13:24 +00:00
test(search): trim acceptance tests that pin whoosh-compat behavior, not ours
Deletes TestCommaValueLists, TestMultitokenInNestedOr, TestRfc3339TZDateRange, TestCreatedTimezoneInvariance, and TestReversedDateRange: none of them exercise any paperless-specific pre/post-processing code. Comma-list AND semantics, multitoken resolution, RFC3339 T/Z UTC math, date-only timezone invariance, and reversed-range disambiguation are all entirely whoosh-compat's own grammar/semantics, already covered by its own test suite. The comma_values flag paperless does own is still covered cheaply in test_fields.py; the date_only flag is still covered in test_registry.py. Also trims verbose docstrings/comments across _query.py and the surviving acceptance tests: cuts references to whoosh-compat's internal DIVERGENCES.md entry numbers and paperless v2/Whoosh-era implementation history down to the user-facing behavior that actually matters, without losing the substance. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01RVj8NFy821G3YhNf68PF6X
This commit is contained in:
Trenton Holmes
co-authored by
Claude Sonnet 5
Claude Sonnet 5
parent
a37bd5bec4
commit
194c2bce48
@@ -84,15 +84,10 @@ _REGEX_TIMEOUT: Final[float] = 1.0
|
||||
# Uses Unicode properties to cover all blocks including Extension B+ planes.
|
||||
_CJK_RE: Final = regex.compile(r"[\p{Han}\p{Hiragana}\p{Katakana}\p{Hangul}]+")
|
||||
|
||||
# The closed multi-word date-keyword vocabulary, unchanged since paperless
|
||||
# v2's rewrite_natural_date_keywords. whoosh-compat's date grammar
|
||||
# understands every one of these natively, but only as a QUOTED value
|
||||
# (its DIVERGENCES.md entry 19: unquoted multi-word values split at
|
||||
# whitespace, faithfully to whoosh); the unquoted spelling has been
|
||||
# honored continuously since the whoosh era by an app-level assist, so
|
||||
# _quote_date_keyword_phrases below keeps honoring it by inserting the
|
||||
# quotes and nothing else. Single-word keywords (today, yesterday) parse
|
||||
# unquoted already and need no entry.
|
||||
# Multi-word date-keyword phrases whoosh-compat only accepts quoted.
|
||||
# Unquoted has always been the honored spelling, so
|
||||
# _quote_date_keyword_phrases below inserts the quotes and nothing else.
|
||||
# Single-word keywords (today, yesterday) already parse unquoted.
|
||||
_DATE_KEYWORD_PHRASES: Final = (
|
||||
"previous week",
|
||||
"previous month",
|
||||
@@ -122,14 +117,10 @@ _DATE_KEYWORD_PHRASE_RE: Final = regex.compile(
|
||||
def _quote_date_keyword_phrases(raw_query: str) -> str:
|
||||
"""Quote unquoted multi-word date keyword phrases on date fields.
|
||||
|
||||
``added:previous month`` becomes ``added:"previous month"``; the
|
||||
already-quoted spellings don't match the pattern (the colon must be
|
||||
followed directly by the phrase), and the same words after a TEXT
|
||||
field or standing alone are ordinary text and untouched. Only quoting
|
||||
happens here: every date computation stays in whoosh-compat's
|
||||
grammar, which parses exactly this phrase vocabulary as quoted
|
||||
values. This is deliberately NOT a revival of the deleted
|
||||
translation layer, which computed the ranges app-side.
|
||||
``added:previous month`` becomes ``added:"previous month"``; already-
|
||||
quoted spellings, TEXT fields, and standalone words are untouched.
|
||||
Only quoting happens here - every date computation stays in
|
||||
whoosh-compat's grammar.
|
||||
"""
|
||||
return _DATE_KEYWORD_PHRASE_RE.sub(
|
||||
r'\1:"\2"',
|
||||
@@ -138,20 +129,13 @@ def _quote_date_keyword_phrases(raw_query: str) -> str:
|
||||
)
|
||||
|
||||
|
||||
# The v2 whoosh schema had plural notes/custom_fields TEXT fields (notes
|
||||
# indexed the joined note texts; custom_fields indexed joined
|
||||
# "name : value" strings), so the bare plural prefixes were valid fielded
|
||||
# searches in released paperless and at the deleted translation layer. On
|
||||
# the whoosh-compat registry they are JSON fields addressable only via
|
||||
# subpaths, and the bare spelling would demote to an unfielded text search
|
||||
# of the words themselves. Rewrite the prefixes live to the same targets
|
||||
# migration 0017 chose for the singular whoosh-era spellings (note: ->
|
||||
# notes.note:, custom_field: -> custom_fields.value:), values untouched.
|
||||
# Trade-off inherited from that migration: custom_fields.value: drops the
|
||||
# name-matching half of v2's "name : value" indexing (custom_fields.name:
|
||||
# remains available for it). Same lookbehind guard as 0017: not preceded
|
||||
# by a word character or dot, so subpath spellings and words that merely
|
||||
# end in the prefix are untouched.
|
||||
# notes:/custom_fields: were valid fielded searches before this migration.
|
||||
# whoosh-compat's registry only exposes them as JSON subpaths, so a bare
|
||||
# prefix would demote to an unfielded text search. Rewrite live to the
|
||||
# equivalent subpath (notes: -> notes.note:, custom_fields: ->
|
||||
# custom_fields.value:); custom_fields.name: remains available separately.
|
||||
# Not preceded by a word character or dot, so subpath spellings and words
|
||||
# merely ending in the prefix are untouched.
|
||||
_BARE_JSON_PREFIX_RES: Final = (
|
||||
(regex.compile(r"(?<![.\w])notes:(?!\.)"), "notes.note:"),
|
||||
(regex.compile(r"(?<![.\w])custom_fields:(?!\.)"), "custom_fields.value:"),
|
||||
|
||||
@@ -9,7 +9,6 @@ Supersedes test_query.py's TestParseUserQuery result-level cases.
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC
|
||||
from datetime import date
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
@@ -20,7 +19,6 @@ from documents.models import CustomField
|
||||
from documents.models import CustomFieldInstance
|
||||
from documents.models import Document
|
||||
from documents.models import Note
|
||||
from documents.models import Tag
|
||||
from documents.search._query import parse_user_query
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -71,8 +69,7 @@ def indexed_documents(backend: TantivyBackend) -> dict[str, int]:
|
||||
|
||||
class TestIssue13568BracketWildcard:
|
||||
"""paperless-ngx#13568: title:202[0-3]* must keep its character class,
|
||||
not fold to a prefix query that silently drops it (whoosh-compat
|
||||
DIVERGENCES.md entry 13)."""
|
||||
not fold to a prefix query that silently drops it."""
|
||||
|
||||
def test_bracket_class_wildcard_matches_only_in_range_years(
|
||||
self,
|
||||
@@ -97,45 +94,6 @@ class TestIssue13568BracketWildcard:
|
||||
)
|
||||
|
||||
|
||||
class TestCommaValueLists:
|
||||
"""whoosh-compat's CommaValuesPlugin splits `tag:foo,bar` into
|
||||
`tag:foo AND tag:bar` (DIVERGENCES.md entries 17/36), matching real
|
||||
Whoosh's KEYWORD(commas=True) analyzer-time comma splitting - not an OR
|
||||
across the listed values. A document must carry every listed tag to
|
||||
match."""
|
||||
|
||||
def test_tag_comma_list_matches_only_documents_with_both_tags(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
) -> None:
|
||||
tag_foo = Tag.objects.create(name="foo")
|
||||
tag_bar = Tag.objects.create(name="bar")
|
||||
tag_baz = Tag.objects.create(name="baz")
|
||||
|
||||
doc_both = Document.objects.create(
|
||||
title="Both",
|
||||
content="x",
|
||||
checksum="acc-comma-both",
|
||||
)
|
||||
doc_both.tags.add(tag_foo, tag_bar)
|
||||
doc_foo_only = Document.objects.create(
|
||||
title="FooOnly",
|
||||
content="x",
|
||||
checksum="acc-comma-foo",
|
||||
)
|
||||
doc_foo_only.tags.add(tag_foo)
|
||||
doc_other = Document.objects.create(
|
||||
title="Other",
|
||||
content="x",
|
||||
checksum="acc-comma-other",
|
||||
)
|
||||
doc_other.tags.add(tag_baz)
|
||||
for doc in (doc_both, doc_foo_only, doc_other):
|
||||
backend.add_or_update(doc)
|
||||
matched = _matched_ids(backend, "tag:foo,bar")
|
||||
assert matched == {doc_both.pk}
|
||||
|
||||
|
||||
class TestFieldBoosts:
|
||||
def test_title_boost_ranks_title_match_above_content_only_match(
|
||||
self,
|
||||
@@ -227,122 +185,11 @@ class TestJsonSubpaths:
|
||||
assert matched == {matching.pk}
|
||||
|
||||
|
||||
class TestMultitokenInNestedOr:
|
||||
"""whoosh-compat DIVERGENCES.md entry 15: Multitoken.DEFAULT resolves by
|
||||
syntactic enclosing group, not the parser's fixed default group. Prove
|
||||
it doesn't matter for paperless's actual data/fields."""
|
||||
|
||||
def test_multitoken_tag_value_inside_top_level_or_matches_either_branch(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
) -> None:
|
||||
# "multi word tag" is a multitoken field value; nested inside a
|
||||
# top-level OR with an unrelated clause.
|
||||
doc_a = Document.objects.create(title="A", content="x", checksum="acc-mt-a")
|
||||
doc_a.tags.create(name="multi word tag")
|
||||
doc_b = Document.objects.create(title="B", content="x", checksum="acc-mt-b")
|
||||
doc_b.tags.create(name="unrelated")
|
||||
backend.add_or_update(doc_a)
|
||||
backend.add_or_update(doc_b)
|
||||
matched = _matched_ids(backend, 'tag:"multi word tag" OR title:B')
|
||||
assert matched == {doc_a.pk, doc_b.pk}
|
||||
|
||||
|
||||
class TestRfc3339TZDateRange:
|
||||
"""paperless-ngx#13010: created/added bracket ranges using RFC3339 T/Z
|
||||
datetime separators (e.g. `[2026-01-01T00:00:00Z TO ...]`) must keep
|
||||
working - this is v2/Whoosh saved-search backward compatibility, not a
|
||||
generic ISO-format nicety. whoosh-compat's own grammar previously had no
|
||||
support for `T`/`Z` at all (fixed upstream, whoosh-compat commit
|
||||
f936143); this proves the fix holds end-to-end against real indexed
|
||||
documents and real timezone-sensitive matching, not just that the
|
||||
library's date_from() parses the text."""
|
||||
|
||||
def test_t_z_range_matches_only_documents_within_bounds(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
) -> None:
|
||||
in_range = Document.objects.create(
|
||||
title="In range",
|
||||
content="x",
|
||||
checksum="acc-rfc3339-in-range",
|
||||
added=datetime(2026, 3, 15, 10, 0, tzinfo=UTC),
|
||||
)
|
||||
out_of_range = Document.objects.create(
|
||||
title="Out of range",
|
||||
content="x",
|
||||
checksum="acc-rfc3339-out-of-range",
|
||||
added=datetime(2026, 8, 1, 10, 0, tzinfo=UTC),
|
||||
)
|
||||
for doc in (in_range, out_of_range):
|
||||
backend.add_or_update(doc)
|
||||
matched = _matched_ids(
|
||||
backend,
|
||||
"added:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z]",
|
||||
)
|
||||
assert matched == {in_range.pk}
|
||||
|
||||
def test_comma_combined_t_z_ranges_across_two_fields(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
) -> None:
|
||||
# The Whoosh v2 comma syntax: two field:[range] expressions joined
|
||||
# by a comma must be ANDed together (not passed to Tantivy as a
|
||||
# literal comma, which it cannot parse).
|
||||
matching = Document.objects.create(
|
||||
title="Matches both ranges",
|
||||
content="x",
|
||||
checksum="acc-rfc3339-comma-match",
|
||||
created=date(2026, 3, 15),
|
||||
added=datetime(2026, 5, 15, 10, 0, tzinfo=UTC),
|
||||
)
|
||||
wrong_added = Document.objects.create(
|
||||
title="created in range, added out of range",
|
||||
content="x",
|
||||
checksum="acc-rfc3339-comma-wrong-added",
|
||||
created=date(2026, 3, 15),
|
||||
added=datetime(2026, 8, 1, 10, 0, tzinfo=UTC),
|
||||
)
|
||||
for doc in (matching, wrong_added):
|
||||
backend.add_or_update(doc)
|
||||
matched = _matched_ids(
|
||||
backend,
|
||||
"created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z],"
|
||||
"added:[2026-05-01T00:00:00Z TO 2026-06-01T00:00:00Z]",
|
||||
)
|
||||
assert matched == {matching.pk}
|
||||
|
||||
def test_z_suffixed_bound_is_absolute_utc_not_local_shifted(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
) -> None:
|
||||
# A document timestamped exactly at a Z-suffixed range boundary must
|
||||
# match under UTC - if Z were (incorrectly) reinterpreted as local
|
||||
# wall-clock time and shifted again, this exact-boundary match would
|
||||
# silently fail or succeed for the wrong reason.
|
||||
at_boundary = Document.objects.create(
|
||||
title="At Z boundary",
|
||||
content="x",
|
||||
checksum="acc-rfc3339-z-boundary",
|
||||
added=datetime(2026, 1, 1, 0, 0, 0, tzinfo=UTC),
|
||||
)
|
||||
backend.add_or_update(at_boundary)
|
||||
matched = _matched_ids(
|
||||
backend,
|
||||
"added:[2026-01-01T00:00:00Z TO 2026-01-01T00:00:01Z]",
|
||||
)
|
||||
assert matched == {at_boundary.pk}
|
||||
|
||||
|
||||
class TestUnregisteredIdFieldFoldsToLiteralText:
|
||||
"""tag_id, owner_id, etc. are intentionally excluded from the
|
||||
FieldRegistry: they were always internal index columns (v2 consumed
|
||||
them for permission filtering and its own criteria), and their
|
||||
queryability as search syntax was an accident of whoosh resolving any
|
||||
schema field name. whoosh-compat parity leniency folds them into a
|
||||
literal text search rather than raising a diagnostic/400. Prove the
|
||||
fold is inert against real data, not just that parsing doesn't
|
||||
raise."""
|
||||
FieldRegistry - always internal index columns, never meant to be
|
||||
query-addressable. Prove an unregistered field folds to a literal
|
||||
text search that matches nothing, rather than erroring."""
|
||||
|
||||
def test_tag_id_query_matches_nothing(
|
||||
self,
|
||||
@@ -413,15 +260,11 @@ class TestFuzzyBlendSurvivesWhooshGrammar:
|
||||
|
||||
|
||||
class TestUnquotedDateKeywordPhrases:
|
||||
"""The unquoted multi-word date keyword spelling (added:previous month)
|
||||
has been honored continuously since the whoosh era, always by an
|
||||
app-level assist, never by any parser: v2 rewrote it to a bracket range
|
||||
before whoosh saw it, and the deleted _translate.py consumed it itself.
|
||||
whoosh-compat scopes the unquoted form out of its parser on purpose
|
||||
(its DIVERGENCES.md entry 19) but understands the quoted form natively,
|
||||
so paperless quotes the closed phrase vocabulary on date fields before
|
||||
parsing. Only quoting happens app-side; every date computation stays in
|
||||
whoosh-compat."""
|
||||
"""The unquoted spelling (added:previous month) has always been
|
||||
honored via an app-level quoting assist, since whoosh-compat's parser
|
||||
only accepts the quoted form natively. paperless quotes the closed
|
||||
phrase vocabulary on date fields before parsing; every date
|
||||
computation still happens in whoosh-compat."""
|
||||
|
||||
@pytest.fixture
|
||||
def period_documents(self, backend: TantivyBackend) -> dict[str, int]:
|
||||
@@ -505,15 +348,10 @@ class TestUnquotedDateKeywordPhrases:
|
||||
|
||||
|
||||
class TestBareJsonFieldPrefixes:
|
||||
"""The v2 whoosh schema had plural notes/custom_fields TEXT fields, so
|
||||
"notes:foo" and "custom_fields:foo" were valid fielded searches in
|
||||
released paperless (and at the tantivy translation layer). On the
|
||||
whoosh-compat registry they are JSON fields addressable only via
|
||||
subpaths, and the bare spelling would demote to a nonsense unfielded
|
||||
text search. parse_user_query rewrites the bare prefixes live to the
|
||||
same targets migration 0017 chose for the singular whoosh-era
|
||||
spellings: notes: -> notes.note:, custom_fields: ->
|
||||
custom_fields.value:."""
|
||||
""" "notes:foo"/"custom_fields:foo" were valid fielded searches before
|
||||
this migration. whoosh-compat's registry only exposes them as JSON
|
||||
subpaths, so parse_user_query rewrites the bare prefixes live: notes:
|
||||
-> notes.note:, custom_fields: -> custom_fields.value:."""
|
||||
|
||||
def test_bare_notes_prefix_searches_note_text(
|
||||
self,
|
||||
@@ -644,53 +482,3 @@ class TestFieldAliases:
|
||||
backend.add_or_update(loose)
|
||||
assert _matched_ids(backend, "path:archive") == {stored.pk}
|
||||
assert _matched_ids(backend, "storage_path:archive") == {stored.pk}
|
||||
|
||||
|
||||
class TestCreatedTimezoneInvariance:
|
||||
def test_created_date_matches_regardless_of_active_timezone(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
) -> None:
|
||||
# "created" is a date-only field indexed at naive midnight: a
|
||||
# document created 2020-06-10 must match created:20200610 whether
|
||||
# the active timezone is far ahead of or behind UTC. (The
|
||||
# timezone-SENSITIVE datetime fields have their own boundary
|
||||
# coverage in test_api_search.py's tz-ahead/tz-behind tests.)
|
||||
from django.utils import timezone as django_tz
|
||||
|
||||
doc = Document.objects.create(
|
||||
title="Dated",
|
||||
content="x",
|
||||
checksum="tz-inv-1",
|
||||
created=date(2020, 6, 10),
|
||||
)
|
||||
backend.add_or_update(doc)
|
||||
for tzname in ("Pacific/Auckland", "America/New_York"):
|
||||
with django_tz.override(tzname):
|
||||
assert _matched_ids(backend, "created:20200610") == {doc.pk}, tzname
|
||||
|
||||
|
||||
class TestReversedDateRange:
|
||||
def test_reversed_bounds_still_match_the_span(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
) -> None:
|
||||
# whoosh's joint disambiguation swaps backwards bounds (both years
|
||||
# explicit -> plain swap), and whoosh-compat reproduces it; a saved
|
||||
# view with created:[2025 TO 2020] must keep matching the span
|
||||
# instead of becoming an empty lo>hi range.
|
||||
inside = Document.objects.create(
|
||||
title="Inside",
|
||||
content="x",
|
||||
checksum="rev-range-1",
|
||||
created=date(2022, 5, 1),
|
||||
)
|
||||
outside = Document.objects.create(
|
||||
title="Outside",
|
||||
content="x",
|
||||
checksum="rev-range-2",
|
||||
created=date(2019, 5, 1),
|
||||
)
|
||||
backend.add_or_update(inside)
|
||||
backend.add_or_update(outside)
|
||||
assert _matched_ids(backend, "created:[2025 TO 2020]") == {inside.pk}
|
||||
|
||||
@@ -328,12 +328,10 @@ class TestSearchQueryErrors:
|
||||
|
||||
|
||||
class TestEmitErrorContract:
|
||||
"""whoosh-compat's emit() documents a two-part host contract: BOTH a
|
||||
non-empty diagnostics list AND the QueryEmitError/UnsupportedQueryError
|
||||
pair raised by emit() itself are user-input errors. Every one must
|
||||
surface as SearchQueryError (HTTP 400), with library-internal
|
||||
vocabulary (DIVERGENCES.md references, fast=True host advice) kept out
|
||||
of the user-facing message."""
|
||||
"""A diagnostics list, or a QueryEmitError/UnsupportedQueryError from
|
||||
emit(), are both user-input errors and must surface as
|
||||
SearchQueryError (HTTP 400), with library-internal wording stripped
|
||||
from the message."""
|
||||
|
||||
@pytest.fixture
|
||||
def query_index(self) -> tantivy.Index:
|
||||
|
||||
Reference in New Issue
Block a user