amorfo77/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2026-08-19 09:13:24 +00:00
Code Issues Projects Releases Wiki Activity

test(search): trim acceptance tests that pin whoosh-compat behavior, not ours

Browse Source 
Deletes TestCommaValueLists, TestMultitokenInNestedOr, TestRfc3339TZDateRange,
TestCreatedTimezoneInvariance, and TestReversedDateRange: none of them
exercise any paperless-specific pre/post-processing code. Comma-list AND
semantics, multitoken resolution, RFC3339 T/Z UTC math, date-only timezone
invariance, and reversed-range disambiguation are all entirely
whoosh-compat's own grammar/semantics, already covered by its own test
suite. The comma_values flag paperless does own is still covered cheaply in
test_fields.py; the date_only flag is still covered in test_registry.py.

Also trims verbose docstrings/comments across _query.py and the surviving
acceptance tests: cuts references to whoosh-compat's internal
DIVERGENCES.md entry numbers and paperless v2/Whoosh-era implementation
history down to the user-facing behavior that actually matters, without
losing the substance.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01RVj8NFy821G3YhNf68PF6X
This commit is contained in:
Trenton Holmes
2026-08-18 11:24:14 -07:00
co-authored by Claude Sonnet 5
parent a37bd5bec4
commit 194c2bce48
3 changed files with 32 additions and 262 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/documents/search/_query.py
+15 -31
View File
@@ -84,15 +84,10 @@ _REGEX_TIMEOUT: Final[float] = 1.0
# Uses Unicode properties to cover all blocks including Extension B+ planes.
_CJK_RE: Final = regex.compile(r"[\p{Han}\p{Hiragana}\p{Katakana}\p{Hangul}]+")
# The closed multi-word date-keyword vocabulary, unchanged since paperless
# v2's rewrite_natural_date_keywords. whoosh-compat's date grammar
# understands every one of these natively, but only as a QUOTED value
# (its DIVERGENCES.md entry 19: unquoted multi-word values split at
# whitespace, faithfully to whoosh); the unquoted spelling has been
# honored continuously since the whoosh era by an app-level assist, so
# _quote_date_keyword_phrases below keeps honoring it by inserting the
# quotes and nothing else. Single-word keywords (today, yesterday) parse
# unquoted already and need no entry.
# Multi-word date-keyword phrases whoosh-compat only accepts quoted.
# Unquoted has always been the honored spelling, so
# _quote_date_keyword_phrases below inserts the quotes and nothing else.
# Single-word keywords (today, yesterday) already parse unquoted.
_DATE_KEYWORD_PHRASES: Final = (
"previous week",
"previous month",
@@ -122,14 +117,10 @@ _DATE_KEYWORD_PHRASE_RE: Final = regex.compile(
def _quote_date_keyword_phrases(raw_query: str) -> str:
"""Quote unquoted multi-word date keyword phrases on date fields.
``added:previous month`` becomes ``added:"previous month"``; the
already-quoted spellings don't match the pattern (the colon must be
followed directly by the phrase), and the same words after a TEXT
field or standing alone are ordinary text and untouched. Only quoting
happens here: every date computation stays in whoosh-compat's
grammar, which parses exactly this phrase vocabulary as quoted
values. This is deliberately NOT a revival of the deleted
translation layer, which computed the ranges app-side.
``added:previous month`` becomes ``added:"previous month"``; already-
quoted spellings, TEXT fields, and standalone words are untouched.
Only quoting happens here - every date computation stays in
whoosh-compat's grammar.
"""
return _DATE_KEYWORD_PHRASE_RE.sub(
r'\1:"\2"',
@@ -138,20 +129,13 @@ def _quote_date_keyword_phrases(raw_query: str) -> str:
)
# The v2 whoosh schema had plural notes/custom_fields TEXT fields (notes
# indexed the joined note texts; custom_fields indexed joined
# "name : value" strings), so the bare plural prefixes were valid fielded
# searches in released paperless and at the deleted translation layer. On
# the whoosh-compat registry they are JSON fields addressable only via
# subpaths, and the bare spelling would demote to an unfielded text search
# of the words themselves. Rewrite the prefixes live to the same targets
# migration 0017 chose for the singular whoosh-era spellings (note: ->
# notes.note:, custom_field: -> custom_fields.value:), values untouched.
# Trade-off inherited from that migration: custom_fields.value: drops the
# name-matching half of v2's "name : value" indexing (custom_fields.name:
# remains available for it). Same lookbehind guard as 0017: not preceded
# by a word character or dot, so subpath spellings and words that merely
# end in the prefix are untouched.
# notes:/custom_fields: were valid fielded searches before this migration.
# whoosh-compat's registry only exposes them as JSON subpaths, so a bare
# prefix would demote to an unfielded text search. Rewrite live to the
# equivalent subpath (notes: -> notes.note:, custom_fields: ->
# custom_fields.value:); custom_fields.name: remains available separately.
# Not preceded by a word character or dot, so subpath spellings and words
# merely ending in the prefix are untouched.
_BARE_JSON_PREFIX_RES: Final = (
(regex.compile(r"(?<![.\w])notes:(?!\.)"), "notes.note:"),
(regex.compile(r"(?<![.\w])custom_fields:(?!\.)"), "custom_fields.value:"),
src/documents/tests/search/test_acceptance.py
+13 -225
View File
@@ -9,7 +9,6 @@ Supersedes test_query.py's TestParseUserQuery result-level cases.
from __future__ import annotations
from datetime import UTC
from datetime import date
from datetime import datetime
from typing import TYPE_CHECKING
@@ -20,7 +19,6 @@ from documents.models import CustomField
from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import Note
from documents.models import Tag
from documents.search._query import parse_user_query
if TYPE_CHECKING:
@@ -71,8 +69,7 @@ def indexed_documents(backend: TantivyBackend) -> dict[str, int]:
class TestIssue13568BracketWildcard:
"""paperless-ngx#13568: title:202[0-3]* must keep its character class,
not fold to a prefix query that silently drops it (whoosh-compat
DIVERGENCES.md entry 13)."""
not fold to a prefix query that silently drops it."""
def test_bracket_class_wildcard_matches_only_in_range_years(
self,
@@ -97,45 +94,6 @@ class TestIssue13568BracketWildcard:
)
class TestCommaValueLists:
"""whoosh-compat's CommaValuesPlugin splits `tag:foo,bar` into
`tag:foo AND tag:bar` (DIVERGENCES.md entries 17/36), matching real
Whoosh's KEYWORD(commas=True) analyzer-time comma splitting - not an OR
across the listed values. A document must carry every listed tag to
match."""
def test_tag_comma_list_matches_only_documents_with_both_tags(
self,
backend: TantivyBackend,
) -> None:
tag_foo = Tag.objects.create(name="foo")
tag_bar = Tag.objects.create(name="bar")
tag_baz = Tag.objects.create(name="baz")
doc_both = Document.objects.create(
title="Both",
content="x",
checksum="acc-comma-both",
)
doc_both.tags.add(tag_foo, tag_bar)
doc_foo_only = Document.objects.create(
title="FooOnly",
content="x",
checksum="acc-comma-foo",
)
doc_foo_only.tags.add(tag_foo)
doc_other = Document.objects.create(
title="Other",
content="x",
checksum="acc-comma-other",
)
doc_other.tags.add(tag_baz)
for doc in (doc_both, doc_foo_only, doc_other):
backend.add_or_update(doc)
matched = _matched_ids(backend, "tag:foo,bar")
assert matched == {doc_both.pk}
class TestFieldBoosts:
def test_title_boost_ranks_title_match_above_content_only_match(
self,
@@ -227,122 +185,11 @@ class TestJsonSubpaths:
assert matched == {matching.pk}
class TestMultitokenInNestedOr:
"""whoosh-compat DIVERGENCES.md entry 15: Multitoken.DEFAULT resolves by
syntactic enclosing group, not the parser's fixed default group. Prove
it doesn't matter for paperless's actual data/fields."""
def test_multitoken_tag_value_inside_top_level_or_matches_either_branch(
self,
backend: TantivyBackend,
) -> None:
# "multi word tag" is a multitoken field value; nested inside a
# top-level OR with an unrelated clause.
doc_a = Document.objects.create(title="A", content="x", checksum="acc-mt-a")
doc_a.tags.create(name="multi word tag")
doc_b = Document.objects.create(title="B", content="x", checksum="acc-mt-b")
doc_b.tags.create(name="unrelated")
backend.add_or_update(doc_a)
backend.add_or_update(doc_b)
matched = _matched_ids(backend, 'tag:"multi word tag" OR title:B')
assert matched == {doc_a.pk, doc_b.pk}
class TestRfc3339TZDateRange:
"""paperless-ngx#13010: created/added bracket ranges using RFC3339 T/Z
datetime separators (e.g. `[2026-01-01T00:00:00Z TO ...]`) must keep
working - this is v2/Whoosh saved-search backward compatibility, not a
generic ISO-format nicety. whoosh-compat's own grammar previously had no
support for `T`/`Z` at all (fixed upstream, whoosh-compat commit
f936143); this proves the fix holds end-to-end against real indexed
documents and real timezone-sensitive matching, not just that the
library's date_from() parses the text."""
def test_t_z_range_matches_only_documents_within_bounds(
self,
backend: TantivyBackend,
) -> None:
in_range = Document.objects.create(
title="In range",
content="x",
checksum="acc-rfc3339-in-range",
added=datetime(2026, 3, 15, 10, 0, tzinfo=UTC),
)
out_of_range = Document.objects.create(
title="Out of range",
content="x",
checksum="acc-rfc3339-out-of-range",
added=datetime(2026, 8, 1, 10, 0, tzinfo=UTC),
)
for doc in (in_range, out_of_range):
backend.add_or_update(doc)
matched = _matched_ids(
backend,
"added:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z]",
)
assert matched == {in_range.pk}
def test_comma_combined_t_z_ranges_across_two_fields(
self,
backend: TantivyBackend,
) -> None:
# The Whoosh v2 comma syntax: two field:[range] expressions joined
# by a comma must be ANDed together (not passed to Tantivy as a
# literal comma, which it cannot parse).
matching = Document.objects.create(
title="Matches both ranges",
content="x",
checksum="acc-rfc3339-comma-match",
created=date(2026, 3, 15),
added=datetime(2026, 5, 15, 10, 0, tzinfo=UTC),
)
wrong_added = Document.objects.create(
title="created in range, added out of range",
content="x",
checksum="acc-rfc3339-comma-wrong-added",
created=date(2026, 3, 15),
added=datetime(2026, 8, 1, 10, 0, tzinfo=UTC),
)
for doc in (matching, wrong_added):
backend.add_or_update(doc)
matched = _matched_ids(
backend,
"created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z],"
"added:[2026-05-01T00:00:00Z TO 2026-06-01T00:00:00Z]",
)
assert matched == {matching.pk}
def test_z_suffixed_bound_is_absolute_utc_not_local_shifted(
self,
backend: TantivyBackend,
) -> None:
# A document timestamped exactly at a Z-suffixed range boundary must
# match under UTC - if Z were (incorrectly) reinterpreted as local
# wall-clock time and shifted again, this exact-boundary match would
# silently fail or succeed for the wrong reason.
at_boundary = Document.objects.create(
title="At Z boundary",
content="x",
checksum="acc-rfc3339-z-boundary",
added=datetime(2026, 1, 1, 0, 0, 0, tzinfo=UTC),
)
backend.add_or_update(at_boundary)
matched = _matched_ids(
backend,
"added:[2026-01-01T00:00:00Z TO 2026-01-01T00:00:01Z]",
)
assert matched == {at_boundary.pk}
class TestUnregisteredIdFieldFoldsToLiteralText:
"""tag_id, owner_id, etc. are intentionally excluded from the
FieldRegistry: they were always internal index columns (v2 consumed
them for permission filtering and its own criteria), and their
queryability as search syntax was an accident of whoosh resolving any
schema field name. whoosh-compat parity leniency folds them into a
literal text search rather than raising a diagnostic/400. Prove the
fold is inert against real data, not just that parsing doesn't
raise."""
FieldRegistry - always internal index columns, never meant to be
query-addressable. Prove an unregistered field folds to a literal
text search that matches nothing, rather than erroring."""
def test_tag_id_query_matches_nothing(
self,
@@ -413,15 +260,11 @@ class TestFuzzyBlendSurvivesWhooshGrammar:
class TestUnquotedDateKeywordPhrases:
"""The unquoted multi-word date keyword spelling (added:previous month)
has been honored continuously since the whoosh era, always by an
app-level assist, never by any parser: v2 rewrote it to a bracket range
before whoosh saw it, and the deleted _translate.py consumed it itself.
whoosh-compat scopes the unquoted form out of its parser on purpose
(its DIVERGENCES.md entry 19) but understands the quoted form natively,
so paperless quotes the closed phrase vocabulary on date fields before
parsing. Only quoting happens app-side; every date computation stays in
whoosh-compat."""
"""The unquoted spelling (added:previous month) has always been
honored via an app-level quoting assist, since whoosh-compat's parser
only accepts the quoted form natively. paperless quotes the closed
phrase vocabulary on date fields before parsing; every date
computation still happens in whoosh-compat."""
@pytest.fixture
def period_documents(self, backend: TantivyBackend) -> dict[str, int]:
@@ -505,15 +348,10 @@ class TestUnquotedDateKeywordPhrases:
class TestBareJsonFieldPrefixes:
"""The v2 whoosh schema had plural notes/custom_fields TEXT fields, so
"notes:foo" and "custom_fields:foo" were valid fielded searches in
released paperless (and at the tantivy translation layer). On the
whoosh-compat registry they are JSON fields addressable only via
subpaths, and the bare spelling would demote to a nonsense unfielded
text search. parse_user_query rewrites the bare prefixes live to the
same targets migration 0017 chose for the singular whoosh-era
spellings: notes: -> notes.note:, custom_fields: ->
custom_fields.value:."""
""" "notes:foo"/"custom_fields:foo" were valid fielded searches before
this migration. whoosh-compat's registry only exposes them as JSON
subpaths, so parse_user_query rewrites the bare prefixes live: notes:
-> notes.note:, custom_fields: -> custom_fields.value:."""
def test_bare_notes_prefix_searches_note_text(
self,
@@ -644,53 +482,3 @@ class TestFieldAliases:
backend.add_or_update(loose)
assert _matched_ids(backend, "path:archive") == {stored.pk}
assert _matched_ids(backend, "storage_path:archive") == {stored.pk}
class TestCreatedTimezoneInvariance:
def test_created_date_matches_regardless_of_active_timezone(
self,
backend: TantivyBackend,
) -> None:
# "created" is a date-only field indexed at naive midnight: a
# document created 2020-06-10 must match created:20200610 whether
# the active timezone is far ahead of or behind UTC. (The
# timezone-SENSITIVE datetime fields have their own boundary
# coverage in test_api_search.py's tz-ahead/tz-behind tests.)
from django.utils import timezone as django_tz
doc = Document.objects.create(
title="Dated",
content="x",
checksum="tz-inv-1",
created=date(2020, 6, 10),
)
backend.add_or_update(doc)
for tzname in ("Pacific/Auckland", "America/New_York"):
with django_tz.override(tzname):
assert _matched_ids(backend, "created:20200610") == {doc.pk}, tzname
class TestReversedDateRange:
def test_reversed_bounds_still_match_the_span(
self,
backend: TantivyBackend,
) -> None:
# whoosh's joint disambiguation swaps backwards bounds (both years
# explicit -> plain swap), and whoosh-compat reproduces it; a saved
# view with created:[2025 TO 2020] must keep matching the span
# instead of becoming an empty lo>hi range.
inside = Document.objects.create(
title="Inside",
content="x",
checksum="rev-range-1",
created=date(2022, 5, 1),
)
outside = Document.objects.create(
title="Outside",
content="x",
checksum="rev-range-2",
created=date(2019, 5, 1),
)
backend.add_or_update(inside)
backend.add_or_update(outside)
assert _matched_ids(backend, "created:[2025 TO 2020]") == {inside.pk}
src/documents/tests/search/test_query.py
+4 -6
View File
@@ -328,12 +328,10 @@ class TestSearchQueryErrors:
class TestEmitErrorContract:
"""whoosh-compat's emit() documents a two-part host contract: BOTH a
non-empty diagnostics list AND the QueryEmitError/UnsupportedQueryError
pair raised by emit() itself are user-input errors. Every one must
surface as SearchQueryError (HTTP 400), with library-internal
vocabulary (DIVERGENCES.md references, fast=True host advice) kept out
of the user-facing message."""
"""A diagnostics list, or a QueryEmitError/UnsupportedQueryError from
emit(), are both user-input errors and must surface as
SearchQueryError (HTTP 400), with library-internal wording stripped
from the message."""
@pytest.fixture
def query_index(self) -> tantivy.Index: