From 194c2bce48a29b7f0438edeee149cb05b8a13512 Mon Sep 17 00:00:00 2001 From: Trenton Holmes <797416+stumpylog@users.noreply.github.com> Date: Tue, 18 Aug 2026 11:24:14 -0700 Subject: [PATCH] test(search): trim acceptance tests that pin whoosh-compat behavior, not ours Deletes TestCommaValueLists, TestMultitokenInNestedOr, TestRfc3339TZDateRange, TestCreatedTimezoneInvariance, and TestReversedDateRange: none of them exercise any paperless-specific pre/post-processing code. Comma-list AND semantics, multitoken resolution, RFC3339 T/Z UTC math, date-only timezone invariance, and reversed-range disambiguation are all entirely whoosh-compat's own grammar/semantics, already covered by its own test suite. The comma_values flag paperless does own is still covered cheaply in test_fields.py; the date_only flag is still covered in test_registry.py. Also trims verbose docstrings/comments across _query.py and the surviving acceptance tests: cuts references to whoosh-compat's internal DIVERGENCES.md entry numbers and paperless v2/Whoosh-era implementation history down to the user-facing behavior that actually matters, without losing the substance. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01RVj8NFy821G3YhNf68PF6X --- src/documents/search/_query.py | 46 ++-- src/documents/tests/search/test_acceptance.py | 238 +----------------- src/documents/tests/search/test_query.py | 10 +- 3 files changed, 32 insertions(+), 262 deletions(-) diff --git a/src/documents/search/_query.py b/src/documents/search/_query.py index b5acfc37c..521159596 100644 --- a/src/documents/search/_query.py +++ b/src/documents/search/_query.py @@ -84,15 +84,10 @@ _REGEX_TIMEOUT: Final[float] = 1.0 # Uses Unicode properties to cover all blocks including Extension B+ planes. _CJK_RE: Final = regex.compile(r"[\p{Han}\p{Hiragana}\p{Katakana}\p{Hangul}]+") -# The closed multi-word date-keyword vocabulary, unchanged since paperless -# v2's rewrite_natural_date_keywords. whoosh-compat's date grammar -# understands every one of these natively, but only as a QUOTED value -# (its DIVERGENCES.md entry 19: unquoted multi-word values split at -# whitespace, faithfully to whoosh); the unquoted spelling has been -# honored continuously since the whoosh era by an app-level assist, so -# _quote_date_keyword_phrases below keeps honoring it by inserting the -# quotes and nothing else. Single-word keywords (today, yesterday) parse -# unquoted already and need no entry. +# Multi-word date-keyword phrases whoosh-compat only accepts quoted. +# Unquoted has always been the honored spelling, so +# _quote_date_keyword_phrases below inserts the quotes and nothing else. +# Single-word keywords (today, yesterday) already parse unquoted. _DATE_KEYWORD_PHRASES: Final = ( "previous week", "previous month", @@ -122,14 +117,10 @@ _DATE_KEYWORD_PHRASE_RE: Final = regex.compile( def _quote_date_keyword_phrases(raw_query: str) -> str: """Quote unquoted multi-word date keyword phrases on date fields. - ``added:previous month`` becomes ``added:"previous month"``; the - already-quoted spellings don't match the pattern (the colon must be - followed directly by the phrase), and the same words after a TEXT - field or standing alone are ordinary text and untouched. Only quoting - happens here: every date computation stays in whoosh-compat's - grammar, which parses exactly this phrase vocabulary as quoted - values. This is deliberately NOT a revival of the deleted - translation layer, which computed the ranges app-side. + ``added:previous month`` becomes ``added:"previous month"``; already- + quoted spellings, TEXT fields, and standalone words are untouched. + Only quoting happens here - every date computation stays in + whoosh-compat's grammar. """ return _DATE_KEYWORD_PHRASE_RE.sub( r'\1:"\2"', @@ -138,20 +129,13 @@ def _quote_date_keyword_phrases(raw_query: str) -> str: ) -# The v2 whoosh schema had plural notes/custom_fields TEXT fields (notes -# indexed the joined note texts; custom_fields indexed joined -# "name : value" strings), so the bare plural prefixes were valid fielded -# searches in released paperless and at the deleted translation layer. On -# the whoosh-compat registry they are JSON fields addressable only via -# subpaths, and the bare spelling would demote to an unfielded text search -# of the words themselves. Rewrite the prefixes live to the same targets -# migration 0017 chose for the singular whoosh-era spellings (note: -> -# notes.note:, custom_field: -> custom_fields.value:), values untouched. -# Trade-off inherited from that migration: custom_fields.value: drops the -# name-matching half of v2's "name : value" indexing (custom_fields.name: -# remains available for it). Same lookbehind guard as 0017: not preceded -# by a word character or dot, so subpath spellings and words that merely -# end in the prefix are untouched. +# notes:/custom_fields: were valid fielded searches before this migration. +# whoosh-compat's registry only exposes them as JSON subpaths, so a bare +# prefix would demote to an unfielded text search. Rewrite live to the +# equivalent subpath (notes: -> notes.note:, custom_fields: -> +# custom_fields.value:); custom_fields.name: remains available separately. +# Not preceded by a word character or dot, so subpath spellings and words +# merely ending in the prefix are untouched. _BARE_JSON_PREFIX_RES: Final = ( (regex.compile(r"(? dict[str, int]: class TestIssue13568BracketWildcard: """paperless-ngx#13568: title:202[0-3]* must keep its character class, - not fold to a prefix query that silently drops it (whoosh-compat - DIVERGENCES.md entry 13).""" + not fold to a prefix query that silently drops it.""" def test_bracket_class_wildcard_matches_only_in_range_years( self, @@ -97,45 +94,6 @@ class TestIssue13568BracketWildcard: ) -class TestCommaValueLists: - """whoosh-compat's CommaValuesPlugin splits `tag:foo,bar` into - `tag:foo AND tag:bar` (DIVERGENCES.md entries 17/36), matching real - Whoosh's KEYWORD(commas=True) analyzer-time comma splitting - not an OR - across the listed values. A document must carry every listed tag to - match.""" - - def test_tag_comma_list_matches_only_documents_with_both_tags( - self, - backend: TantivyBackend, - ) -> None: - tag_foo = Tag.objects.create(name="foo") - tag_bar = Tag.objects.create(name="bar") - tag_baz = Tag.objects.create(name="baz") - - doc_both = Document.objects.create( - title="Both", - content="x", - checksum="acc-comma-both", - ) - doc_both.tags.add(tag_foo, tag_bar) - doc_foo_only = Document.objects.create( - title="FooOnly", - content="x", - checksum="acc-comma-foo", - ) - doc_foo_only.tags.add(tag_foo) - doc_other = Document.objects.create( - title="Other", - content="x", - checksum="acc-comma-other", - ) - doc_other.tags.add(tag_baz) - for doc in (doc_both, doc_foo_only, doc_other): - backend.add_or_update(doc) - matched = _matched_ids(backend, "tag:foo,bar") - assert matched == {doc_both.pk} - - class TestFieldBoosts: def test_title_boost_ranks_title_match_above_content_only_match( self, @@ -227,122 +185,11 @@ class TestJsonSubpaths: assert matched == {matching.pk} -class TestMultitokenInNestedOr: - """whoosh-compat DIVERGENCES.md entry 15: Multitoken.DEFAULT resolves by - syntactic enclosing group, not the parser's fixed default group. Prove - it doesn't matter for paperless's actual data/fields.""" - - def test_multitoken_tag_value_inside_top_level_or_matches_either_branch( - self, - backend: TantivyBackend, - ) -> None: - # "multi word tag" is a multitoken field value; nested inside a - # top-level OR with an unrelated clause. - doc_a = Document.objects.create(title="A", content="x", checksum="acc-mt-a") - doc_a.tags.create(name="multi word tag") - doc_b = Document.objects.create(title="B", content="x", checksum="acc-mt-b") - doc_b.tags.create(name="unrelated") - backend.add_or_update(doc_a) - backend.add_or_update(doc_b) - matched = _matched_ids(backend, 'tag:"multi word tag" OR title:B') - assert matched == {doc_a.pk, doc_b.pk} - - -class TestRfc3339TZDateRange: - """paperless-ngx#13010: created/added bracket ranges using RFC3339 T/Z - datetime separators (e.g. `[2026-01-01T00:00:00Z TO ...]`) must keep - working - this is v2/Whoosh saved-search backward compatibility, not a - generic ISO-format nicety. whoosh-compat's own grammar previously had no - support for `T`/`Z` at all (fixed upstream, whoosh-compat commit - f936143); this proves the fix holds end-to-end against real indexed - documents and real timezone-sensitive matching, not just that the - library's date_from() parses the text.""" - - def test_t_z_range_matches_only_documents_within_bounds( - self, - backend: TantivyBackend, - ) -> None: - in_range = Document.objects.create( - title="In range", - content="x", - checksum="acc-rfc3339-in-range", - added=datetime(2026, 3, 15, 10, 0, tzinfo=UTC), - ) - out_of_range = Document.objects.create( - title="Out of range", - content="x", - checksum="acc-rfc3339-out-of-range", - added=datetime(2026, 8, 1, 10, 0, tzinfo=UTC), - ) - for doc in (in_range, out_of_range): - backend.add_or_update(doc) - matched = _matched_ids( - backend, - "added:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z]", - ) - assert matched == {in_range.pk} - - def test_comma_combined_t_z_ranges_across_two_fields( - self, - backend: TantivyBackend, - ) -> None: - # The Whoosh v2 comma syntax: two field:[range] expressions joined - # by a comma must be ANDed together (not passed to Tantivy as a - # literal comma, which it cannot parse). - matching = Document.objects.create( - title="Matches both ranges", - content="x", - checksum="acc-rfc3339-comma-match", - created=date(2026, 3, 15), - added=datetime(2026, 5, 15, 10, 0, tzinfo=UTC), - ) - wrong_added = Document.objects.create( - title="created in range, added out of range", - content="x", - checksum="acc-rfc3339-comma-wrong-added", - created=date(2026, 3, 15), - added=datetime(2026, 8, 1, 10, 0, tzinfo=UTC), - ) - for doc in (matching, wrong_added): - backend.add_or_update(doc) - matched = _matched_ids( - backend, - "created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z]," - "added:[2026-05-01T00:00:00Z TO 2026-06-01T00:00:00Z]", - ) - assert matched == {matching.pk} - - def test_z_suffixed_bound_is_absolute_utc_not_local_shifted( - self, - backend: TantivyBackend, - ) -> None: - # A document timestamped exactly at a Z-suffixed range boundary must - # match under UTC - if Z were (incorrectly) reinterpreted as local - # wall-clock time and shifted again, this exact-boundary match would - # silently fail or succeed for the wrong reason. - at_boundary = Document.objects.create( - title="At Z boundary", - content="x", - checksum="acc-rfc3339-z-boundary", - added=datetime(2026, 1, 1, 0, 0, 0, tzinfo=UTC), - ) - backend.add_or_update(at_boundary) - matched = _matched_ids( - backend, - "added:[2026-01-01T00:00:00Z TO 2026-01-01T00:00:01Z]", - ) - assert matched == {at_boundary.pk} - - class TestUnregisteredIdFieldFoldsToLiteralText: """tag_id, owner_id, etc. are intentionally excluded from the - FieldRegistry: they were always internal index columns (v2 consumed - them for permission filtering and its own criteria), and their - queryability as search syntax was an accident of whoosh resolving any - schema field name. whoosh-compat parity leniency folds them into a - literal text search rather than raising a diagnostic/400. Prove the - fold is inert against real data, not just that parsing doesn't - raise.""" + FieldRegistry - always internal index columns, never meant to be + query-addressable. Prove an unregistered field folds to a literal + text search that matches nothing, rather than erroring.""" def test_tag_id_query_matches_nothing( self, @@ -413,15 +260,11 @@ class TestFuzzyBlendSurvivesWhooshGrammar: class TestUnquotedDateKeywordPhrases: - """The unquoted multi-word date keyword spelling (added:previous month) - has been honored continuously since the whoosh era, always by an - app-level assist, never by any parser: v2 rewrote it to a bracket range - before whoosh saw it, and the deleted _translate.py consumed it itself. - whoosh-compat scopes the unquoted form out of its parser on purpose - (its DIVERGENCES.md entry 19) but understands the quoted form natively, - so paperless quotes the closed phrase vocabulary on date fields before - parsing. Only quoting happens app-side; every date computation stays in - whoosh-compat.""" + """The unquoted spelling (added:previous month) has always been + honored via an app-level quoting assist, since whoosh-compat's parser + only accepts the quoted form natively. paperless quotes the closed + phrase vocabulary on date fields before parsing; every date + computation still happens in whoosh-compat.""" @pytest.fixture def period_documents(self, backend: TantivyBackend) -> dict[str, int]: @@ -505,15 +348,10 @@ class TestUnquotedDateKeywordPhrases: class TestBareJsonFieldPrefixes: - """The v2 whoosh schema had plural notes/custom_fields TEXT fields, so - "notes:foo" and "custom_fields:foo" were valid fielded searches in - released paperless (and at the tantivy translation layer). On the - whoosh-compat registry they are JSON fields addressable only via - subpaths, and the bare spelling would demote to a nonsense unfielded - text search. parse_user_query rewrites the bare prefixes live to the - same targets migration 0017 chose for the singular whoosh-era - spellings: notes: -> notes.note:, custom_fields: -> - custom_fields.value:.""" + """ "notes:foo"/"custom_fields:foo" were valid fielded searches before + this migration. whoosh-compat's registry only exposes them as JSON + subpaths, so parse_user_query rewrites the bare prefixes live: notes: + -> notes.note:, custom_fields: -> custom_fields.value:.""" def test_bare_notes_prefix_searches_note_text( self, @@ -644,53 +482,3 @@ class TestFieldAliases: backend.add_or_update(loose) assert _matched_ids(backend, "path:archive") == {stored.pk} assert _matched_ids(backend, "storage_path:archive") == {stored.pk} - - -class TestCreatedTimezoneInvariance: - def test_created_date_matches_regardless_of_active_timezone( - self, - backend: TantivyBackend, - ) -> None: - # "created" is a date-only field indexed at naive midnight: a - # document created 2020-06-10 must match created:20200610 whether - # the active timezone is far ahead of or behind UTC. (The - # timezone-SENSITIVE datetime fields have their own boundary - # coverage in test_api_search.py's tz-ahead/tz-behind tests.) - from django.utils import timezone as django_tz - - doc = Document.objects.create( - title="Dated", - content="x", - checksum="tz-inv-1", - created=date(2020, 6, 10), - ) - backend.add_or_update(doc) - for tzname in ("Pacific/Auckland", "America/New_York"): - with django_tz.override(tzname): - assert _matched_ids(backend, "created:20200610") == {doc.pk}, tzname - - -class TestReversedDateRange: - def test_reversed_bounds_still_match_the_span( - self, - backend: TantivyBackend, - ) -> None: - # whoosh's joint disambiguation swaps backwards bounds (both years - # explicit -> plain swap), and whoosh-compat reproduces it; a saved - # view with created:[2025 TO 2020] must keep matching the span - # instead of becoming an empty lo>hi range. - inside = Document.objects.create( - title="Inside", - content="x", - checksum="rev-range-1", - created=date(2022, 5, 1), - ) - outside = Document.objects.create( - title="Outside", - content="x", - checksum="rev-range-2", - created=date(2019, 5, 1), - ) - backend.add_or_update(inside) - backend.add_or_update(outside) - assert _matched_ids(backend, "created:[2025 TO 2020]") == {inside.pk} diff --git a/src/documents/tests/search/test_query.py b/src/documents/tests/search/test_query.py index 0c0479cfb..028aac7d6 100644 --- a/src/documents/tests/search/test_query.py +++ b/src/documents/tests/search/test_query.py @@ -328,12 +328,10 @@ class TestSearchQueryErrors: class TestEmitErrorContract: - """whoosh-compat's emit() documents a two-part host contract: BOTH a - non-empty diagnostics list AND the QueryEmitError/UnsupportedQueryError - pair raised by emit() itself are user-input errors. Every one must - surface as SearchQueryError (HTTP 400), with library-internal - vocabulary (DIVERGENCES.md references, fast=True host advice) kept out - of the user-facing message.""" + """A diagnostics list, or a QueryEmitError/UnsupportedQueryError from + emit(), are both user-input errors and must surface as + SearchQueryError (HTTP 400), with library-internal wording stripped + from the message.""" @pytest.fixture def query_index(self) -> tantivy.Index: