From 846c6f88eedade304d235558ec4ac99b509bee23 Mon Sep 17 00:00:00 2001 From: Trenton Holmes <797416+stumpylog@users.noreply.github.com> Date: Mon, 17 Aug 2026 17:19:30 -0700 Subject: [PATCH] test(search): add result-level acceptance corpus, trim internals-only test_query.py classes Replaces test_query.py's intermediate-AST/query-string checks with a result-level acceptance corpus that indexes real documents and asserts matched-ID sets through parse_user_query(), covering the #13568 bracket-wildcard regression, comma value lists, field boosts, JSON subpaths, and Multitoken-in-OR nesting. Removes TestCreatedDateField, TestDateTimeFields, TestWhooshQueryRewriting, TestYearRangeRewriting, TestNonDateFieldsNotRewritten, TestPassthrough, TestNormalizeQuery, and TestParseUserQuery's test_advanced_search_queries_do_not_raise from test_query.py, since they test translate_query/_dates.py internals or a diagnostics-free-parse guarantee whoosh-compat's own suite already covers. --- src/documents/tests/search/test_acceptance.py | 247 +++++++ src/documents/tests/search/test_query.py | 681 ------------------ 2 files changed, 247 insertions(+), 681 deletions(-) create mode 100644 src/documents/tests/search/test_acceptance.py diff --git a/src/documents/tests/search/test_acceptance.py b/src/documents/tests/search/test_acceptance.py new file mode 100644 index 000000000..efbc890a2 --- /dev/null +++ b/src/documents/tests/search/test_acceptance.py @@ -0,0 +1,247 @@ +"""Result-level acceptance corpus: real documents indexed via build_schema(), +real queries run through parse_user_query(), matched-document-ID sets +asserted — not intermediate ASTs or query strings. This is paperless-ngx's +analogue of whoosh-compat's own tests/emitter/test_acceptance_e2e.py. + +Supersedes test_query.py's TestParseUserQuery result-level cases and the +now-deleted test_date_grammar_parity.py. +""" + +from __future__ import annotations + +from datetime import UTC +from datetime import datetime +from typing import TYPE_CHECKING + +import pytest + +from documents.models import CustomField +from documents.models import CustomFieldInstance +from documents.models import Document +from documents.models import Note +from documents.models import Tag +from documents.search._query import parse_user_query + +if TYPE_CHECKING: + from documents.search._backend import TantivyBackend + +pytestmark = [pytest.mark.search, pytest.mark.django_db] + +FROZEN_NOW = datetime(2026, 6, 15, 12, 0, tzinfo=UTC) + + +def _matched_ids(backend: TantivyBackend, query: str) -> set[int]: + return set(backend.search_ids(query, user=None)) + + +@pytest.fixture +def indexed_documents(backend: TantivyBackend) -> dict[str, int]: + """Index a small fixture set, return {label: doc_id} for corpus queries.""" + docs = { + "invoice_2020": Document.objects.create( + title="Invoice 2020", + content="invoice total due", + checksum="acc-invoice-2020", + archive_serial_number=100, + ), + "invoice_2021": Document.objects.create( + title="Invoice 2021", + content="invoice total due", + checksum="acc-invoice-2021", + archive_serial_number=101, + ), + "invoice_2023": Document.objects.create( + title="Invoice 2023", + content="invoice total due", + checksum="acc-invoice-2023", + archive_serial_number=102, + ), + "receipt_2022": Document.objects.create( + title="Receipt 2022", + content="receipt total due", + checksum="acc-receipt-2022", + archive_serial_number=103, + ), + } + for doc in docs.values(): + backend.add_or_update(doc) + return {label: doc.pk for label, doc in docs.items()} + + +class TestIssue13568BracketWildcard: + """paperless-ngx#13568: title:202[0-3]* must keep its character class, + not fold to a prefix query that silently drops it (whoosh-compat + DIVERGENCES.md entry 13).""" + + def test_bracket_class_wildcard_matches_only_in_range_years( + self, + backend: TantivyBackend, + indexed_documents: dict[str, int], + ) -> None: + # [0-1] (not [0-3]) is deliberate: the fixture's four years are + # 2020/2021/2022/2023, i.e. their trailing digit is 0/1/2/3 + # respectively - a [0-3] class would match all four and the test + # would pass even if the character class were silently dropped and + # folded to an unconstrained "202*" prefix. [0-1] partitions the + # fixture into a genuine in-range/out-of-range split. + matched = _matched_ids(backend, "title:202[0-1]*") + expected = { + indexed_documents["invoice_2020"], + indexed_documents["invoice_2021"], + } + assert matched == expected, ( + "title:202[0-1]* must match 2020/2021 titles and exclude 2022/2023 " + "- if this matches everything, the wildcard's character class was " + "silently dropped (issue #13568's original bug)" + ) + + +class TestCommaValueLists: + """whoosh-compat's CommaValuesPlugin splits `tag:foo,bar` into + `tag:foo AND tag:bar` (DIVERGENCES.md entries 17/36), matching real + Whoosh's KEYWORD(commas=True) analyzer-time comma splitting - not an OR + across the listed values. A document must carry every listed tag to + match.""" + + def test_tag_comma_list_matches_only_documents_with_both_tags( + self, + backend: TantivyBackend, + ) -> None: + tag_foo = Tag.objects.create(name="foo") + tag_bar = Tag.objects.create(name="bar") + tag_baz = Tag.objects.create(name="baz") + + doc_both = Document.objects.create( + title="Both", + content="x", + checksum="acc-comma-both", + ) + doc_both.tags.add(tag_foo, tag_bar) + doc_foo_only = Document.objects.create( + title="FooOnly", + content="x", + checksum="acc-comma-foo", + ) + doc_foo_only.tags.add(tag_foo) + doc_other = Document.objects.create( + title="Other", + content="x", + checksum="acc-comma-other", + ) + doc_other.tags.add(tag_baz) + for doc in (doc_both, doc_foo_only, doc_other): + backend.add_or_update(doc) + matched = _matched_ids(backend, "tag:foo,bar") + assert matched == {doc_both.pk} + + +class TestFieldBoosts: + def test_title_boost_ranks_title_match_above_content_only_match( + self, + backend: TantivyBackend, + ) -> None: + title_match = Document.objects.create( + title="urgent", + content="nothing else relevant", + checksum="acc-boost-title", + ) + content_match = Document.objects.create( + title="nothing", + content="urgent matter here", + checksum="acc-boost-content", + ) + backend.add_or_update(title_match) + backend.add_or_update(content_match) + query = parse_user_query(backend._index, "urgent", UTC) + searcher = backend._index.searcher() + results = searcher.search(query, limit=10) + ranked_ids = [ + searcher.doc(addr).to_dict()["id"][0] for _score, addr in results.hits + ] + assert ranked_ids[0] == title_match.pk + + +class TestJsonSubpaths: + def test_notes_user_matches_document_with_that_note_author( + self, + backend: TantivyBackend, + ) -> None: + from django.contrib.auth.models import User + + alice = User.objects.create_user(username="alice") + doc_with_note = Document.objects.create( + title="Has note", + content="x", + checksum="acc-note-with", + ) + Note.objects.create(document=doc_with_note, user=alice, note="reminder") + doc_without = Document.objects.create( + title="No note", + content="x", + checksum="acc-note-without", + ) + backend.add_or_update(doc_with_note) + backend.add_or_update(doc_without) + matched = _matched_ids(backend, "notes.user:alice") + assert matched == {doc_with_note.pk} + + def test_custom_fields_name_and_value_combine( + self, + backend: TantivyBackend, + ) -> None: + field = CustomField.objects.create( + name="Contract Number", + data_type=CustomField.FieldDataType.STRING, + ) + other_field = CustomField.objects.create( + name="Other Field", + data_type=CustomField.FieldDataType.STRING, + ) + matching = Document.objects.create( + title="Matching", + content="x", + checksum="acc-cf-matching", + ) + CustomFieldInstance.objects.create( + document=matching, + field=field, + value_text="policy", + ) + non_matching = Document.objects.create( + title="Non-matching", + content="x", + checksum="acc-cf-nonmatching", + ) + CustomFieldInstance.objects.create( + document=non_matching, + field=other_field, + value_text="policy", + ) + backend.add_or_update(matching) + backend.add_or_update(non_matching) + matched = _matched_ids( + backend, + 'custom_fields.name:"Contract Number" custom_fields.value:policy', + ) + assert matched == {matching.pk} + + +class TestMultitokenInNestedOr: + """whoosh-compat DIVERGENCES.md entry 15: Multitoken.DEFAULT resolves by + syntactic enclosing group, not the parser's fixed default group. Prove + it doesn't matter for paperless's actual data/fields.""" + + def test_multitoken_tag_value_inside_top_level_or_matches_either_branch( + self, + backend: TantivyBackend, + ) -> None: + # "multi word tag" is a multitoken field value; nested inside a + # top-level OR with an unrelated clause. + doc_a = Document.objects.create(title="A", content="x", checksum="acc-mt-a") + doc_a.tags.create(name="multi word tag") + doc_b = Document.objects.create(title="B", content="x", checksum="acc-mt-b") + doc_b.tags.create(name="unrelated") + backend.add_or_update(doc_a) + backend.add_or_update(doc_b) + matched = _matched_ids(backend, 'tag:"multi word tag" OR title:B') + assert matched == {doc_a.pk, doc_b.pk} diff --git a/src/documents/tests/search/test_query.py b/src/documents/tests/search/test_query.py index dc14d770e..6b50d5ec9 100644 --- a/src/documents/tests/search/test_query.py +++ b/src/documents/tests/search/test_query.py @@ -1,18 +1,13 @@ from __future__ import annotations -import re from datetime import UTC from datetime import datetime -from datetime import tzinfo from typing import TYPE_CHECKING -from zoneinfo import ZoneInfo import pytest import tantivy import time_machine -from documents.search._dates import _date_only_range -from documents.search._dates import _datetime_range from documents.search._query import InvalidNumberQuery from documents.search._query import MultipleSearchQueryErrors from documents.search._query import build_permission_filter @@ -21,402 +16,12 @@ from documents.search._query import parse_user_query from documents.search._schema import build_schema from documents.search._tokenizer import register_tokenizers from documents.search._translate import InvalidDateQuery -from documents.search._translate import translate_query if TYPE_CHECKING: from django.contrib.auth.base_user import AbstractBaseUser pytestmark = pytest.mark.search -EASTERN = ZoneInfo("America/New_York") # UTC-5 / UTC-4 (DST) -AUCKLAND = ZoneInfo("Pacific/Auckland") # UTC+13 in southern-hemisphere summer - - -def _range(result: str, field: str) -> tuple[str, str]: - # Half-open period ranges close with "}" (exclusive); exact-instant ranges - # (full ISO datetimes, "now", relative offsets) close with "]" (inclusive). - m = re.search(rf"{field}:\[(.+?) TO (.+?)[\]}}]", result) - assert m, f"No range for {field!r} in: {result!r}" - return m.group(1), m.group(2) - - -class TestCreatedDateField: - """ - created is a Django DateField: indexed as midnight UTC of the local calendar - date. No offset arithmetic needed - the local calendar date is what matters. - """ - - @pytest.mark.parametrize( - ("tz", "expected_lo", "expected_hi"), - [ - pytest.param(UTC, "2026-03-28T00:00:00Z", "2026-03-29T00:00:00Z", id="utc"), - pytest.param( - EASTERN, - "2026-03-28T00:00:00Z", - "2026-03-29T00:00:00Z", - id="eastern_same_calendar_date", - ), - ], - ) - @time_machine.travel(datetime(2026, 3, 28, 15, 30, tzinfo=UTC), tick=False) - def test_today(self, tz: tzinfo, expected_lo: str, expected_hi: str) -> None: - lo, hi = _range(translate_query("created:today", tz), "created") - assert lo == expected_lo - assert hi == expected_hi - - @time_machine.travel(datetime(2026, 3, 28, 3, 0, tzinfo=UTC), tick=False) - def test_today_auckland_ahead_of_utc(self) -> None: - # UTC 03:00 -> Auckland (UTC+13) = 16:00 same date; local date = 2026-03-28 - lo, _ = _range( - translate_query("created:today", AUCKLAND), - "created", - ) - assert lo == "2026-03-28T00:00:00Z" - - @pytest.mark.parametrize( - ("field", "keyword", "expected_lo", "expected_hi"), - [ - pytest.param( - "created", - "yesterday", - "2026-03-27T00:00:00Z", - "2026-03-28T00:00:00Z", - id="yesterday", - ), - pytest.param( - "created", - "previous week", - "2026-03-16T00:00:00Z", - "2026-03-23T00:00:00Z", - id="previous_week", - ), - pytest.param( - "created", - "this month", - "2026-03-01T00:00:00Z", - "2026-04-01T00:00:00Z", - id="this_month", - ), - pytest.param( - "created", - "previous month", - "2026-02-01T00:00:00Z", - "2026-03-01T00:00:00Z", - id="previous_month", - ), - pytest.param( - "created", - "this year", - "2026-01-01T00:00:00Z", - "2027-01-01T00:00:00Z", - id="this_year", - ), - pytest.param( - "created", - "previous year", - "2025-01-01T00:00:00Z", - "2026-01-01T00:00:00Z", - id="previous_year", - ), - ], - ) - @time_machine.travel(datetime(2026, 3, 28, 15, 0, tzinfo=UTC), tick=False) - def test_date_keywords( - self, - field: str, - keyword: str, - expected_lo: str, - expected_hi: str, - ) -> None: - # 2026-03-28 is Saturday; Mon-Sun week calculation built into expectations - query = f"{field}:{keyword}" - lo, hi = _range(translate_query(query, UTC), field) - assert lo == expected_lo - assert hi == expected_hi - - @time_machine.travel(datetime(2026, 12, 15, 12, 0, tzinfo=UTC), tick=False) - def test_this_month_december_wraps_to_next_year(self) -> None: - # December: next month must roll over to January 1 of next year - lo, hi = _range( - translate_query("created:this month", UTC), - "created", - ) - assert lo == "2026-12-01T00:00:00Z" - assert hi == "2027-01-01T00:00:00Z" - - @time_machine.travel(datetime(2026, 1, 15, 12, 0, tzinfo=UTC), tick=False) - def test_last_month_january_wraps_to_previous_year(self) -> None: - # January: last month must roll back to December 1 of previous year - lo, hi = _range( - translate_query("created:previous month", UTC), - "created", - ) - assert lo == "2025-12-01T00:00:00Z" - assert hi == "2026-01-01T00:00:00Z" - - @time_machine.travel(datetime(2026, 7, 15, 12, 0, tzinfo=UTC), tick=False) - def test_previous_quarter(self) -> None: - lo, hi = _range( - translate_query('created:"previous quarter"', UTC), - "created", - ) - assert lo == "2026-04-01T00:00:00Z" - assert hi == "2026-07-01T00:00:00Z" - - def test_unknown_keyword_raises(self) -> None: - with pytest.raises(ValueError, match="Unknown keyword"): - _date_only_range("bogus_keyword", UTC) - - -class TestDateTimeFields: - """ - added/modified store full UTC datetimes. Natural keywords must convert - the local day boundaries to UTC - timezone offset arithmetic IS required. - """ - - @time_machine.travel(datetime(2026, 3, 28, 15, 30, tzinfo=UTC), tick=False) - def test_added_today_eastern(self) -> None: - # EDT = UTC-4; local midnight 2026-03-28 00:00 EDT = 2026-03-28 04:00 UTC - lo, hi = _range(translate_query("added:today", EASTERN), "added") - assert lo == "2026-03-28T04:00:00Z" - assert hi == "2026-03-29T04:00:00Z" - - @time_machine.travel(datetime(2026, 3, 29, 2, 0, tzinfo=UTC), tick=False) - def test_added_today_auckland_midnight_crossing(self) -> None: - # UTC 02:00 on 2026-03-29 -> Auckland (UTC+13) = 2026-03-29 15:00 local - # Auckland midnight = UTC 2026-03-28 11:00 - lo, hi = _range(translate_query("added:today", AUCKLAND), "added") - assert lo == "2026-03-28T11:00:00Z" - assert hi == "2026-03-29T11:00:00Z" - - @time_machine.travel(datetime(2026, 3, 28, 15, 0, tzinfo=UTC), tick=False) - def test_modified_today_utc(self) -> None: - lo, hi = _range( - translate_query("modified:today", UTC), - "modified", - ) - assert lo == "2026-03-28T00:00:00Z" - assert hi == "2026-03-29T00:00:00Z" - - @pytest.mark.parametrize( - ("keyword", "expected_lo", "expected_hi"), - [ - pytest.param( - "yesterday", - "2026-03-27T00:00:00Z", - "2026-03-28T00:00:00Z", - id="yesterday", - ), - pytest.param( - "previous week", - "2026-03-16T00:00:00Z", - "2026-03-23T00:00:00Z", - id="previous_week", - ), - pytest.param( - "this month", - "2026-03-01T00:00:00Z", - "2026-04-01T00:00:00Z", - id="this_month", - ), - pytest.param( - "previous month", - "2026-02-01T00:00:00Z", - "2026-03-01T00:00:00Z", - id="previous_month", - ), - pytest.param( - "this year", - "2026-01-01T00:00:00Z", - "2027-01-01T00:00:00Z", - id="this_year", - ), - pytest.param( - "previous year", - "2025-01-01T00:00:00Z", - "2026-01-01T00:00:00Z", - id="previous_year", - ), - ], - ) - @time_machine.travel(datetime(2026, 3, 28, 12, 0, tzinfo=UTC), tick=False) - def test_datetime_keywords_utc( - self, - keyword: str, - expected_lo: str, - expected_hi: str, - ) -> None: - # 2026-03-28 is Saturday; weekday()==5 so Monday=2026-03-23 - lo, hi = _range(translate_query(f"added:{keyword}", UTC), "added") - assert lo == expected_lo - assert hi == expected_hi - - @time_machine.travel(datetime(2026, 12, 15, 12, 0, tzinfo=UTC), tick=False) - def test_this_month_december_wraps_to_next_year(self) -> None: - # December: next month wraps to January of next year - lo, hi = _range(translate_query("added:this month", UTC), "added") - assert lo == "2026-12-01T00:00:00Z" - assert hi == "2027-01-01T00:00:00Z" - - @time_machine.travel(datetime(2026, 1, 15, 12, 0, tzinfo=UTC), tick=False) - def test_last_month_january_wraps_to_previous_year(self) -> None: - # January: last month wraps back to December of previous year - lo, hi = _range( - translate_query("added:previous month", UTC), - "added", - ) - assert lo == "2025-12-01T00:00:00Z" - assert hi == "2026-01-01T00:00:00Z" - - @pytest.mark.parametrize( - ("query", "expected_lo", "expected_hi"), - [ - pytest.param( - 'added:"previous quarter"', - "2026-04-01T00:00:00Z", - "2026-07-01T00:00:00Z", - id="quoted_previous_quarter", - ), - pytest.param( - "added:previous month", - "2026-06-01T00:00:00Z", - "2026-07-01T00:00:00Z", - id="bare_previous_month", - ), - pytest.param( - "added:this month", - "2026-07-01T00:00:00Z", - "2026-08-01T00:00:00Z", - id="bare_this_month", - ), - ], - ) - @time_machine.travel(datetime(2026, 7, 15, 12, 0, tzinfo=UTC), tick=False) - def test_legacy_natural_language_aliases( - self, - query: str, - expected_lo: str, - expected_hi: str, - ) -> None: - lo, hi = _range(translate_query(query, UTC), "added") - assert lo == expected_lo - assert hi == expected_hi - - def test_unknown_keyword_raises(self) -> None: - with pytest.raises(ValueError, match="Unknown keyword"): - _datetime_range("bogus_keyword", UTC) - - -class TestWhooshQueryRewriting: - """All Whoosh query syntax variants must be rewritten to ISO 8601 before Tantivy parses them.""" - - @time_machine.travel(datetime(2026, 3, 28, 15, 0, tzinfo=UTC), tick=False) - def test_compact_date_shim_rewrites_to_iso(self) -> None: - result = translate_query("created:20240115120000", UTC) - assert "2024-01-15" in result - assert "20240115120000" not in result - - @time_machine.travel(datetime(2026, 3, 28, 15, 0, tzinfo=UTC), tick=False) - def test_relative_range_shim_removes_now(self) -> None: - result = translate_query("added:[now-7d TO now]", UTC) - assert "now" not in result - assert "2026-03-" in result - - @time_machine.travel(datetime(2026, 3, 28, 12, 0, tzinfo=UTC), tick=False) - def test_bracket_minus_7_days(self) -> None: - lo, hi = _range( - translate_query("added:[-7 days to now]", UTC), - "added", - ) - assert lo == "2026-03-21T12:00:00Z" - assert hi == "2026-03-28T12:00:00Z" - - @time_machine.travel(datetime(2026, 3, 28, 12, 0, tzinfo=UTC), tick=False) - def test_bracket_minus_1_week(self) -> None: - lo, hi = _range( - translate_query("added:[-1 week to now]", UTC), - "added", - ) - assert lo == "2026-03-21T12:00:00Z" - assert hi == "2026-03-28T12:00:00Z" - - @time_machine.travel(datetime(2026, 3, 28, 12, 0, tzinfo=UTC), tick=False) - def test_bracket_minus_1_month_uses_relativedelta(self) -> None: - # relativedelta(months=1) from 2026-03-28 = 2026-02-28 (not 29) - lo, hi = _range( - translate_query("created:[-1 month to now]", UTC), - "created", - ) - assert lo == "2026-02-28T12:00:00Z" - assert hi == "2026-03-28T12:00:00Z" - - @time_machine.travel(datetime(2026, 3, 28, 12, 0, tzinfo=UTC), tick=False) - def test_bracket_minus_1_year(self) -> None: - lo, hi = _range( - translate_query("modified:[-1 year to now]", UTC), - "modified", - ) - assert lo == "2025-03-28T12:00:00Z" - assert hi == "2026-03-28T12:00:00Z" - - @time_machine.travel(datetime(2026, 3, 28, 12, 0, tzinfo=UTC), tick=False) - def test_bracket_plural_unit_hours(self) -> None: - lo, hi = _range( - translate_query("added:[-3 hours to now]", UTC), - "added", - ) - assert lo == "2026-03-28T09:00:00Z" - assert hi == "2026-03-28T12:00:00Z" - - @time_machine.travel(datetime(2026, 3, 28, 12, 0, tzinfo=UTC), tick=False) - def test_bracket_case_insensitive(self) -> None: - result = translate_query("added:[-1 WEEK TO NOW]", UTC) - assert "now" not in result.lower() - lo, hi = _range(result, "added") - assert lo == "2026-03-21T12:00:00Z" - assert hi == "2026-03-28T12:00:00Z" - - @time_machine.travel(datetime(2026, 3, 28, 12, 0, tzinfo=UTC), tick=False) - def test_relative_range_swaps_bounds_when_lo_exceeds_hi(self) -> None: - # [now+1h TO now-1h] has lo > hi before substitution; they must be swapped - lo, hi = _range( - translate_query("added:[now+1h TO now-1h]", UTC), - "added", - ) - assert lo == "2026-03-28T11:00:00Z" - assert hi == "2026-03-28T13:00:00Z" - - def test_8digit_created_date_field_always_uses_utc_midnight(self) -> None: - # created is a DateField: boundaries are always UTC midnight, no TZ offset - result = translate_query("created:20231201", EASTERN) - lo, hi = _range(result, "created") - assert lo == "2023-12-01T00:00:00Z" - assert hi == "2023-12-02T00:00:00Z" - - def test_8digit_added_datetime_field_converts_local_midnight_to_utc(self) -> None: - # added is DateTimeField: midnight Dec 1 Eastern (EST = UTC-5) = 05:00 UTC - result = translate_query("added:20231201", EASTERN) - lo, hi = _range(result, "added") - assert lo == "2023-12-01T05:00:00Z" - assert hi == "2023-12-02T05:00:00Z" - - def test_8digit_modified_datetime_field_converts_local_midnight_to_utc( - self, - ) -> None: - result = translate_query("modified:20231201", EASTERN) - lo, hi = _range(result, "modified") - assert lo == "2023-12-01T05:00:00Z" - assert hi == "2023-12-02T05:00:00Z" - - def test_8digit_invalid_date_raises(self) -> None: - # The translation pipeline raises InvalidDateQuery for unparsable dates - # (e.g. month=13) so the API can surface a 400 telling the user the date - # is malformed instead of silently returning zero results. - with pytest.raises(InvalidDateQuery) as exc_info: - translate_query("added:20231340", UTC) - assert exc_info.value.field == "added" - assert exc_info.value.value == "20231340" - class TestParseUserQuery: """parse_user_query runs the full preprocessing pipeline.""" @@ -468,68 +73,6 @@ class TestParseUserQuery: ) -> None: assert isinstance(parse_user_query(query_index, raw_query, UTC), tantivy.Query) - @pytest.mark.parametrize( - "raw_query", - [ - # Partial date scalar (year only) - pytest.param("created:2020", id="created_year_scalar"), - # 8-digit compact date range in brackets - pytest.param( - "created:[20200101 TO 20201231]", - id="created_8digit_bracket_range", - ), - # Comma-separated field + date range (Whoosh v2 multi-clause syntax) - pytest.param( - "title:x,created:[2020 TO 2021]", - id="title_comma_created_range", - ), - # Field alias: type -> document_type - pytest.param("type:invoice", id="type_alias"), - # Multi-word date keyword, quoted. whoosh-compat's DateParserPlugin - # deliberately drops whoosh's "free" undelimited-date tagging mode - # (see whoosh_compat.parser.dateparse.DateParserPlugin docstring): - # unquoted "created:previous week" no longer parses (confirmed via - # whoosh_compat.parser.dateparse.English().date_from returning None - # for the bare "previous" token). This matches how the current - # frontend already emits this query (filter-editor.component.ts - # always quotes non-range relative date values), so the quoted - # form here reflects real traffic, not weakened coverage. - pytest.param('created:"previous week"', id="created_previous_week"), - # Full ISO datetime range ("T"/"Z" RFC3339 style). This exact query - # shape was added in PR #13010 to restore v2 (Whoosh) advanced-search - # compatibility, i.e. it represents real saved-search back-compat. - # whoosh-compat's date grammar now accepts "T" as a date/time - # separator and a trailing "Z" UTC designator (see - # whoosh_compat.parser.dateparse's English/_split_rfc3339_utc). - pytest.param( - "created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z]", - id="created_iso_range", - ), - # Comma-separated ISO ranges (Whoosh v2 syntax) -- same shape as above. - pytest.param( - "created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z]," - "added:[2026-05-01T00:00:00Z TO 2026-06-01T00:00:00Z]", - id="comma_iso_ranges", - ), - ], - ) - def test_advanced_search_queries_do_not_raise( - self, - query_index: tantivy.Index, - raw_query: str, - ) -> None: - """ - End-to-end: queries that the frontend sends must parse without raising. - - This tests the full pipeline: translate_query -> tantivy parse_query. - Equivalent to asserting HTTP 200 (not 400) for each query form. - """ - with time_machine.travel(datetime(2026, 6, 15, 12, 0, tzinfo=UTC), tick=False): - assert isinstance( - parse_user_query(query_index, raw_query, UTC), - tantivy.Query, - ) - def test_invalid_date_propagates_not_swallowed( self, query_index: tantivy.Index, @@ -598,230 +141,6 @@ class TestParseUserQuery: assert isinstance(q, tantivy.Query) -class TestYearRangeRewriting: - """Whoosh-style year-only date ranges must be rewritten to ISO 8601.""" - - @pytest.mark.parametrize( - ("query", "field", "expected_lo", "expected_hi"), - [ - pytest.param( - "created:[2020 TO 2020]", - "created", - "2020-01-01T00:00:00Z", - "2021-01-01T00:00:00Z", - id="single_year_created", - ), - pytest.param( - "created:[2018 TO 2021]", - "created", - "2018-01-01T00:00:00Z", - "2022-01-01T00:00:00Z", - id="multi_year_range_created", - ), - pytest.param( - "added:[2022 TO 2023]", - "added", - "2022-01-01T00:00:00Z", - "2024-01-01T00:00:00Z", - id="added_field", - ), - pytest.param( - "modified:[2021 TO 2021]", - "modified", - "2021-01-01T00:00:00Z", - "2022-01-01T00:00:00Z", - id="modified_field", - ), - pytest.param( - "created:[2020 to 2020]", - "created", - "2020-01-01T00:00:00Z", - "2021-01-01T00:00:00Z", - id="lowercase_to_keyword", - ), - ], - ) - def test_year_range_rewritten( - self, - query: str, - field: str, - expected_lo: str, - expected_hi: str, - ) -> None: - result = translate_query(query, UTC) - lo, hi = _range(result, field) - assert lo == expected_lo - assert hi == expected_hi - - def test_reversed_year_range_is_swapped(self) -> None: - # A reversed range must not yield lo > hi, which Tantivy treats as an - # empty range (silently zero results). The bounds are swapped instead. - result = translate_query("created:[2025 TO 2020]", UTC) - lo, hi = _range(result, "created") - assert lo == "2020-01-01T00:00:00Z" - assert hi == "2026-01-01T00:00:00Z" - - def test_year_range_in_complex_boolean_query(self) -> None: - query = "tag:steuer AND (title:2020 OR (NOT title:2019 AND NOT title:2018 AND created:[2020 TO 2020]))" - result = translate_query(query, UTC) - lo, hi = _range(result, "created") - assert lo == "2020-01-01T00:00:00Z" - assert hi == "2021-01-01T00:00:00Z" - assert "title:2020" in result - assert "title:2019" in result - assert "title:2018" in result - - def test_already_iso_date_range_passes_through_unchanged(self) -> None: - original = "created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z]" - assert translate_query(original, UTC) == original - - def test_8digit_in_brackets_not_matched_as_year_range(self) -> None: - # [YYYYMMDD TO YYYYMMDD]: the translation layer converts 8-digit bounds to - # ISO day ranges. 20200101 -> 2020-01-01T00:00:00Z (lo of that day); - # 20201231 -> the ceil of Dec 31 = 2021-01-01T00:00:00Z (exclusive end). - # This is the correct and accepted behavior: old compact form becomes a - # proper Tantivy-parseable ISO range. - original = "created:[20200101 TO 20201231]" - result = translate_query(original, UTC) - lo, hi = _range(result, "created") - assert lo == "2020-01-01T00:00:00Z" - assert hi == "2021-01-01T00:00:00Z" - - -class TestNonDateFieldsNotRewritten: - """Date rewriters must only fire on the date fields (created/modified/added). - - Integer fields like asn/id/page_count and unknown fields would otherwise be - rewritten into date ranges and rejected by Tantivy as type mismatches. - """ - - @pytest.mark.parametrize( - "query", - [ - pytest.param("asn:20240101", id="asn_8digit"), - pytest.param("id:20240101", id="id_8digit"), - pytest.param("page_count:12345678", id="page_count_8digit"), - pytest.param("num_notes:20231201", id="num_notes_8digit"), - ], - ) - def test_8digit_on_integer_field_passes_through_unchanged(self, query: str) -> None: - assert translate_query(query, EASTERN) == query - - @pytest.mark.parametrize( - "query", - [ - pytest.param("asn:[2000 TO 2024]", id="asn_year_range"), - pytest.param("id:[2000 TO 2024]", id="id_year_range"), - pytest.param("page_count:[2000 TO 2024]", id="page_count_year_range"), - ], - ) - def test_year_range_on_integer_field_passes_through_unchanged( - self, - query: str, - ) -> None: - assert translate_query(query, UTC) == query - - def test_unknown_field_keyword_passes_through_unchanged(self) -> None: - # foobar is not a date field: 'foobar:today' must not become a date range, - # which Tantivy would otherwise reject as an unknown/typed field. - assert translate_query("foobar:today", UTC) == "foobar:today" - - -class TestPassthrough: - """Queries without field prefixes or unrelated content pass through unchanged.""" - - def test_bare_keyword_no_field_prefix_unchanged(self) -> None: - # Bare 'today' with no field: prefix passes through unchanged - result = translate_query("bank statement today", UTC) - assert "today" in result - - def test_unrelated_query_unchanged(self) -> None: - assert translate_query("title:invoice", UTC) == "title:invoice" - - -class TestNormalizeQuery: - """translate_query expands comma-separated values and collapses whitespace.""" - - def test_normalize_expands_comma_separated_tags(self) -> None: - assert translate_query("tag:foo,bar", UTC) == "tag:foo AND tag:bar" - - def test_normalize_comma_between_range_expressions(self) -> None: - # Comma-separated field range expressions (Whoosh v2 syntax) must be - # converted to AND so Tantivy does not receive an invalid comma. - q = "created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z],added:[2026-05-01T00:00:00Z TO 2026-06-01T00:00:00Z]" - assert translate_query(q, UTC) == ( - "created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z]" - " AND " - "added:[2026-05-01T00:00:00Z TO 2026-06-01T00:00:00Z]" - ) - - def test_normalize_expands_three_values(self) -> None: - assert ( - translate_query("tag:foo,bar,baz", UTC) == "tag:foo AND tag:bar AND tag:baz" - ) - - def test_normalize_collapses_whitespace(self) -> None: - assert translate_query("bank statement", UTC) == "bank statement" - - def test_normalize_no_commas_unchanged(self) -> None: - assert translate_query("bank statement", UTC) == "bank statement" - - @pytest.mark.parametrize( - ("raw", "expected"), - [ - pytest.param( - "h52.1 - kurzsichtigkeit", - "h52.1 kurzsichtigkeit", - id="icd_code_dash_description", - ), - pytest.param( - "H52.1 - asd", - "H52.1 asd", - id="icd_code_uppercase_dash", - ), - pytest.param( - "h52.1 -", - "h52.1", - id="trailing_minus", - ), - pytest.param( - ". -", - ".", - id="dot_trailing_minus", - ), - pytest.param( - "h52. -", - "h52.", - id="partial_code_trailing_minus", - ), - pytest.param( - "foo - bar - baz", - "foo bar baz", - id="multiple_dashes", - ), - pytest.param( - "foo + bar", - "foo bar", - id="spaced_plus_operator", - ), - ], - ) - def test_normalize_strips_dangling_operators(self, raw: str, expected: str) -> None: - assert translate_query(raw, UTC) == expected - - @pytest.mark.parametrize( - "query", - [ - pytest.param("term -other", id="adjacent_not_operator"), - pytest.param("-term", id="leading_not_operator"), - pytest.param("+term", id="leading_must_operator"), - pytest.param("foo -bar +baz", id="mixed_adjacent_operators"), - ], - ) - def test_normalize_preserves_valid_operators(self, query: str) -> None: - assert translate_query(query, UTC) == query - - class TestParseSimpleTextHighlightQuery: """parse_simple_text_highlight_query must not raise on natural-language queries."""