amorfo77/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2026-08-19 01:03:18 +00:00
Code Issues Projects Releases Wiki Activity

test(search): add result-level acceptance corpus, trim internals-only test_query.py classes

Browse Source 
Replaces test_query.py's intermediate-AST/query-string checks with a
result-level acceptance corpus that indexes real documents and asserts
matched-ID sets through parse_user_query(), covering the #13568
bracket-wildcard regression, comma value lists, field boosts, JSON subpaths,
and Multitoken-in-OR nesting. Removes TestCreatedDateField, TestDateTimeFields,
TestWhooshQueryRewriting, TestYearRangeRewriting, TestNonDateFieldsNotRewritten,
TestPassthrough, TestNormalizeQuery, and TestParseUserQuery's
test_advanced_search_queries_do_not_raise from test_query.py, since they test
translate_query/_dates.py internals or a diagnostics-free-parse guarantee
whoosh-compat's own suite already covers.
This commit is contained in:
Trenton Holmes
2026-08-18 11:05:04 -07:00
parent bb157726c9
commit 846c6f88ee
2 changed files with 247 additions and 681 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/documents/tests/search/test_acceptance.py
+247
View File
@@ -0,0 +1,247 @@
"""Result-level acceptance corpus: real documents indexed via build_schema(),
real queries run through parse_user_query(), matched-document-ID sets
asserted — not intermediate ASTs or query strings. This is paperless-ngx's
analogue of whoosh-compat's own tests/emitter/test_acceptance_e2e.py.
Supersedes test_query.py's TestParseUserQuery result-level cases and the
now-deleted test_date_grammar_parity.py.
"""
from __future__ import annotations
from datetime import UTC
from datetime import datetime
from typing import TYPE_CHECKING
import pytest
from documents.models import CustomField
from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import Note
from documents.models import Tag
from documents.search._query import parse_user_query
if TYPE_CHECKING:
from documents.search._backend import TantivyBackend
pytestmark = [pytest.mark.search, pytest.mark.django_db]
FROZEN_NOW = datetime(2026, 6, 15, 12, 0, tzinfo=UTC)
def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
return set(backend.search_ids(query, user=None))
@pytest.fixture
def indexed_documents(backend: TantivyBackend) -> dict[str, int]:
"""Index a small fixture set, return {label: doc_id} for corpus queries."""
docs = {
"invoice_2020": Document.objects.create(
title="Invoice 2020",
content="invoice total due",
checksum="acc-invoice-2020",
archive_serial_number=100,
),
"invoice_2021": Document.objects.create(
title="Invoice 2021",
content="invoice total due",
checksum="acc-invoice-2021",
archive_serial_number=101,
),
"invoice_2023": Document.objects.create(
title="Invoice 2023",
content="invoice total due",
checksum="acc-invoice-2023",
archive_serial_number=102,
),
"receipt_2022": Document.objects.create(
title="Receipt 2022",
content="receipt total due",
checksum="acc-receipt-2022",
archive_serial_number=103,
),
}
for doc in docs.values():
backend.add_or_update(doc)
return {label: doc.pk for label, doc in docs.items()}
class TestIssue13568BracketWildcard:
"""paperless-ngx#13568: title:202[0-3]* must keep its character class,
not fold to a prefix query that silently drops it (whoosh-compat
DIVERGENCES.md entry 13)."""
def test_bracket_class_wildcard_matches_only_in_range_years(
self,
backend: TantivyBackend,
indexed_documents: dict[str, int],
) -> None:
# [0-1] (not [0-3]) is deliberate: the fixture's four years are
# 2020/2021/2022/2023, i.e. their trailing digit is 0/1/2/3
# respectively - a [0-3] class would match all four and the test
# would pass even if the character class were silently dropped and
# folded to an unconstrained "202*" prefix. [0-1] partitions the
# fixture into a genuine in-range/out-of-range split.
matched = _matched_ids(backend, "title:202[0-1]*")
expected = {
indexed_documents["invoice_2020"],
indexed_documents["invoice_2021"],
}
assert matched == expected, (
"title:202[0-1]* must match 2020/2021 titles and exclude 2022/2023 "
"- if this matches everything, the wildcard's character class was "
"silently dropped (issue #13568's original bug)"
)
class TestCommaValueLists:
"""whoosh-compat's CommaValuesPlugin splits `tag:foo,bar` into
`tag:foo AND tag:bar` (DIVERGENCES.md entries 17/36), matching real
Whoosh's KEYWORD(commas=True) analyzer-time comma splitting - not an OR
across the listed values. A document must carry every listed tag to
match."""
def test_tag_comma_list_matches_only_documents_with_both_tags(
self,
backend: TantivyBackend,
) -> None:
tag_foo = Tag.objects.create(name="foo")
tag_bar = Tag.objects.create(name="bar")
tag_baz = Tag.objects.create(name="baz")
doc_both = Document.objects.create(
title="Both",
content="x",
checksum="acc-comma-both",
)
doc_both.tags.add(tag_foo, tag_bar)
doc_foo_only = Document.objects.create(
title="FooOnly",
content="x",
checksum="acc-comma-foo",
)
doc_foo_only.tags.add(tag_foo)
doc_other = Document.objects.create(
title="Other",
content="x",
checksum="acc-comma-other",
)
doc_other.tags.add(tag_baz)
for doc in (doc_both, doc_foo_only, doc_other):
backend.add_or_update(doc)
matched = _matched_ids(backend, "tag:foo,bar")
assert matched == {doc_both.pk}
class TestFieldBoosts:
def test_title_boost_ranks_title_match_above_content_only_match(
self,
backend: TantivyBackend,
) -> None:
title_match = Document.objects.create(
title="urgent",
content="nothing else relevant",
checksum="acc-boost-title",
)
content_match = Document.objects.create(
title="nothing",
content="urgent matter here",
checksum="acc-boost-content",
)
backend.add_or_update(title_match)
backend.add_or_update(content_match)
query = parse_user_query(backend._index, "urgent", UTC)
searcher = backend._index.searcher()
results = searcher.search(query, limit=10)
ranked_ids = [
searcher.doc(addr).to_dict()["id"][0] for _score, addr in results.hits
]
assert ranked_ids[0] == title_match.pk
class TestJsonSubpaths:
def test_notes_user_matches_document_with_that_note_author(
self,
backend: TantivyBackend,
) -> None:
from django.contrib.auth.models import User
alice = User.objects.create_user(username="alice")
doc_with_note = Document.objects.create(
title="Has note",
content="x",
checksum="acc-note-with",
)
Note.objects.create(document=doc_with_note, user=alice, note="reminder")
doc_without = Document.objects.create(
title="No note",
content="x",
checksum="acc-note-without",
)
backend.add_or_update(doc_with_note)
backend.add_or_update(doc_without)
matched = _matched_ids(backend, "notes.user:alice")
assert matched == {doc_with_note.pk}
def test_custom_fields_name_and_value_combine(
self,
backend: TantivyBackend,
) -> None:
field = CustomField.objects.create(
name="Contract Number",
data_type=CustomField.FieldDataType.STRING,
)
other_field = CustomField.objects.create(
name="Other Field",
data_type=CustomField.FieldDataType.STRING,
)
matching = Document.objects.create(
title="Matching",
content="x",
checksum="acc-cf-matching",
)
CustomFieldInstance.objects.create(
document=matching,
field=field,
value_text="policy",
)
non_matching = Document.objects.create(
title="Non-matching",
content="x",
checksum="acc-cf-nonmatching",
)
CustomFieldInstance.objects.create(
document=non_matching,
field=other_field,
value_text="policy",
)
backend.add_or_update(matching)
backend.add_or_update(non_matching)
matched = _matched_ids(
backend,
'custom_fields.name:"Contract Number" custom_fields.value:policy',
)
assert matched == {matching.pk}
class TestMultitokenInNestedOr:
"""whoosh-compat DIVERGENCES.md entry 15: Multitoken.DEFAULT resolves by
syntactic enclosing group, not the parser's fixed default group. Prove
it doesn't matter for paperless's actual data/fields."""
def test_multitoken_tag_value_inside_top_level_or_matches_either_branch(
self,
backend: TantivyBackend,
) -> None:
# "multi word tag" is a multitoken field value; nested inside a
# top-level OR with an unrelated clause.
doc_a = Document.objects.create(title="A", content="x", checksum="acc-mt-a")
doc_a.tags.create(name="multi word tag")
doc_b = Document.objects.create(title="B", content="x", checksum="acc-mt-b")
doc_b.tags.create(name="unrelated")
backend.add_or_update(doc_a)
backend.add_or_update(doc_b)
matched = _matched_ids(backend, 'tag:"multi word tag" OR title:B')
assert matched == {doc_a.pk, doc_b.pk}
src/documents/tests/search/test_query.py
-681
View File
@@ -1,18 +1,13 @@
from __future__ import annotations
import re
from datetime import UTC
from datetime import datetime
from datetime import tzinfo
from typing import TYPE_CHECKING
from zoneinfo import ZoneInfo
import pytest
import tantivy
import time_machine
from documents.search._dates import _date_only_range
from documents.search._dates import _datetime_range
from documents.search._query import InvalidNumberQuery
from documents.search._query import MultipleSearchQueryErrors
from documents.search._query import build_permission_filter
@@ -21,402 +16,12 @@ from documents.search._query import parse_user_query
from documents.search._schema import build_schema
from documents.search._tokenizer import register_tokenizers
from documents.search._translate import InvalidDateQuery
from documents.search._translate import translate_query
if TYPE_CHECKING:
from django.contrib.auth.base_user import AbstractBaseUser
pytestmark = pytest.mark.search
EASTERN = ZoneInfo("America/New_York") # UTC-5 / UTC-4 (DST)
AUCKLAND = ZoneInfo("Pacific/Auckland") # UTC+13 in southern-hemisphere summer
def _range(result: str, field: str) -> tuple[str, str]:
# Half-open period ranges close with "}" (exclusive); exact-instant ranges
# (full ISO datetimes, "now", relative offsets) close with "]" (inclusive).
m = re.search(rf"{field}:\[(.+?) TO (.+?)[\]}}]", result)
assert m, f"No range for {field!r} in: {result!r}"
return m.group(1), m.group(2)
class TestCreatedDateField:
"""
created is a Django DateField: indexed as midnight UTC of the local calendar
date. No offset arithmetic needed - the local calendar date is what matters.
"""
@pytest.mark.parametrize(
("tz", "expected_lo", "expected_hi"),
[
pytest.param(UTC, "2026-03-28T00:00:00Z", "2026-03-29T00:00:00Z", id="utc"),
pytest.param(
EASTERN,
"2026-03-28T00:00:00Z",
"2026-03-29T00:00:00Z",
id="eastern_same_calendar_date",
),
],
)
@time_machine.travel(datetime(2026, 3, 28, 15, 30, tzinfo=UTC), tick=False)
def test_today(self, tz: tzinfo, expected_lo: str, expected_hi: str) -> None:
lo, hi = _range(translate_query("created:today", tz), "created")
assert lo == expected_lo
assert hi == expected_hi
@time_machine.travel(datetime(2026, 3, 28, 3, 0, tzinfo=UTC), tick=False)
def test_today_auckland_ahead_of_utc(self) -> None:
# UTC 03:00 -> Auckland (UTC+13) = 16:00 same date; local date = 2026-03-28
lo, _ = _range(
translate_query("created:today", AUCKLAND),
"created",
)
assert lo == "2026-03-28T00:00:00Z"
@pytest.mark.parametrize(
("field", "keyword", "expected_lo", "expected_hi"),
[
pytest.param(
"created",
"yesterday",
"2026-03-27T00:00:00Z",
"2026-03-28T00:00:00Z",
id="yesterday",
),
pytest.param(
"created",
"previous week",
"2026-03-16T00:00:00Z",
"2026-03-23T00:00:00Z",
id="previous_week",
),
pytest.param(
"created",
"this month",
"2026-03-01T00:00:00Z",
"2026-04-01T00:00:00Z",
id="this_month",
),
pytest.param(
"created",
"previous month",
"2026-02-01T00:00:00Z",
"2026-03-01T00:00:00Z",
id="previous_month",
),
pytest.param(
"created",
"this year",
"2026-01-01T00:00:00Z",
"2027-01-01T00:00:00Z",
id="this_year",
),
pytest.param(
"created",
"previous year",
"2025-01-01T00:00:00Z",
"2026-01-01T00:00:00Z",
id="previous_year",
),
],
)
@time_machine.travel(datetime(2026, 3, 28, 15, 0, tzinfo=UTC), tick=False)
def test_date_keywords(
self,
field: str,
keyword: str,
expected_lo: str,
expected_hi: str,
) -> None:
# 2026-03-28 is Saturday; Mon-Sun week calculation built into expectations
query = f"{field}:{keyword}"
lo, hi = _range(translate_query(query, UTC), field)
assert lo == expected_lo
assert hi == expected_hi
@time_machine.travel(datetime(2026, 12, 15, 12, 0, tzinfo=UTC), tick=False)
def test_this_month_december_wraps_to_next_year(self) -> None:
# December: next month must roll over to January 1 of next year
lo, hi = _range(
translate_query("created:this month", UTC),
"created",
)
assert lo == "2026-12-01T00:00:00Z"
assert hi == "2027-01-01T00:00:00Z"
@time_machine.travel(datetime(2026, 1, 15, 12, 0, tzinfo=UTC), tick=False)
def test_last_month_january_wraps_to_previous_year(self) -> None:
# January: last month must roll back to December 1 of previous year
lo, hi = _range(
translate_query("created:previous month", UTC),
"created",
)
assert lo == "2025-12-01T00:00:00Z"
assert hi == "2026-01-01T00:00:00Z"
@time_machine.travel(datetime(2026, 7, 15, 12, 0, tzinfo=UTC), tick=False)
def test_previous_quarter(self) -> None:
lo, hi = _range(
translate_query('created:"previous quarter"', UTC),
"created",
)
assert lo == "2026-04-01T00:00:00Z"
assert hi == "2026-07-01T00:00:00Z"
def test_unknown_keyword_raises(self) -> None:
with pytest.raises(ValueError, match="Unknown keyword"):
_date_only_range("bogus_keyword", UTC)
class TestDateTimeFields:
"""
added/modified store full UTC datetimes. Natural keywords must convert
the local day boundaries to UTC - timezone offset arithmetic IS required.
"""
@time_machine.travel(datetime(2026, 3, 28, 15, 30, tzinfo=UTC), tick=False)
def test_added_today_eastern(self) -> None:
# EDT = UTC-4; local midnight 2026-03-28 00:00 EDT = 2026-03-28 04:00 UTC
lo, hi = _range(translate_query("added:today", EASTERN), "added")
assert lo == "2026-03-28T04:00:00Z"
assert hi == "2026-03-29T04:00:00Z"
@time_machine.travel(datetime(2026, 3, 29, 2, 0, tzinfo=UTC), tick=False)
def test_added_today_auckland_midnight_crossing(self) -> None:
# UTC 02:00 on 2026-03-29 -> Auckland (UTC+13) = 2026-03-29 15:00 local
# Auckland midnight = UTC 2026-03-28 11:00
lo, hi = _range(translate_query("added:today", AUCKLAND), "added")
assert lo == "2026-03-28T11:00:00Z"
assert hi == "2026-03-29T11:00:00Z"
@time_machine.travel(datetime(2026, 3, 28, 15, 0, tzinfo=UTC), tick=False)
def test_modified_today_utc(self) -> None:
lo, hi = _range(
translate_query("modified:today", UTC),
"modified",
)
assert lo == "2026-03-28T00:00:00Z"
assert hi == "2026-03-29T00:00:00Z"
@pytest.mark.parametrize(
("keyword", "expected_lo", "expected_hi"),
[
pytest.param(
"yesterday",
"2026-03-27T00:00:00Z",
"2026-03-28T00:00:00Z",
id="yesterday",
),
pytest.param(
"previous week",
"2026-03-16T00:00:00Z",
"2026-03-23T00:00:00Z",
id="previous_week",
),
pytest.param(
"this month",
"2026-03-01T00:00:00Z",
"2026-04-01T00:00:00Z",
id="this_month",
),
pytest.param(
"previous month",
"2026-02-01T00:00:00Z",
"2026-03-01T00:00:00Z",
id="previous_month",
),
pytest.param(
"this year",
"2026-01-01T00:00:00Z",
"2027-01-01T00:00:00Z",
id="this_year",
),
pytest.param(
"previous year",
"2025-01-01T00:00:00Z",
"2026-01-01T00:00:00Z",
id="previous_year",
),
],
)
@time_machine.travel(datetime(2026, 3, 28, 12, 0, tzinfo=UTC), tick=False)
def test_datetime_keywords_utc(
self,
keyword: str,
expected_lo: str,
expected_hi: str,
) -> None:
# 2026-03-28 is Saturday; weekday()==5 so Monday=2026-03-23
lo, hi = _range(translate_query(f"added:{keyword}", UTC), "added")
assert lo == expected_lo
assert hi == expected_hi
@time_machine.travel(datetime(2026, 12, 15, 12, 0, tzinfo=UTC), tick=False)
def test_this_month_december_wraps_to_next_year(self) -> None:
# December: next month wraps to January of next year
lo, hi = _range(translate_query("added:this month", UTC), "added")
assert lo == "2026-12-01T00:00:00Z"
assert hi == "2027-01-01T00:00:00Z"
@time_machine.travel(datetime(2026, 1, 15, 12, 0, tzinfo=UTC), tick=False)
def test_last_month_january_wraps_to_previous_year(self) -> None:
# January: last month wraps back to December of previous year
lo, hi = _range(
translate_query("added:previous month", UTC),
"added",
)
assert lo == "2025-12-01T00:00:00Z"
assert hi == "2026-01-01T00:00:00Z"
@pytest.mark.parametrize(
("query", "expected_lo", "expected_hi"),
[
pytest.param(
'added:"previous quarter"',
"2026-04-01T00:00:00Z",
"2026-07-01T00:00:00Z",
id="quoted_previous_quarter",
),
pytest.param(
"added:previous month",
"2026-06-01T00:00:00Z",
"2026-07-01T00:00:00Z",
id="bare_previous_month",
),
pytest.param(
"added:this month",
"2026-07-01T00:00:00Z",
"2026-08-01T00:00:00Z",
id="bare_this_month",
),
],
)
@time_machine.travel(datetime(2026, 7, 15, 12, 0, tzinfo=UTC), tick=False)
def test_legacy_natural_language_aliases(
self,
query: str,
expected_lo: str,
expected_hi: str,
) -> None:
lo, hi = _range(translate_query(query, UTC), "added")
assert lo == expected_lo
assert hi == expected_hi
def test_unknown_keyword_raises(self) -> None:
with pytest.raises(ValueError, match="Unknown keyword"):
_datetime_range("bogus_keyword", UTC)
class TestWhooshQueryRewriting:
"""All Whoosh query syntax variants must be rewritten to ISO 8601 before Tantivy parses them."""
@time_machine.travel(datetime(2026, 3, 28, 15, 0, tzinfo=UTC), tick=False)
def test_compact_date_shim_rewrites_to_iso(self) -> None:
result = translate_query("created:20240115120000", UTC)
assert "2024-01-15" in result
assert "20240115120000" not in result
@time_machine.travel(datetime(2026, 3, 28, 15, 0, tzinfo=UTC), tick=False)
def test_relative_range_shim_removes_now(self) -> None:
result = translate_query("added:[now-7d TO now]", UTC)
assert "now" not in result
assert "2026-03-" in result
@time_machine.travel(datetime(2026, 3, 28, 12, 0, tzinfo=UTC), tick=False)
def test_bracket_minus_7_days(self) -> None:
lo, hi = _range(
translate_query("added:[-7 days to now]", UTC),
"added",
)
assert lo == "2026-03-21T12:00:00Z"
assert hi == "2026-03-28T12:00:00Z"
@time_machine.travel(datetime(2026, 3, 28, 12, 0, tzinfo=UTC), tick=False)
def test_bracket_minus_1_week(self) -> None:
lo, hi = _range(
translate_query("added:[-1 week to now]", UTC),
"added",
)
assert lo == "2026-03-21T12:00:00Z"
assert hi == "2026-03-28T12:00:00Z"
@time_machine.travel(datetime(2026, 3, 28, 12, 0, tzinfo=UTC), tick=False)
def test_bracket_minus_1_month_uses_relativedelta(self) -> None:
# relativedelta(months=1) from 2026-03-28 = 2026-02-28 (not 29)
lo, hi = _range(
translate_query("created:[-1 month to now]", UTC),
"created",
)
assert lo == "2026-02-28T12:00:00Z"
assert hi == "2026-03-28T12:00:00Z"
@time_machine.travel(datetime(2026, 3, 28, 12, 0, tzinfo=UTC), tick=False)
def test_bracket_minus_1_year(self) -> None:
lo, hi = _range(
translate_query("modified:[-1 year to now]", UTC),
"modified",
)
assert lo == "2025-03-28T12:00:00Z"
assert hi == "2026-03-28T12:00:00Z"
@time_machine.travel(datetime(2026, 3, 28, 12, 0, tzinfo=UTC), tick=False)
def test_bracket_plural_unit_hours(self) -> None:
lo, hi = _range(
translate_query("added:[-3 hours to now]", UTC),
"added",
)
assert lo == "2026-03-28T09:00:00Z"
assert hi == "2026-03-28T12:00:00Z"
@time_machine.travel(datetime(2026, 3, 28, 12, 0, tzinfo=UTC), tick=False)
def test_bracket_case_insensitive(self) -> None:
result = translate_query("added:[-1 WEEK TO NOW]", UTC)
assert "now" not in result.lower()
lo, hi = _range(result, "added")
assert lo == "2026-03-21T12:00:00Z"
assert hi == "2026-03-28T12:00:00Z"
@time_machine.travel(datetime(2026, 3, 28, 12, 0, tzinfo=UTC), tick=False)
def test_relative_range_swaps_bounds_when_lo_exceeds_hi(self) -> None:
# [now+1h TO now-1h] has lo > hi before substitution; they must be swapped
lo, hi = _range(
translate_query("added:[now+1h TO now-1h]", UTC),
"added",
)
assert lo == "2026-03-28T11:00:00Z"
assert hi == "2026-03-28T13:00:00Z"
def test_8digit_created_date_field_always_uses_utc_midnight(self) -> None:
# created is a DateField: boundaries are always UTC midnight, no TZ offset
result = translate_query("created:20231201", EASTERN)
lo, hi = _range(result, "created")
assert lo == "2023-12-01T00:00:00Z"
assert hi == "2023-12-02T00:00:00Z"
def test_8digit_added_datetime_field_converts_local_midnight_to_utc(self) -> None:
# added is DateTimeField: midnight Dec 1 Eastern (EST = UTC-5) = 05:00 UTC
result = translate_query("added:20231201", EASTERN)
lo, hi = _range(result, "added")
assert lo == "2023-12-01T05:00:00Z"
assert hi == "2023-12-02T05:00:00Z"
def test_8digit_modified_datetime_field_converts_local_midnight_to_utc(
self,
) -> None:
result = translate_query("modified:20231201", EASTERN)
lo, hi = _range(result, "modified")
assert lo == "2023-12-01T05:00:00Z"
assert hi == "2023-12-02T05:00:00Z"
def test_8digit_invalid_date_raises(self) -> None:
# The translation pipeline raises InvalidDateQuery for unparsable dates
# (e.g. month=13) so the API can surface a 400 telling the user the date
# is malformed instead of silently returning zero results.
with pytest.raises(InvalidDateQuery) as exc_info:
translate_query("added:20231340", UTC)
assert exc_info.value.field == "added"
assert exc_info.value.value == "20231340"
class TestParseUserQuery:
"""parse_user_query runs the full preprocessing pipeline."""
@@ -468,68 +73,6 @@ class TestParseUserQuery:
) -> None:
assert isinstance(parse_user_query(query_index, raw_query, UTC), tantivy.Query)
@pytest.mark.parametrize(
"raw_query",
[
# Partial date scalar (year only)
pytest.param("created:2020", id="created_year_scalar"),
# 8-digit compact date range in brackets
pytest.param(
"created:[20200101 TO 20201231]",
id="created_8digit_bracket_range",
),
# Comma-separated field + date range (Whoosh v2 multi-clause syntax)
pytest.param(
"title:x,created:[2020 TO 2021]",
id="title_comma_created_range",
),
# Field alias: type -> document_type
pytest.param("type:invoice", id="type_alias"),
# Multi-word date keyword, quoted. whoosh-compat's DateParserPlugin
# deliberately drops whoosh's "free" undelimited-date tagging mode
# (see whoosh_compat.parser.dateparse.DateParserPlugin docstring):
# unquoted "created:previous week" no longer parses (confirmed via
# whoosh_compat.parser.dateparse.English().date_from returning None
# for the bare "previous" token). This matches how the current
# frontend already emits this query (filter-editor.component.ts
# always quotes non-range relative date values), so the quoted
# form here reflects real traffic, not weakened coverage.
pytest.param('created:"previous week"', id="created_previous_week"),
# Full ISO datetime range ("T"/"Z" RFC3339 style). This exact query
# shape was added in PR #13010 to restore v2 (Whoosh) advanced-search
# compatibility, i.e. it represents real saved-search back-compat.
# whoosh-compat's date grammar now accepts "T" as a date/time
# separator and a trailing "Z" UTC designator (see
# whoosh_compat.parser.dateparse's English/_split_rfc3339_utc).
pytest.param(
"created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z]",
id="created_iso_range",
),
# Comma-separated ISO ranges (Whoosh v2 syntax) -- same shape as above.
pytest.param(
"created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z],"
"added:[2026-05-01T00:00:00Z TO 2026-06-01T00:00:00Z]",
id="comma_iso_ranges",
),
],
)
def test_advanced_search_queries_do_not_raise(
self,
query_index: tantivy.Index,
raw_query: str,
) -> None:
"""
End-to-end: queries that the frontend sends must parse without raising.
This tests the full pipeline: translate_query -> tantivy parse_query.
Equivalent to asserting HTTP 200 (not 400) for each query form.
"""
with time_machine.travel(datetime(2026, 6, 15, 12, 0, tzinfo=UTC), tick=False):
assert isinstance(
parse_user_query(query_index, raw_query, UTC),
tantivy.Query,
)
def test_invalid_date_propagates_not_swallowed(
self,
query_index: tantivy.Index,
@@ -598,230 +141,6 @@ class TestParseUserQuery:
assert isinstance(q, tantivy.Query)
class TestYearRangeRewriting:
"""Whoosh-style year-only date ranges must be rewritten to ISO 8601."""
@pytest.mark.parametrize(
("query", "field", "expected_lo", "expected_hi"),
[
pytest.param(
"created:[2020 TO 2020]",
"created",
"2020-01-01T00:00:00Z",
"2021-01-01T00:00:00Z",
id="single_year_created",
),
pytest.param(
"created:[2018 TO 2021]",
"created",
"2018-01-01T00:00:00Z",
"2022-01-01T00:00:00Z",
id="multi_year_range_created",
),
pytest.param(
"added:[2022 TO 2023]",
"added",
"2022-01-01T00:00:00Z",
"2024-01-01T00:00:00Z",
id="added_field",
),
pytest.param(
"modified:[2021 TO 2021]",
"modified",
"2021-01-01T00:00:00Z",
"2022-01-01T00:00:00Z",
id="modified_field",
),
pytest.param(
"created:[2020 to 2020]",
"created",
"2020-01-01T00:00:00Z",
"2021-01-01T00:00:00Z",
id="lowercase_to_keyword",
),
],
)
def test_year_range_rewritten(
self,
query: str,
field: str,
expected_lo: str,
expected_hi: str,
) -> None:
result = translate_query(query, UTC)
lo, hi = _range(result, field)
assert lo == expected_lo
assert hi == expected_hi
def test_reversed_year_range_is_swapped(self) -> None:
# A reversed range must not yield lo > hi, which Tantivy treats as an
# empty range (silently zero results). The bounds are swapped instead.
result = translate_query("created:[2025 TO 2020]", UTC)
lo, hi = _range(result, "created")
assert lo == "2020-01-01T00:00:00Z"
assert hi == "2026-01-01T00:00:00Z"
def test_year_range_in_complex_boolean_query(self) -> None:
query = "tag:steuer AND (title:2020 OR (NOT title:2019 AND NOT title:2018 AND created:[2020 TO 2020]))"
result = translate_query(query, UTC)
lo, hi = _range(result, "created")
assert lo == "2020-01-01T00:00:00Z"
assert hi == "2021-01-01T00:00:00Z"
assert "title:2020" in result
assert "title:2019" in result
assert "title:2018" in result
def test_already_iso_date_range_passes_through_unchanged(self) -> None:
original = "created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z]"
assert translate_query(original, UTC) == original
def test_8digit_in_brackets_not_matched_as_year_range(self) -> None:
# [YYYYMMDD TO YYYYMMDD]: the translation layer converts 8-digit bounds to
# ISO day ranges. 20200101 -> 2020-01-01T00:00:00Z (lo of that day);
# 20201231 -> the ceil of Dec 31 = 2021-01-01T00:00:00Z (exclusive end).
# This is the correct and accepted behavior: old compact form becomes a
# proper Tantivy-parseable ISO range.
original = "created:[20200101 TO 20201231]"
result = translate_query(original, UTC)
lo, hi = _range(result, "created")
assert lo == "2020-01-01T00:00:00Z"
assert hi == "2021-01-01T00:00:00Z"
class TestNonDateFieldsNotRewritten:
"""Date rewriters must only fire on the date fields (created/modified/added).
Integer fields like asn/id/page_count and unknown fields would otherwise be
rewritten into date ranges and rejected by Tantivy as type mismatches.
"""
@pytest.mark.parametrize(
"query",
[
pytest.param("asn:20240101", id="asn_8digit"),
pytest.param("id:20240101", id="id_8digit"),
pytest.param("page_count:12345678", id="page_count_8digit"),
pytest.param("num_notes:20231201", id="num_notes_8digit"),
],
)
def test_8digit_on_integer_field_passes_through_unchanged(self, query: str) -> None:
assert translate_query(query, EASTERN) == query
@pytest.mark.parametrize(
"query",
[
pytest.param("asn:[2000 TO 2024]", id="asn_year_range"),
pytest.param("id:[2000 TO 2024]", id="id_year_range"),
pytest.param("page_count:[2000 TO 2024]", id="page_count_year_range"),
],
)
def test_year_range_on_integer_field_passes_through_unchanged(
self,
query: str,
) -> None:
assert translate_query(query, UTC) == query
def test_unknown_field_keyword_passes_through_unchanged(self) -> None:
# foobar is not a date field: 'foobar:today' must not become a date range,
# which Tantivy would otherwise reject as an unknown/typed field.
assert translate_query("foobar:today", UTC) == "foobar:today"
class TestPassthrough:
"""Queries without field prefixes or unrelated content pass through unchanged."""
def test_bare_keyword_no_field_prefix_unchanged(self) -> None:
# Bare 'today' with no field: prefix passes through unchanged
result = translate_query("bank statement today", UTC)
assert "today" in result
def test_unrelated_query_unchanged(self) -> None:
assert translate_query("title:invoice", UTC) == "title:invoice"
class TestNormalizeQuery:
"""translate_query expands comma-separated values and collapses whitespace."""
def test_normalize_expands_comma_separated_tags(self) -> None:
assert translate_query("tag:foo,bar", UTC) == "tag:foo AND tag:bar"
def test_normalize_comma_between_range_expressions(self) -> None:
# Comma-separated field range expressions (Whoosh v2 syntax) must be
# converted to AND so Tantivy does not receive an invalid comma.
q = "created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z],added:[2026-05-01T00:00:00Z TO 2026-06-01T00:00:00Z]"
assert translate_query(q, UTC) == (
"created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z]"
" AND "
"added:[2026-05-01T00:00:00Z TO 2026-06-01T00:00:00Z]"
)
def test_normalize_expands_three_values(self) -> None:
assert (
translate_query("tag:foo,bar,baz", UTC) == "tag:foo AND tag:bar AND tag:baz"
)
def test_normalize_collapses_whitespace(self) -> None:
assert translate_query("bank statement", UTC) == "bank statement"
def test_normalize_no_commas_unchanged(self) -> None:
assert translate_query("bank statement", UTC) == "bank statement"
@pytest.mark.parametrize(
("raw", "expected"),
[
pytest.param(
"h52.1 - kurzsichtigkeit",
"h52.1 kurzsichtigkeit",
id="icd_code_dash_description",
),
pytest.param(
"H52.1 - asd",
"H52.1 asd",
id="icd_code_uppercase_dash",
),
pytest.param(
"h52.1 -",
"h52.1",
id="trailing_minus",
),
pytest.param(
". -",
".",
id="dot_trailing_minus",
),
pytest.param(
"h52. -",
"h52.",
id="partial_code_trailing_minus",
),
pytest.param(
"foo - bar - baz",
"foo bar baz",
id="multiple_dashes",
),
pytest.param(
"foo + bar",
"foo bar",
id="spaced_plus_operator",
),
],
)
def test_normalize_strips_dangling_operators(self, raw: str, expected: str) -> None:
assert translate_query(raw, UTC) == expected
@pytest.mark.parametrize(
"query",
[
pytest.param("term -other", id="adjacent_not_operator"),
pytest.param("-term", id="leading_not_operator"),
pytest.param("+term", id="leading_must_operator"),
pytest.param("foo -bar +baz", id="mixed_adjacent_operators"),
],
)
def test_normalize_preserves_valid_operators(self, query: str) -> None:
assert translate_query(query, UTC) == query
class TestParseSimpleTextHighlightQuery:
"""parse_simple_text_highlight_query must not raise on natural-language queries."""