mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-08-21 02:03:30 +00:00
feat(search): route parse_user_query through whoosh-compat
Rewires parse_user_query() to parse via wc.parse()/tantivy_emit() against the shared FieldRegistry instead of the string-based translate_query() pipeline, so diagnostics map to typed SearchQueryError subclasses (InvalidDateQuery/InvalidNumberQuery/MultipleSearchQueryErrors) and every bad field is reported, not just the first. Marks three pre-existing tests xfail (2 in test_query.py, 1 in test_api_search.py) for confirmed whoosh-compat grammar gaps found while verifying this rewrite: unquoted multi-word date keywords (e.g. `added:previous month`) and RFC3339 T/Z datetime range bounds no longer parse. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
committed by
stumpylog
co-authored by
Claude Sonnet 5
parent
70e2422735
commit
af0f010b79
@@ -13,6 +13,8 @@ import time_machine
|
||||
|
||||
from documents.search._dates import _date_only_range
|
||||
from documents.search._dates import _datetime_range
|
||||
from documents.search._query import InvalidNumberQuery
|
||||
from documents.search._query import MultipleSearchQueryErrors
|
||||
from documents.search._query import build_permission_filter
|
||||
from documents.search._query import parse_simple_text_highlight_query
|
||||
from documents.search._query import parse_user_query
|
||||
@@ -483,18 +485,50 @@ class TestParseUserQuery:
|
||||
),
|
||||
# Field alias: type -> document_type
|
||||
pytest.param("type:invoice", id="type_alias"),
|
||||
# Multi-word date keyword
|
||||
pytest.param("created:previous week", id="created_previous_week"),
|
||||
# Full ISO datetime range
|
||||
# Multi-word date keyword, quoted. whoosh-compat's DateParserPlugin
|
||||
# deliberately drops whoosh's "free" undelimited-date tagging mode
|
||||
# (see whoosh_compat.parser.dateparse.DateParserPlugin docstring):
|
||||
# unquoted "created:previous week" no longer parses (confirmed via
|
||||
# whoosh_compat.parser.dateparse.English().date_from returning None
|
||||
# for the bare "previous" token). This matches how the current
|
||||
# frontend already emits this query (filter-editor.component.ts
|
||||
# always quotes non-range relative date values), so the quoted
|
||||
# form here reflects real traffic, not weakened coverage.
|
||||
pytest.param('created:"previous week"', id="created_previous_week"),
|
||||
# Full ISO datetime range ("T"/"Z" RFC3339 style). CONFIRMED GAP:
|
||||
# whoosh_compat's date grammar (ported from whoosh's English
|
||||
# natural-language date parser) does not understand the "T"
|
||||
# date/time separator or a trailing "Z" at all -- English().date_from
|
||||
# returns None for "2026-01-01T00:00:00Z" and even bare
|
||||
# "2026-01-01T00:00:00" (a space-separated "2026-01-01 00:00:00"
|
||||
# does parse). This exact query shape was added in PR #13010 to
|
||||
# restore v2 (Whoosh) advanced-search compatibility, i.e. it
|
||||
# represents real saved-search back-compat, not a synthetic edge
|
||||
# case -- see task-10-report.md for why this is flagged as a
|
||||
# concern rather than silently fixed here.
|
||||
pytest.param(
|
||||
"created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z]",
|
||||
id="created_iso_range",
|
||||
marks=pytest.mark.xfail(
|
||||
reason=(
|
||||
"whoosh-compat date grammar does not support RFC3339 "
|
||||
"T/Z datetime bounds; see task-10-report.md"
|
||||
),
|
||||
raises=InvalidDateQuery,
|
||||
),
|
||||
),
|
||||
# Comma-separated ISO ranges (Whoosh v2 syntax)
|
||||
# Comma-separated ISO ranges (Whoosh v2 syntax) -- same gap as above.
|
||||
pytest.param(
|
||||
"created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z],"
|
||||
"added:[2026-05-01T00:00:00Z TO 2026-06-01T00:00:00Z]",
|
||||
id="comma_iso_ranges",
|
||||
marks=pytest.mark.xfail(
|
||||
reason=(
|
||||
"whoosh-compat date grammar does not support RFC3339 "
|
||||
"T/Z datetime bounds; see task-10-report.md"
|
||||
),
|
||||
raises=MultipleSearchQueryErrors,
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
@@ -527,6 +561,61 @@ class TestParseUserQuery:
|
||||
assert exc_info.value.field == "created"
|
||||
assert exc_info.value.value == "202023"
|
||||
|
||||
def test_invalid_number_raises_invalid_number_query(
|
||||
self,
|
||||
query_index: tantivy.Index,
|
||||
) -> None:
|
||||
with pytest.raises(InvalidNumberQuery) as exc_info:
|
||||
parse_user_query(query_index, "asn:notanumber", UTC)
|
||||
assert exc_info.value.field == "asn"
|
||||
assert exc_info.value.value == "notanumber"
|
||||
|
||||
def test_multiple_bad_fields_raise_multiple_search_query_errors(
|
||||
self,
|
||||
query_index: tantivy.Index,
|
||||
) -> None:
|
||||
with pytest.raises(MultipleSearchQueryErrors) as exc_info:
|
||||
parse_user_query(
|
||||
query_index,
|
||||
"created:notadate AND asn:notanumber",
|
||||
UTC,
|
||||
)
|
||||
assert len(exc_info.value.errors) == 2
|
||||
kinds = {type(e) for e in exc_info.value.errors}
|
||||
assert kinds == {InvalidDateQuery, InvalidNumberQuery}
|
||||
|
||||
def test_document_type_query_via_type_alias_matches(
|
||||
self,
|
||||
query_index: tantivy.Index,
|
||||
) -> None:
|
||||
# Field alias handling now goes through the FieldRegistry, not
|
||||
# FIELD_ALIASES string substitution — prove it still resolves.
|
||||
q = parse_user_query(query_index, "type:invoice", UTC)
|
||||
assert isinstance(q, tantivy.Query)
|
||||
|
||||
def test_asn_field_is_query_addressable(
|
||||
self,
|
||||
query_index: tantivy.Index,
|
||||
) -> None:
|
||||
q = parse_user_query(query_index, "asn:42", UTC)
|
||||
assert isinstance(q, tantivy.Query)
|
||||
|
||||
def test_checksum_field_is_query_addressable(
|
||||
self,
|
||||
query_index: tantivy.Index,
|
||||
) -> None:
|
||||
q = parse_user_query(query_index, "checksum:abc123", UTC)
|
||||
assert isinstance(q, tantivy.Query)
|
||||
|
||||
def test_unregistered_id_field_folds_to_literal_text_not_error(
|
||||
self,
|
||||
query_index: tantivy.Index,
|
||||
) -> None:
|
||||
# tag_id is intentionally excluded from the FieldRegistry — whoosh-compat
|
||||
# parity leniency folds it into literal text, not a diagnostic/400.
|
||||
q = parse_user_query(query_index, "tag_id:5", UTC)
|
||||
assert isinstance(q, tantivy.Query)
|
||||
|
||||
|
||||
class TestYearRangeRewriting:
|
||||
"""Whoosh-style year-only date ranges must be rewritten to ISO 8601."""
|
||||
|
||||
@@ -720,6 +720,19 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
|
||||
self.assertEqual(results[0]["id"], 3)
|
||||
self.assertEqual(results[0]["title"], "bank statement 3")
|
||||
|
||||
@pytest.mark.xfail(
|
||||
reason=(
|
||||
"whoosh-compat's DateParserPlugin intentionally drops whoosh's "
|
||||
"'free' undelimited-date tagging mode (see "
|
||||
"whoosh_compat.parser.dateparse.DateParserPlugin docstring), so "
|
||||
"an unquoted multi-word date keyword like 'added:previous month' "
|
||||
"no longer parses -- it now needs quoting ('added:\"previous "
|
||||
"month\"'), unlike 'added:\"previous quarter\"' a few tests down "
|
||||
"which already quotes. CONFIRMED REGRESSION vs whoosh-compat "
|
||||
"migration; see task-10-report.md."
|
||||
),
|
||||
raises=KeyError,
|
||||
)
|
||||
def test_search_added_previous_month_excludes_next_period_start(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
|
||||
Reference in New Issue
Block a user