fix(search): restore unquoted multi-word date keywords via pre-parse quoting

"added:previous month" returned HTTP 400 after the whoosh-compat
migration. The unquoted spelling was never parser-native anywhere: v2
rewrote it to explicit bracket ranges app-side before whoosh saw the
string, and the deleted translation layer consumed it itself, so users
and saved views have relied on it continuously while whoosh-compat
deliberately scopes it out of its parser (its DIVERGENCES.md entry 19)
and understands the phrases natively only as quoted values.

parse_user_query now quotes the closed six-phrase vocabulary (previous
week/month/quarter/year, this month/year) when it directly follows a
date field's colon, before parsing. Only quoting happens app-side; every
date computation stays in whoosh-compat's grammar, unlike v2's rewrite,
which computed the ranges itself. Date field names derive from
PUBLIC_FIELDS, the field name matches case-sensitively (the parser's own
field tagging is case-sensitive), the phrase case-insensitively (the
grammar accepts any case in the quoted form), and already-quoted
spellings, TEXT fields, unfielded words and bracketed ranges are
untouched.

The previously xfailed end-to-end regression test now passes as a plain
test, and a new acceptance class pins unquoted == quoted == mixed-case
result sets on a boundary fixture, no-error parsing for the whole
vocabulary across all three date fields, and that "title:previous month"
stays an ordinary text search. docs/usage.md now states the two
spellings are equivalent after a date field.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01WMsn6DgzbvSqh1pwy66VVF
This commit is contained in:
Trenton Holmes
2026-08-18 11:05:04 -07:00
co-authored by Claude Fable 5
parent a717684a60
commit 7bab9622c8
4 changed files with 154 additions and 16 deletions
+59 -1
View File
@@ -13,6 +13,7 @@ from whoosh_compat.errors import Diagnostic
from whoosh_compat.errors import DiagnosticKind
from whoosh_compat.errors import UnsupportedQueryError
from documents.search._fields import PUBLIC_FIELDS
from documents.search._registry import get_field_registry
from documents.search._tokenizer import simple_search_tokens
@@ -70,6 +71,59 @@ _REGEX_TIMEOUT: Final[float] = 1.0
# Uses Unicode properties to cover all blocks including Extension B+ planes.
_CJK_RE: Final = regex.compile(r"[\p{Han}\p{Hiragana}\p{Katakana}\p{Hangul}]+")
# The closed multi-word date-keyword vocabulary, unchanged since paperless
# v2's rewrite_natural_date_keywords. whoosh-compat's date grammar
# understands every one of these natively, but only as a QUOTED value
# (its DIVERGENCES.md entry 19: unquoted multi-word values split at
# whitespace, faithfully to whoosh); the unquoted spelling has been
# honored continuously since the whoosh era by an app-level assist, so
# _quote_date_keyword_phrases below keeps honoring it by inserting the
# quotes and nothing else. Single-word keywords (today, yesterday) parse
# unquoted already and need no entry.
_DATE_KEYWORD_PHRASES: Final = (
"previous week",
"previous month",
"previous quarter",
"previous year",
"this month",
"this year",
)
# Field names are case-sensitive (matching the parser's own field
# tagging); the keyword phrase is case-insensitive (matching the date
# grammar's leniency for the quoted form). Date fields derived from
# PUBLIC_FIELDS, never hand-listed.
_DATE_KEYWORD_PHRASE_RE: Final = regex.compile(
r"\b("
+ "|".join(
regex.escape(f.name)
for f in PUBLIC_FIELDS
if f.kind in (wc.FieldKind.DATE, wc.FieldKind.DATETIME)
)
+ r"):((?i:"
+ "|".join(_DATE_KEYWORD_PHRASES)
+ r"))\b",
)
def _quote_date_keyword_phrases(raw_query: str) -> str:
"""Quote unquoted multi-word date keyword phrases on date fields.
``added:previous month`` becomes ``added:"previous month"``; the
already-quoted spellings don't match the pattern (the colon must be
followed directly by the phrase), and the same words after a TEXT
field or standing alone are ordinary text and untouched. Only quoting
happens here: every date computation stays in whoosh-compat's
grammar, which parses exactly this phrase vocabulary as quoted
values. This is deliberately NOT a revival of the deleted
translation layer, which computed the ranges app-side.
"""
return _DATE_KEYWORD_PHRASE_RE.sub(
r'\1:"\2"',
raw_query,
timeout=_REGEX_TIMEOUT,
)
def _has_cjk(text: str) -> bool:
"""Return True if text contains any CJK characters."""
@@ -285,7 +339,10 @@ def parse_user_query(
"""
Parse user query through whoosh-compat, then blend in fuzzy/CJK clauses.
1. wc.parse() against the shared FieldRegistry (whoosh grammar -> AST).
1. Unquoted multi-word date keyword phrases on date fields are quoted
(_quote_date_keyword_phrases) so the historically honored
"added:previous month" spelling keeps working; then wc.parse()
against the shared FieldRegistry (whoosh grammar -> AST).
2. Any diagnostics (bad dates/numbers) map to SearchQueryError subclasses
and raise — the view returns HTTP 400 with every offending field
listed, not just the first.
@@ -303,6 +360,7 @@ def parse_user_query(
never went through the pre-whoosh-compat translation layer either.
"""
registry = get_field_registry(settings.SEARCH_LANGUAGE)
raw_query = _quote_date_keyword_phrases(raw_query)
result = wc.parse(
raw_query,
registry=registry,
@@ -408,3 +408,95 @@ class TestFuzzyBlendSurvivesWhooshGrammar:
)
backend.add_or_update(receipt_only)
assert _matched_ids(backend, "added:today total NOT receipt") == set()
class TestUnquotedDateKeywordPhrases:
"""The unquoted multi-word date keyword spelling (added:previous month)
has been honored continuously since the whoosh era, always by an
app-level assist, never by any parser: v2 rewrote it to a bracket range
before whoosh saw it, and the deleted _translate.py consumed it itself.
whoosh-compat scopes the unquoted form out of its parser on purpose
(its DIVERGENCES.md entry 19) but understands the quoted form natively,
so paperless quotes the closed phrase vocabulary on date fields before
parsing. Only quoting happens app-side; every date computation stays in
whoosh-compat."""
@pytest.fixture
def period_documents(self, backend: TantivyBackend) -> dict[str, int]:
with time_machine.travel(FROZEN_NOW, tick=False):
in_may = Document.objects.create(
title="May Doc",
content="statement",
checksum="kw-may",
archive_serial_number=910,
added=datetime(2026, 5, 20, 12, 0, tzinfo=UTC),
)
in_june = Document.objects.create(
title="June Doc",
content="statement",
checksum="kw-june",
archive_serial_number=911,
added=datetime(2026, 6, 10, 12, 0, tzinfo=UTC),
)
for doc in (in_may, in_june):
backend.add_or_update(doc)
return {"in_may": in_may.pk, "in_june": in_june.pk}
@pytest.mark.parametrize(
"query",
[
pytest.param("added:previous month", id="unquoted"),
pytest.param('added:"previous month"', id="quoted"),
pytest.param("added:Previous Month", id="unquoted-mixed-case"),
],
)
def test_unquoted_matches_the_same_documents_as_quoted(
self,
backend: TantivyBackend,
period_documents: dict[str, int],
query: str,
) -> None:
with time_machine.travel(FROZEN_NOW, tick=False):
assert _matched_ids(backend, query) == {period_documents["in_may"]}
@pytest.mark.parametrize(
"query",
[
pytest.param("added:this month", id="this-month"),
pytest.param("added:this year", id="this-year"),
pytest.param("added:previous week", id="previous-week"),
pytest.param("added:previous quarter", id="previous-quarter"),
pytest.param("added:previous year", id="previous-year"),
pytest.param("created:previous month", id="created-field"),
pytest.param("modified:previous month", id="modified-field"),
],
)
def test_every_phrase_and_date_field_parses_without_error(
self,
backend: TantivyBackend,
period_documents: dict[str, int],
query: str,
) -> None:
# The whole vocabulary times every date field must at least parse
# and search cleanly (no SearchQueryError -> no HTTP 400); exact
# window semantics are whoosh-compat's, pinned in its own suite.
with time_machine.travel(FROZEN_NOW, tick=False):
_matched_ids(backend, query)
def test_text_field_keyword_words_are_not_rewritten(
self,
backend: TantivyBackend,
period_documents: dict[str, int],
) -> None:
# "previous month" after a TEXT field (or unfielded) is ordinary
# text, not a date phrase: a title actually containing the words
# matches, and the date-window documents do not.
with time_machine.travel(FROZEN_NOW, tick=False):
wordy = Document.objects.create(
title="Notes from the previous month",
content="meeting notes",
checksum="kw-text",
archive_serial_number=912,
)
backend.add_or_update(wordy)
assert _matched_ids(backend, "title:previous month") == {wordy.pk}
-13
View File
@@ -720,19 +720,6 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
self.assertEqual(results[0]["id"], 3)
self.assertEqual(results[0]["title"], "bank statement 3")
@pytest.mark.xfail(
reason=(
"whoosh-compat's DateParserPlugin intentionally drops whoosh's "
"'free' undelimited-date tagging mode (see "
"whoosh_compat.parser.dateparse.DateParserPlugin docstring), so "
"an unquoted multi-word date keyword like 'added:previous month' "
"no longer parses -- it now needs quoting ('added:\"previous "
"month\"'), unlike 'added:\"previous quarter\"' a few tests down "
"which already quotes. CONFIRMED REGRESSION vs whoosh-compat "
"migration; see task-10-report.md."
),
raises=AssertionError,
)
def test_search_added_previous_month_excludes_next_period_start(self) -> None:
"""
GIVEN: