From 7bab9622c8d3de061dcc776c1c453f1ff0c29dfa Mon Sep 17 00:00:00 2001 From: Trenton Holmes <797416+stumpylog@users.noreply.github.com> Date: Tue, 18 Aug 2026 08:55:16 -0700 Subject: [PATCH] fix(search): restore unquoted multi-word date keywords via pre-parse quoting "added:previous month" returned HTTP 400 after the whoosh-compat migration. The unquoted spelling was never parser-native anywhere: v2 rewrote it to explicit bracket ranges app-side before whoosh saw the string, and the deleted translation layer consumed it itself, so users and saved views have relied on it continuously while whoosh-compat deliberately scopes it out of its parser (its DIVERGENCES.md entry 19) and understands the phrases natively only as quoted values. parse_user_query now quotes the closed six-phrase vocabulary (previous week/month/quarter/year, this month/year) when it directly follows a date field's colon, before parsing. Only quoting happens app-side; every date computation stays in whoosh-compat's grammar, unlike v2's rewrite, which computed the ranges itself. Date field names derive from PUBLIC_FIELDS, the field name matches case-sensitively (the parser's own field tagging is case-sensitive), the phrase case-insensitively (the grammar accepts any case in the quoted form), and already-quoted spellings, TEXT fields, unfielded words and bracketed ranges are untouched. The previously xfailed end-to-end regression test now passes as a plain test, and a new acceptance class pins unquoted == quoted == mixed-case result sets on a boundary fixture, no-error parsing for the whole vocabulary across all three date fields, and that "title:previous month" stays an ordinary text search. docs/usage.md now states the two spellings are equivalent after a date field. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01WMsn6DgzbvSqh1pwy66VVF --- docs/usage.md | 5 +- src/documents/search/_query.py | 60 +++++++++++- src/documents/tests/search/test_acceptance.py | 92 +++++++++++++++++++ src/documents/tests/test_api_search.py | 13 --- 4 files changed, 154 insertions(+), 16 deletions(-) diff --git a/docs/usage.md b/docs/usage.md index 832ad1f1a..d1cd2b24c 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -923,8 +923,9 @@ produ*name Matching natural date keywords: -Multi-word date keywords must be quoted (e.g. `added:"previous month"`); an -unquoted multi-word keyword is not recognized as a date keyword. +Multi-word date keywords work quoted or unquoted after a date field +(`added:"previous month"` and `added:previous month` are equivalent); +elsewhere in a query the same words are treated as ordinary search text. ``` added:today diff --git a/src/documents/search/_query.py b/src/documents/search/_query.py index 9eeb02870..bb12dde79 100644 --- a/src/documents/search/_query.py +++ b/src/documents/search/_query.py @@ -13,6 +13,7 @@ from whoosh_compat.errors import Diagnostic from whoosh_compat.errors import DiagnosticKind from whoosh_compat.errors import UnsupportedQueryError +from documents.search._fields import PUBLIC_FIELDS from documents.search._registry import get_field_registry from documents.search._tokenizer import simple_search_tokens @@ -70,6 +71,59 @@ _REGEX_TIMEOUT: Final[float] = 1.0 # Uses Unicode properties to cover all blocks including Extension B+ planes. _CJK_RE: Final = regex.compile(r"[\p{Han}\p{Hiragana}\p{Katakana}\p{Hangul}]+") +# The closed multi-word date-keyword vocabulary, unchanged since paperless +# v2's rewrite_natural_date_keywords. whoosh-compat's date grammar +# understands every one of these natively, but only as a QUOTED value +# (its DIVERGENCES.md entry 19: unquoted multi-word values split at +# whitespace, faithfully to whoosh); the unquoted spelling has been +# honored continuously since the whoosh era by an app-level assist, so +# _quote_date_keyword_phrases below keeps honoring it by inserting the +# quotes and nothing else. Single-word keywords (today, yesterday) parse +# unquoted already and need no entry. +_DATE_KEYWORD_PHRASES: Final = ( + "previous week", + "previous month", + "previous quarter", + "previous year", + "this month", + "this year", +) + +# Field names are case-sensitive (matching the parser's own field +# tagging); the keyword phrase is case-insensitive (matching the date +# grammar's leniency for the quoted form). Date fields derived from +# PUBLIC_FIELDS, never hand-listed. +_DATE_KEYWORD_PHRASE_RE: Final = regex.compile( + r"\b(" + + "|".join( + regex.escape(f.name) + for f in PUBLIC_FIELDS + if f.kind in (wc.FieldKind.DATE, wc.FieldKind.DATETIME) + ) + + r"):((?i:" + + "|".join(_DATE_KEYWORD_PHRASES) + + r"))\b", +) + + +def _quote_date_keyword_phrases(raw_query: str) -> str: + """Quote unquoted multi-word date keyword phrases on date fields. + + ``added:previous month`` becomes ``added:"previous month"``; the + already-quoted spellings don't match the pattern (the colon must be + followed directly by the phrase), and the same words after a TEXT + field or standing alone are ordinary text and untouched. Only quoting + happens here: every date computation stays in whoosh-compat's + grammar, which parses exactly this phrase vocabulary as quoted + values. This is deliberately NOT a revival of the deleted + translation layer, which computed the ranges app-side. + """ + return _DATE_KEYWORD_PHRASE_RE.sub( + r'\1:"\2"', + raw_query, + timeout=_REGEX_TIMEOUT, + ) + def _has_cjk(text: str) -> bool: """Return True if text contains any CJK characters.""" @@ -285,7 +339,10 @@ def parse_user_query( """ Parse user query through whoosh-compat, then blend in fuzzy/CJK clauses. - 1. wc.parse() against the shared FieldRegistry (whoosh grammar -> AST). + 1. Unquoted multi-word date keyword phrases on date fields are quoted + (_quote_date_keyword_phrases) so the historically honored + "added:previous month" spelling keeps working; then wc.parse() + against the shared FieldRegistry (whoosh grammar -> AST). 2. Any diagnostics (bad dates/numbers) map to SearchQueryError subclasses and raise — the view returns HTTP 400 with every offending field listed, not just the first. @@ -303,6 +360,7 @@ def parse_user_query( never went through the pre-whoosh-compat translation layer either. """ registry = get_field_registry(settings.SEARCH_LANGUAGE) + raw_query = _quote_date_keyword_phrases(raw_query) result = wc.parse( raw_query, registry=registry, diff --git a/src/documents/tests/search/test_acceptance.py b/src/documents/tests/search/test_acceptance.py index eebf25244..807ca8a17 100644 --- a/src/documents/tests/search/test_acceptance.py +++ b/src/documents/tests/search/test_acceptance.py @@ -408,3 +408,95 @@ class TestFuzzyBlendSurvivesWhooshGrammar: ) backend.add_or_update(receipt_only) assert _matched_ids(backend, "added:today total NOT receipt") == set() + + +class TestUnquotedDateKeywordPhrases: + """The unquoted multi-word date keyword spelling (added:previous month) + has been honored continuously since the whoosh era, always by an + app-level assist, never by any parser: v2 rewrote it to a bracket range + before whoosh saw it, and the deleted _translate.py consumed it itself. + whoosh-compat scopes the unquoted form out of its parser on purpose + (its DIVERGENCES.md entry 19) but understands the quoted form natively, + so paperless quotes the closed phrase vocabulary on date fields before + parsing. Only quoting happens app-side; every date computation stays in + whoosh-compat.""" + + @pytest.fixture + def period_documents(self, backend: TantivyBackend) -> dict[str, int]: + with time_machine.travel(FROZEN_NOW, tick=False): + in_may = Document.objects.create( + title="May Doc", + content="statement", + checksum="kw-may", + archive_serial_number=910, + added=datetime(2026, 5, 20, 12, 0, tzinfo=UTC), + ) + in_june = Document.objects.create( + title="June Doc", + content="statement", + checksum="kw-june", + archive_serial_number=911, + added=datetime(2026, 6, 10, 12, 0, tzinfo=UTC), + ) + for doc in (in_may, in_june): + backend.add_or_update(doc) + return {"in_may": in_may.pk, "in_june": in_june.pk} + + @pytest.mark.parametrize( + "query", + [ + pytest.param("added:previous month", id="unquoted"), + pytest.param('added:"previous month"', id="quoted"), + pytest.param("added:Previous Month", id="unquoted-mixed-case"), + ], + ) + def test_unquoted_matches_the_same_documents_as_quoted( + self, + backend: TantivyBackend, + period_documents: dict[str, int], + query: str, + ) -> None: + with time_machine.travel(FROZEN_NOW, tick=False): + assert _matched_ids(backend, query) == {period_documents["in_may"]} + + @pytest.mark.parametrize( + "query", + [ + pytest.param("added:this month", id="this-month"), + pytest.param("added:this year", id="this-year"), + pytest.param("added:previous week", id="previous-week"), + pytest.param("added:previous quarter", id="previous-quarter"), + pytest.param("added:previous year", id="previous-year"), + pytest.param("created:previous month", id="created-field"), + pytest.param("modified:previous month", id="modified-field"), + ], + ) + def test_every_phrase_and_date_field_parses_without_error( + self, + backend: TantivyBackend, + period_documents: dict[str, int], + query: str, + ) -> None: + # The whole vocabulary times every date field must at least parse + # and search cleanly (no SearchQueryError -> no HTTP 400); exact + # window semantics are whoosh-compat's, pinned in its own suite. + with time_machine.travel(FROZEN_NOW, tick=False): + _matched_ids(backend, query) + + def test_text_field_keyword_words_are_not_rewritten( + self, + backend: TantivyBackend, + period_documents: dict[str, int], + ) -> None: + # "previous month" after a TEXT field (or unfielded) is ordinary + # text, not a date phrase: a title actually containing the words + # matches, and the date-window documents do not. + with time_machine.travel(FROZEN_NOW, tick=False): + wordy = Document.objects.create( + title="Notes from the previous month", + content="meeting notes", + checksum="kw-text", + archive_serial_number=912, + ) + backend.add_or_update(wordy) + assert _matched_ids(backend, "title:previous month") == {wordy.pk} diff --git a/src/documents/tests/test_api_search.py b/src/documents/tests/test_api_search.py index 8bbe4cf52..9d585855c 100644 --- a/src/documents/tests/test_api_search.py +++ b/src/documents/tests/test_api_search.py @@ -720,19 +720,6 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase): self.assertEqual(results[0]["id"], 3) self.assertEqual(results[0]["title"], "bank statement 3") - @pytest.mark.xfail( - reason=( - "whoosh-compat's DateParserPlugin intentionally drops whoosh's " - "'free' undelimited-date tagging mode (see " - "whoosh_compat.parser.dateparse.DateParserPlugin docstring), so " - "an unquoted multi-word date keyword like 'added:previous month' " - "no longer parses -- it now needs quoting ('added:\"previous " - "month\"'), unlike 'added:\"previous quarter\"' a few tests down " - "which already quotes. CONFIRMED REGRESSION vs whoosh-compat " - "migration; see task-10-report.md." - ), - raises=AssertionError, - ) def test_search_added_previous_month_excludes_next_period_start(self) -> None: """ GIVEN: