From 7fddad849ab014a1c5900c9569437c35cbca70aa Mon Sep 17 00:00:00 2001 From: stumpylog <797416+stumpylog@users.noreply.github.com> Date: Thu, 20 Aug 2026 08:27:07 -0700 Subject: [PATCH] refactor(search): delete the date-keyword-phrase pre-parse rewrite whoosh-compat's grammar already accepts the closed multi-word date keyword vocabulary (previous month, this year, etc.) unquoted after a date field, making _quote_date_keyword_phrases redundant. Like its sibling rewrite removed in an earlier commit, it was not quote-aware and could insert quotes mid-phrase inside an unrelated quoted string (e.g. title:"see added:previous month notes"), corrupting the parse. Deleting it removes that hazard entirely. Docs are adjusted to scope the quoted-or-unquoted equivalence to the documented keyword list; other date expressions the grammar accepts (relative offsets, absolute dates) still require quoting. Co-Authored-By: Claude Opus 5 --- docs/usage.md | 8 ++- src/documents/search/_query.py | 59 +-------------- src/documents/tests/search/test_acceptance.py | 11 ++- .../test_date_keyword_phrase_removal.py | 72 +++++++++++++++++++ 4 files changed, 84 insertions(+), 66 deletions(-) create mode 100644 src/documents/tests/search/test_date_keyword_phrase_removal.py diff --git a/docs/usage.md b/docs/usage.md index 1611de109..9a6d7fd5f 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -936,9 +936,11 @@ continues past where stemming cuts a word off cannot match at all: Matching natural date keywords: -Multi-word date keywords work quoted or unquoted after a date field -(`added:"previous month"` and `added:previous month` are equivalent); -elsewhere in a query the same words are treated as ordinary search text. +The multi-word date keywords listed below work quoted or unquoted after a +date field (`added:"previous month"` and `added:previous month` are +equivalent); elsewhere in a query the same words are treated as ordinary +search text. Other date expressions the parser accepts (relative offsets +like `-1 week`, or specific dates like `12 december 2019`) must be quoted. ``` added:today diff --git a/src/documents/search/_query.py b/src/documents/search/_query.py index b6809ca65..62f6e0c92 100644 --- a/src/documents/search/_query.py +++ b/src/documents/search/_query.py @@ -18,7 +18,6 @@ from documents.search._errors import InvalidDateQuery from documents.search._errors import InvalidNumberQuery from documents.search._errors import MultipleSearchQueryErrors from documents.search._errors import SearchQueryError -from documents.search._fields import PUBLIC_FIELDS from documents.search._registry import get_field_registry from documents.search._tokenizer import simple_search_tokens @@ -35,56 +34,6 @@ _REGEX_TIMEOUT: Final[float] = 1.0 # Uses Unicode properties to cover all blocks including Extension B+ planes. _CJK_RE: Final = regex.compile(r"[\p{Han}\p{Hiragana}\p{Katakana}\p{Hangul}]+") -# Multi-word date-keyword phrases whoosh-compat only accepts quoted. -# Unquoted has always been the honored spelling, so -# _quote_date_keyword_phrases below inserts the quotes and nothing else. -# Single-word keywords (today, yesterday) already parse unquoted. -_DATE_KEYWORD_PHRASES: Final = ( - "previous week", - "previous month", - "previous quarter", - "previous year", - "this month", - "this year", -) - -# Field names are case-sensitive (matching the parser's own field -# tagging); the keyword phrase is case-insensitive (matching the date -# grammar's leniency for the quoted form). Date fields derived from -# PUBLIC_FIELDS, never hand-listed. -_DATE_KEYWORD_PHRASE_RE: Final = regex.compile( - r"\b(" - + "|".join( - regex.escape(f.name) - for f in PUBLIC_FIELDS - if f.kind in (wc.FieldKind.DATE, wc.FieldKind.DATETIME) - ) - + r"):((?i:" - + "|".join(_DATE_KEYWORD_PHRASES) - + r"))\b", -) - - -def _quote_date_keyword_phrases(raw_query: str) -> str: - """Quote unquoted multi-word date keyword phrases on date fields. - - ``added:previous month`` becomes ``added:"previous month"``; already- - quoted spellings, TEXT fields, and standalone words are untouched. - Only quoting happens here - every date computation stays in - whoosh-compat's grammar. - - Not quote-aware: matches anywhere in raw_query, including inside an - existing quoted phrase (e.g. ``title:"see added:previous month - notes"`` would get quotes inserted mid-phrase). Accepted as an - unlikely-in-practice edge case rather than implementing quote-aware - scanning. - """ - return _DATE_KEYWORD_PHRASE_RE.sub( - r'\1:"\2"', - raw_query, - timeout=_REGEX_TIMEOUT, - ) - def _user_facing_emit_message(d: Diagnostic) -> str: """A user-safe message for an emit-time QueryError's Diagnostic. @@ -314,11 +263,8 @@ def parse_user_query( """ Parse user query through whoosh-compat, then blend in fuzzy/CJK clauses. - 1. A small pre-parse rewrite keeps a historically honored spelling - working: unquoted multi-word date keyword phrases on date fields - are quoted (_quote_date_keyword_phrases). Then wc.parse() against - the shared FieldRegistry (whoosh grammar -> AST). Bare - notes:/custom_fields: prefixes resolve to their default subpath + 1. wc.parse() against the shared FieldRegistry (whoosh grammar -> AST). + Bare notes:/custom_fields: prefixes resolve to their default subpath (notes.note:/custom_fields.value:) directly in the registry, via each JSON field's SubpathSpec(default=True). 2. Any diagnostics (bad dates/numbers) map to SearchQueryError subclasses @@ -340,7 +286,6 @@ def parse_user_query( never went through the pre-whoosh-compat translation layer either. """ registry = get_field_registry(settings.SEARCH_LANGUAGE) - raw_query = _quote_date_keyword_phrases(raw_query) result = wc.parse( raw_query, registry=registry, diff --git a/src/documents/tests/search/test_acceptance.py b/src/documents/tests/search/test_acceptance.py index 22e500504..1a1557c71 100644 --- a/src/documents/tests/search/test_acceptance.py +++ b/src/documents/tests/search/test_acceptance.py @@ -267,11 +267,10 @@ class TestFuzzyBlendSurvivesWhooshGrammar: class TestUnquotedDateKeywordPhrases: - """The unquoted spelling (added:previous month) has always been - honored via an app-level quoting assist, since whoosh-compat's parser - only accepts the quoted form natively. paperless quotes the closed - phrase vocabulary on date fields before parsing; every date - computation still happens in whoosh-compat.""" + """The unquoted spelling (added:previous month) is honored natively by + whoosh-compat's own grammar for this closed phrase vocabulary — no + app-level rewrite is involved. Pins that the historically supported + spelling keeps working now that paperless no longer pre-quotes it.""" @pytest.fixture def period_documents(self, backend: TantivyBackend) -> dict[str, int]: @@ -335,7 +334,7 @@ class TestUnquotedDateKeywordPhrases: with time_machine.travel(FROZEN_NOW, tick=False): _matched_ids(backend, query) - def test_text_field_keyword_words_are_not_rewritten( + def test_text_field_keyword_words_are_ordinary_text( self, backend: TantivyBackend, period_documents: dict[str, int], diff --git a/src/documents/tests/search/test_date_keyword_phrase_removal.py b/src/documents/tests/search/test_date_keyword_phrase_removal.py new file mode 100644 index 000000000..06ddb3f3c --- /dev/null +++ b/src/documents/tests/search/test_date_keyword_phrase_removal.py @@ -0,0 +1,72 @@ +"""Pins the correctness gained by deleting the pre-parse +_quote_date_keyword_phrases rewrite. + +That rewrite matched date-keyword phrases (e.g. "previous month" after a +date field) anywhere in the raw query string, including inside an +unrelated quoted string, and inserted quotes mid-phrase there too — its +own docstring gave ``title:"see added:previous month notes"`` as the +example of what it corrupted. whoosh-compat's grammar accepts the same +phrase vocabulary unquoted natively (see TestUnquotedDateKeywordPhrases +in test_acceptance.py), so the rewrite was redundant everywhere it was +safe and actively wrong everywhere it was not. This is the one case that +tells the two apart: a literal title phrase that happens to contain +"added:previous month" as running text. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from documents.models import Document + +if TYPE_CHECKING: + from documents.search._backend import TantivyBackend + +pytestmark = [pytest.mark.search, pytest.mark.django_db] + + +def _matched_ids(backend: TantivyBackend, query: str) -> set[int]: + return set(backend.search_ids(query, user=None)) + + +def _index(backend: TantivyBackend, **kwargs: object) -> Document: + doc = Document.objects.create(**kwargs) + backend.add_or_update(doc) + return doc + + +class TestQuotedStringContainingDateKeywordText: + """A quoted title phrase containing the literal text + "added:previous month" as running words must match on that literal + text alone, never spill into an unfielded search for "previous" and + "month" across the default search fields the way the deleted rewrite + would have decomposed it into.""" + + def test_matches_only_the_literal_phrase( + self, + backend: TantivyBackend, + ) -> None: + literal = _index( + backend, + title="see added:previous month notes", + content="quarterly filing", + checksum="dkp-literal", + archive_serial_number=920, + ) + # Under the deleted rewrite, this decoy would incorrectly match: + # its title contains the "see added:" and " notes" fragments the + # corrupted parse required as title phrases, and its content + # supplies "previous" and "month" as the decomposed word-match + # clauses the rewrite turned the middle of the phrase into. + decoy = _index( + backend, + title="see added: quarterly report notes", + content="we reviewed the previous statement about month end", + checksum="dkp-decoy", + archive_serial_number=921, + ) + query = 'title:"see added:previous month notes"' + assert _matched_ids(backend, query) == {literal.pk} + assert decoy.pk not in _matched_ids(backend, query)