refactor(search): delete the date-keyword-phrase pre-parse rewrite

whoosh-compat's grammar already accepts the closed multi-word date
keyword vocabulary (previous month, this year, etc.) unquoted after a
date field, making _quote_date_keyword_phrases redundant. Like its
sibling rewrite removed in an earlier commit, it was not quote-aware
and could insert quotes mid-phrase inside an unrelated quoted string
(e.g. title:"see added:previous month notes"), corrupting the parse.
Deleting it removes that hazard entirely.

Docs are adjusted to scope the quoted-or-unquoted equivalence to the
documented keyword list; other date expressions the grammar accepts
(relative offsets, absolute dates) still require quoting.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
stumpylog
2026-08-20 08:27:07 -07:00
co-authored by Claude Opus 5
parent 0fb36dae43
commit 7fddad849a
4 changed files with 84 additions and 66 deletions
+5 -3
View File
@@ -936,9 +936,11 @@ continues past where stemming cuts a word off cannot match at all:
Matching natural date keywords:
Multi-word date keywords work quoted or unquoted after a date field
(`added:"previous month"` and `added:previous month` are equivalent);
elsewhere in a query the same words are treated as ordinary search text.
The multi-word date keywords listed below work quoted or unquoted after a
date field (`added:"previous month"` and `added:previous month` are
equivalent); elsewhere in a query the same words are treated as ordinary
search text. Other date expressions the parser accepts (relative offsets
like `-1 week`, or specific dates like `12 december 2019`) must be quoted.
```
added:today
+2 -57
View File
@@ -18,7 +18,6 @@ from documents.search._errors import InvalidDateQuery
from documents.search._errors import InvalidNumberQuery
from documents.search._errors import MultipleSearchQueryErrors
from documents.search._errors import SearchQueryError
from documents.search._fields import PUBLIC_FIELDS
from documents.search._registry import get_field_registry
from documents.search._tokenizer import simple_search_tokens
@@ -35,56 +34,6 @@ _REGEX_TIMEOUT: Final[float] = 1.0
# Uses Unicode properties to cover all blocks including Extension B+ planes.
_CJK_RE: Final = regex.compile(r"[\p{Han}\p{Hiragana}\p{Katakana}\p{Hangul}]+")
# Multi-word date-keyword phrases whoosh-compat only accepts quoted.
# Unquoted has always been the honored spelling, so
# _quote_date_keyword_phrases below inserts the quotes and nothing else.
# Single-word keywords (today, yesterday) already parse unquoted.
_DATE_KEYWORD_PHRASES: Final = (
"previous week",
"previous month",
"previous quarter",
"previous year",
"this month",
"this year",
)
# Field names are case-sensitive (matching the parser's own field
# tagging); the keyword phrase is case-insensitive (matching the date
# grammar's leniency for the quoted form). Date fields derived from
# PUBLIC_FIELDS, never hand-listed.
_DATE_KEYWORD_PHRASE_RE: Final = regex.compile(
r"\b("
+ "|".join(
regex.escape(f.name)
for f in PUBLIC_FIELDS
if f.kind in (wc.FieldKind.DATE, wc.FieldKind.DATETIME)
)
+ r"):((?i:"
+ "|".join(_DATE_KEYWORD_PHRASES)
+ r"))\b",
)
def _quote_date_keyword_phrases(raw_query: str) -> str:
"""Quote unquoted multi-word date keyword phrases on date fields.
``added:previous month`` becomes ``added:"previous month"``; already-
quoted spellings, TEXT fields, and standalone words are untouched.
Only quoting happens here - every date computation stays in
whoosh-compat's grammar.
Not quote-aware: matches anywhere in raw_query, including inside an
existing quoted phrase (e.g. ``title:"see added:previous month
notes"`` would get quotes inserted mid-phrase). Accepted as an
unlikely-in-practice edge case rather than implementing quote-aware
scanning.
"""
return _DATE_KEYWORD_PHRASE_RE.sub(
r'\1:"\2"',
raw_query,
timeout=_REGEX_TIMEOUT,
)
def _user_facing_emit_message(d: Diagnostic) -> str:
"""A user-safe message for an emit-time QueryError's Diagnostic.
@@ -314,11 +263,8 @@ def parse_user_query(
"""
Parse user query through whoosh-compat, then blend in fuzzy/CJK clauses.
1. A small pre-parse rewrite keeps a historically honored spelling
working: unquoted multi-word date keyword phrases on date fields
are quoted (_quote_date_keyword_phrases). Then wc.parse() against
the shared FieldRegistry (whoosh grammar -> AST). Bare
notes:/custom_fields: prefixes resolve to their default subpath
1. wc.parse() against the shared FieldRegistry (whoosh grammar -> AST).
Bare notes:/custom_fields: prefixes resolve to their default subpath
(notes.note:/custom_fields.value:) directly in the registry, via
each JSON field's SubpathSpec(default=True).
2. Any diagnostics (bad dates/numbers) map to SearchQueryError subclasses
@@ -340,7 +286,6 @@ def parse_user_query(
never went through the pre-whoosh-compat translation layer either.
"""
registry = get_field_registry(settings.SEARCH_LANGUAGE)
raw_query = _quote_date_keyword_phrases(raw_query)
result = wc.parse(
raw_query,
registry=registry,
@@ -267,11 +267,10 @@ class TestFuzzyBlendSurvivesWhooshGrammar:
class TestUnquotedDateKeywordPhrases:
"""The unquoted spelling (added:previous month) has always been
honored via an app-level quoting assist, since whoosh-compat's parser
only accepts the quoted form natively. paperless quotes the closed
phrase vocabulary on date fields before parsing; every date
computation still happens in whoosh-compat."""
"""The unquoted spelling (added:previous month) is honored natively by
whoosh-compat's own grammar for this closed phrase vocabulary — no
app-level rewrite is involved. Pins that the historically supported
spelling keeps working now that paperless no longer pre-quotes it."""
@pytest.fixture
def period_documents(self, backend: TantivyBackend) -> dict[str, int]:
@@ -335,7 +334,7 @@ class TestUnquotedDateKeywordPhrases:
with time_machine.travel(FROZEN_NOW, tick=False):
_matched_ids(backend, query)
def test_text_field_keyword_words_are_not_rewritten(
def test_text_field_keyword_words_are_ordinary_text(
self,
backend: TantivyBackend,
period_documents: dict[str, int],
@@ -0,0 +1,72 @@
"""Pins the correctness gained by deleting the pre-parse
_quote_date_keyword_phrases rewrite.
That rewrite matched date-keyword phrases (e.g. "previous month" after a
date field) anywhere in the raw query string, including inside an
unrelated quoted string, and inserted quotes mid-phrase there too — its
own docstring gave ``title:"see added:previous month notes"`` as the
example of what it corrupted. whoosh-compat's grammar accepts the same
phrase vocabulary unquoted natively (see TestUnquotedDateKeywordPhrases
in test_acceptance.py), so the rewrite was redundant everywhere it was
safe and actively wrong everywhere it was not. This is the one case that
tells the two apart: a literal title phrase that happens to contain
"added:previous month" as running text.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import pytest
from documents.models import Document
if TYPE_CHECKING:
from documents.search._backend import TantivyBackend
pytestmark = [pytest.mark.search, pytest.mark.django_db]
def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
return set(backend.search_ids(query, user=None))
def _index(backend: TantivyBackend, **kwargs: object) -> Document:
doc = Document.objects.create(**kwargs)
backend.add_or_update(doc)
return doc
class TestQuotedStringContainingDateKeywordText:
"""A quoted title phrase containing the literal text
"added:previous month" as running words must match on that literal
text alone, never spill into an unfielded search for "previous" and
"month" across the default search fields the way the deleted rewrite
would have decomposed it into."""
def test_matches_only_the_literal_phrase(
self,
backend: TantivyBackend,
) -> None:
literal = _index(
backend,
title="see added:previous month notes",
content="quarterly filing",
checksum="dkp-literal",
archive_serial_number=920,
)
# Under the deleted rewrite, this decoy would incorrectly match:
# its title contains the "see added:" and " notes" fragments the
# corrupted parse required as title phrases, and its content
# supplies "previous" and "month" as the decomposed word-match
# clauses the rewrite turned the middle of the phrase into.
decoy = _index(
backend,
title="see added: quarterly report notes",
content="we reviewed the previous statement about month end",
checksum="dkp-decoy",
archive_serial_number=921,
)
query = 'title:"see added:previous month notes"'
assert _matched_ids(backend, query) == {literal.pk}
assert decoy.pk not in _matched_ids(backend, query)