mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-08-19 09:13:24 +00:00
fix(search): restore unquoted multi-word date keywords via pre-parse quoting
"added:previous month" returned HTTP 400 after the whoosh-compat migration. The unquoted spelling was never parser-native anywhere: v2 rewrote it to explicit bracket ranges app-side before whoosh saw the string, and the deleted translation layer consumed it itself, so users and saved views have relied on it continuously while whoosh-compat deliberately scopes it out of its parser (its DIVERGENCES.md entry 19) and understands the phrases natively only as quoted values. parse_user_query now quotes the closed six-phrase vocabulary (previous week/month/quarter/year, this month/year) when it directly follows a date field's colon, before parsing. Only quoting happens app-side; every date computation stays in whoosh-compat's grammar, unlike v2's rewrite, which computed the ranges itself. Date field names derive from PUBLIC_FIELDS, the field name matches case-sensitively (the parser's own field tagging is case-sensitive), the phrase case-insensitively (the grammar accepts any case in the quoted form), and already-quoted spellings, TEXT fields, unfielded words and bracketed ranges are untouched. The previously xfailed end-to-end regression test now passes as a plain test, and a new acceptance class pins unquoted == quoted == mixed-case result sets on a boundary fixture, no-error parsing for the whole vocabulary across all three date fields, and that "title:previous month" stays an ordinary text search. docs/usage.md now states the two spellings are equivalent after a date field. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01WMsn6DgzbvSqh1pwy66VVF
This commit is contained in:
co-authored by
Claude Fable 5
parent
a717684a60
commit
7bab9622c8
+3
-2
@@ -923,8 +923,9 @@ produ*name
|
||||
|
||||
Matching natural date keywords:
|
||||
|
||||
Multi-word date keywords must be quoted (e.g. `added:"previous month"`); an
|
||||
unquoted multi-word keyword is not recognized as a date keyword.
|
||||
Multi-word date keywords work quoted or unquoted after a date field
|
||||
(`added:"previous month"` and `added:previous month` are equivalent);
|
||||
elsewhere in a query the same words are treated as ordinary search text.
|
||||
|
||||
```
|
||||
added:today
|
||||
|
||||
@@ -13,6 +13,7 @@ from whoosh_compat.errors import Diagnostic
|
||||
from whoosh_compat.errors import DiagnosticKind
|
||||
from whoosh_compat.errors import UnsupportedQueryError
|
||||
|
||||
from documents.search._fields import PUBLIC_FIELDS
|
||||
from documents.search._registry import get_field_registry
|
||||
from documents.search._tokenizer import simple_search_tokens
|
||||
|
||||
@@ -70,6 +71,59 @@ _REGEX_TIMEOUT: Final[float] = 1.0
|
||||
# Uses Unicode properties to cover all blocks including Extension B+ planes.
|
||||
_CJK_RE: Final = regex.compile(r"[\p{Han}\p{Hiragana}\p{Katakana}\p{Hangul}]+")
|
||||
|
||||
# The closed multi-word date-keyword vocabulary, unchanged since paperless
|
||||
# v2's rewrite_natural_date_keywords. whoosh-compat's date grammar
|
||||
# understands every one of these natively, but only as a QUOTED value
|
||||
# (its DIVERGENCES.md entry 19: unquoted multi-word values split at
|
||||
# whitespace, faithfully to whoosh); the unquoted spelling has been
|
||||
# honored continuously since the whoosh era by an app-level assist, so
|
||||
# _quote_date_keyword_phrases below keeps honoring it by inserting the
|
||||
# quotes and nothing else. Single-word keywords (today, yesterday) parse
|
||||
# unquoted already and need no entry.
|
||||
_DATE_KEYWORD_PHRASES: Final = (
|
||||
"previous week",
|
||||
"previous month",
|
||||
"previous quarter",
|
||||
"previous year",
|
||||
"this month",
|
||||
"this year",
|
||||
)
|
||||
|
||||
# Field names are case-sensitive (matching the parser's own field
|
||||
# tagging); the keyword phrase is case-insensitive (matching the date
|
||||
# grammar's leniency for the quoted form). Date fields derived from
|
||||
# PUBLIC_FIELDS, never hand-listed.
|
||||
_DATE_KEYWORD_PHRASE_RE: Final = regex.compile(
|
||||
r"\b("
|
||||
+ "|".join(
|
||||
regex.escape(f.name)
|
||||
for f in PUBLIC_FIELDS
|
||||
if f.kind in (wc.FieldKind.DATE, wc.FieldKind.DATETIME)
|
||||
)
|
||||
+ r"):((?i:"
|
||||
+ "|".join(_DATE_KEYWORD_PHRASES)
|
||||
+ r"))\b",
|
||||
)
|
||||
|
||||
|
||||
def _quote_date_keyword_phrases(raw_query: str) -> str:
|
||||
"""Quote unquoted multi-word date keyword phrases on date fields.
|
||||
|
||||
``added:previous month`` becomes ``added:"previous month"``; the
|
||||
already-quoted spellings don't match the pattern (the colon must be
|
||||
followed directly by the phrase), and the same words after a TEXT
|
||||
field or standing alone are ordinary text and untouched. Only quoting
|
||||
happens here: every date computation stays in whoosh-compat's
|
||||
grammar, which parses exactly this phrase vocabulary as quoted
|
||||
values. This is deliberately NOT a revival of the deleted
|
||||
translation layer, which computed the ranges app-side.
|
||||
"""
|
||||
return _DATE_KEYWORD_PHRASE_RE.sub(
|
||||
r'\1:"\2"',
|
||||
raw_query,
|
||||
timeout=_REGEX_TIMEOUT,
|
||||
)
|
||||
|
||||
|
||||
def _has_cjk(text: str) -> bool:
|
||||
"""Return True if text contains any CJK characters."""
|
||||
@@ -285,7 +339,10 @@ def parse_user_query(
|
||||
"""
|
||||
Parse user query through whoosh-compat, then blend in fuzzy/CJK clauses.
|
||||
|
||||
1. wc.parse() against the shared FieldRegistry (whoosh grammar -> AST).
|
||||
1. Unquoted multi-word date keyword phrases on date fields are quoted
|
||||
(_quote_date_keyword_phrases) so the historically honored
|
||||
"added:previous month" spelling keeps working; then wc.parse()
|
||||
against the shared FieldRegistry (whoosh grammar -> AST).
|
||||
2. Any diagnostics (bad dates/numbers) map to SearchQueryError subclasses
|
||||
and raise — the view returns HTTP 400 with every offending field
|
||||
listed, not just the first.
|
||||
@@ -303,6 +360,7 @@ def parse_user_query(
|
||||
never went through the pre-whoosh-compat translation layer either.
|
||||
"""
|
||||
registry = get_field_registry(settings.SEARCH_LANGUAGE)
|
||||
raw_query = _quote_date_keyword_phrases(raw_query)
|
||||
result = wc.parse(
|
||||
raw_query,
|
||||
registry=registry,
|
||||
|
||||
@@ -408,3 +408,95 @@ class TestFuzzyBlendSurvivesWhooshGrammar:
|
||||
)
|
||||
backend.add_or_update(receipt_only)
|
||||
assert _matched_ids(backend, "added:today total NOT receipt") == set()
|
||||
|
||||
|
||||
class TestUnquotedDateKeywordPhrases:
|
||||
"""The unquoted multi-word date keyword spelling (added:previous month)
|
||||
has been honored continuously since the whoosh era, always by an
|
||||
app-level assist, never by any parser: v2 rewrote it to a bracket range
|
||||
before whoosh saw it, and the deleted _translate.py consumed it itself.
|
||||
whoosh-compat scopes the unquoted form out of its parser on purpose
|
||||
(its DIVERGENCES.md entry 19) but understands the quoted form natively,
|
||||
so paperless quotes the closed phrase vocabulary on date fields before
|
||||
parsing. Only quoting happens app-side; every date computation stays in
|
||||
whoosh-compat."""
|
||||
|
||||
@pytest.fixture
|
||||
def period_documents(self, backend: TantivyBackend) -> dict[str, int]:
|
||||
with time_machine.travel(FROZEN_NOW, tick=False):
|
||||
in_may = Document.objects.create(
|
||||
title="May Doc",
|
||||
content="statement",
|
||||
checksum="kw-may",
|
||||
archive_serial_number=910,
|
||||
added=datetime(2026, 5, 20, 12, 0, tzinfo=UTC),
|
||||
)
|
||||
in_june = Document.objects.create(
|
||||
title="June Doc",
|
||||
content="statement",
|
||||
checksum="kw-june",
|
||||
archive_serial_number=911,
|
||||
added=datetime(2026, 6, 10, 12, 0, tzinfo=UTC),
|
||||
)
|
||||
for doc in (in_may, in_june):
|
||||
backend.add_or_update(doc)
|
||||
return {"in_may": in_may.pk, "in_june": in_june.pk}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"query",
|
||||
[
|
||||
pytest.param("added:previous month", id="unquoted"),
|
||||
pytest.param('added:"previous month"', id="quoted"),
|
||||
pytest.param("added:Previous Month", id="unquoted-mixed-case"),
|
||||
],
|
||||
)
|
||||
def test_unquoted_matches_the_same_documents_as_quoted(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
period_documents: dict[str, int],
|
||||
query: str,
|
||||
) -> None:
|
||||
with time_machine.travel(FROZEN_NOW, tick=False):
|
||||
assert _matched_ids(backend, query) == {period_documents["in_may"]}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"query",
|
||||
[
|
||||
pytest.param("added:this month", id="this-month"),
|
||||
pytest.param("added:this year", id="this-year"),
|
||||
pytest.param("added:previous week", id="previous-week"),
|
||||
pytest.param("added:previous quarter", id="previous-quarter"),
|
||||
pytest.param("added:previous year", id="previous-year"),
|
||||
pytest.param("created:previous month", id="created-field"),
|
||||
pytest.param("modified:previous month", id="modified-field"),
|
||||
],
|
||||
)
|
||||
def test_every_phrase_and_date_field_parses_without_error(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
period_documents: dict[str, int],
|
||||
query: str,
|
||||
) -> None:
|
||||
# The whole vocabulary times every date field must at least parse
|
||||
# and search cleanly (no SearchQueryError -> no HTTP 400); exact
|
||||
# window semantics are whoosh-compat's, pinned in its own suite.
|
||||
with time_machine.travel(FROZEN_NOW, tick=False):
|
||||
_matched_ids(backend, query)
|
||||
|
||||
def test_text_field_keyword_words_are_not_rewritten(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
period_documents: dict[str, int],
|
||||
) -> None:
|
||||
# "previous month" after a TEXT field (or unfielded) is ordinary
|
||||
# text, not a date phrase: a title actually containing the words
|
||||
# matches, and the date-window documents do not.
|
||||
with time_machine.travel(FROZEN_NOW, tick=False):
|
||||
wordy = Document.objects.create(
|
||||
title="Notes from the previous month",
|
||||
content="meeting notes",
|
||||
checksum="kw-text",
|
||||
archive_serial_number=912,
|
||||
)
|
||||
backend.add_or_update(wordy)
|
||||
assert _matched_ids(backend, "title:previous month") == {wordy.pk}
|
||||
|
||||
@@ -720,19 +720,6 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
|
||||
self.assertEqual(results[0]["id"], 3)
|
||||
self.assertEqual(results[0]["title"], "bank statement 3")
|
||||
|
||||
@pytest.mark.xfail(
|
||||
reason=(
|
||||
"whoosh-compat's DateParserPlugin intentionally drops whoosh's "
|
||||
"'free' undelimited-date tagging mode (see "
|
||||
"whoosh_compat.parser.dateparse.DateParserPlugin docstring), so "
|
||||
"an unquoted multi-word date keyword like 'added:previous month' "
|
||||
"no longer parses -- it now needs quoting ('added:\"previous "
|
||||
"month\"'), unlike 'added:\"previous quarter\"' a few tests down "
|
||||
"which already quotes. CONFIRMED REGRESSION vs whoosh-compat "
|
||||
"migration; see task-10-report.md."
|
||||
),
|
||||
raises=AssertionError,
|
||||
)
|
||||
def test_search_added_previous_month_excludes_next_period_start(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
|
||||
Reference in New Issue
Block a user