mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-08-26 04:33:20 +00:00
refactor(search): delete the date-keyword-phrase pre-parse rewrite
whoosh-compat's grammar already accepts the closed multi-word date keyword vocabulary (previous month, this year, etc.) unquoted after a date field, making _quote_date_keyword_phrases redundant. Like its sibling rewrite removed in an earlier commit, it was not quote-aware and could insert quotes mid-phrase inside an unrelated quoted string (e.g. title:"see added:previous month notes"), corrupting the parse. Deleting it removes that hazard entirely. Docs are adjusted to scope the quoted-or-unquoted equivalence to the documented keyword list; other date expressions the grammar accepts (relative offsets, absolute dates) still require quoting. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
0fb36dae43
commit
7fddad849a
+5
-3
@@ -936,9 +936,11 @@ continues past where stemming cuts a word off cannot match at all:
|
||||
|
||||
Matching natural date keywords:
|
||||
|
||||
Multi-word date keywords work quoted or unquoted after a date field
|
||||
(`added:"previous month"` and `added:previous month` are equivalent);
|
||||
elsewhere in a query the same words are treated as ordinary search text.
|
||||
The multi-word date keywords listed below work quoted or unquoted after a
|
||||
date field (`added:"previous month"` and `added:previous month` are
|
||||
equivalent); elsewhere in a query the same words are treated as ordinary
|
||||
search text. Other date expressions the parser accepts (relative offsets
|
||||
like `-1 week`, or specific dates like `12 december 2019`) must be quoted.
|
||||
|
||||
```
|
||||
added:today
|
||||
|
||||
@@ -18,7 +18,6 @@ from documents.search._errors import InvalidDateQuery
|
||||
from documents.search._errors import InvalidNumberQuery
|
||||
from documents.search._errors import MultipleSearchQueryErrors
|
||||
from documents.search._errors import SearchQueryError
|
||||
from documents.search._fields import PUBLIC_FIELDS
|
||||
from documents.search._registry import get_field_registry
|
||||
from documents.search._tokenizer import simple_search_tokens
|
||||
|
||||
@@ -35,56 +34,6 @@ _REGEX_TIMEOUT: Final[float] = 1.0
|
||||
# Uses Unicode properties to cover all blocks including Extension B+ planes.
|
||||
_CJK_RE: Final = regex.compile(r"[\p{Han}\p{Hiragana}\p{Katakana}\p{Hangul}]+")
|
||||
|
||||
# Multi-word date-keyword phrases whoosh-compat only accepts quoted.
|
||||
# Unquoted has always been the honored spelling, so
|
||||
# _quote_date_keyword_phrases below inserts the quotes and nothing else.
|
||||
# Single-word keywords (today, yesterday) already parse unquoted.
|
||||
_DATE_KEYWORD_PHRASES: Final = (
|
||||
"previous week",
|
||||
"previous month",
|
||||
"previous quarter",
|
||||
"previous year",
|
||||
"this month",
|
||||
"this year",
|
||||
)
|
||||
|
||||
# Field names are case-sensitive (matching the parser's own field
|
||||
# tagging); the keyword phrase is case-insensitive (matching the date
|
||||
# grammar's leniency for the quoted form). Date fields derived from
|
||||
# PUBLIC_FIELDS, never hand-listed.
|
||||
_DATE_KEYWORD_PHRASE_RE: Final = regex.compile(
|
||||
r"\b("
|
||||
+ "|".join(
|
||||
regex.escape(f.name)
|
||||
for f in PUBLIC_FIELDS
|
||||
if f.kind in (wc.FieldKind.DATE, wc.FieldKind.DATETIME)
|
||||
)
|
||||
+ r"):((?i:"
|
||||
+ "|".join(_DATE_KEYWORD_PHRASES)
|
||||
+ r"))\b",
|
||||
)
|
||||
|
||||
|
||||
def _quote_date_keyword_phrases(raw_query: str) -> str:
|
||||
"""Quote unquoted multi-word date keyword phrases on date fields.
|
||||
|
||||
``added:previous month`` becomes ``added:"previous month"``; already-
|
||||
quoted spellings, TEXT fields, and standalone words are untouched.
|
||||
Only quoting happens here - every date computation stays in
|
||||
whoosh-compat's grammar.
|
||||
|
||||
Not quote-aware: matches anywhere in raw_query, including inside an
|
||||
existing quoted phrase (e.g. ``title:"see added:previous month
|
||||
notes"`` would get quotes inserted mid-phrase). Accepted as an
|
||||
unlikely-in-practice edge case rather than implementing quote-aware
|
||||
scanning.
|
||||
"""
|
||||
return _DATE_KEYWORD_PHRASE_RE.sub(
|
||||
r'\1:"\2"',
|
||||
raw_query,
|
||||
timeout=_REGEX_TIMEOUT,
|
||||
)
|
||||
|
||||
|
||||
def _user_facing_emit_message(d: Diagnostic) -> str:
|
||||
"""A user-safe message for an emit-time QueryError's Diagnostic.
|
||||
@@ -314,11 +263,8 @@ def parse_user_query(
|
||||
"""
|
||||
Parse user query through whoosh-compat, then blend in fuzzy/CJK clauses.
|
||||
|
||||
1. A small pre-parse rewrite keeps a historically honored spelling
|
||||
working: unquoted multi-word date keyword phrases on date fields
|
||||
are quoted (_quote_date_keyword_phrases). Then wc.parse() against
|
||||
the shared FieldRegistry (whoosh grammar -> AST). Bare
|
||||
notes:/custom_fields: prefixes resolve to their default subpath
|
||||
1. wc.parse() against the shared FieldRegistry (whoosh grammar -> AST).
|
||||
Bare notes:/custom_fields: prefixes resolve to their default subpath
|
||||
(notes.note:/custom_fields.value:) directly in the registry, via
|
||||
each JSON field's SubpathSpec(default=True).
|
||||
2. Any diagnostics (bad dates/numbers) map to SearchQueryError subclasses
|
||||
@@ -340,7 +286,6 @@ def parse_user_query(
|
||||
never went through the pre-whoosh-compat translation layer either.
|
||||
"""
|
||||
registry = get_field_registry(settings.SEARCH_LANGUAGE)
|
||||
raw_query = _quote_date_keyword_phrases(raw_query)
|
||||
result = wc.parse(
|
||||
raw_query,
|
||||
registry=registry,
|
||||
|
||||
@@ -267,11 +267,10 @@ class TestFuzzyBlendSurvivesWhooshGrammar:
|
||||
|
||||
|
||||
class TestUnquotedDateKeywordPhrases:
|
||||
"""The unquoted spelling (added:previous month) has always been
|
||||
honored via an app-level quoting assist, since whoosh-compat's parser
|
||||
only accepts the quoted form natively. paperless quotes the closed
|
||||
phrase vocabulary on date fields before parsing; every date
|
||||
computation still happens in whoosh-compat."""
|
||||
"""The unquoted spelling (added:previous month) is honored natively by
|
||||
whoosh-compat's own grammar for this closed phrase vocabulary — no
|
||||
app-level rewrite is involved. Pins that the historically supported
|
||||
spelling keeps working now that paperless no longer pre-quotes it."""
|
||||
|
||||
@pytest.fixture
|
||||
def period_documents(self, backend: TantivyBackend) -> dict[str, int]:
|
||||
@@ -335,7 +334,7 @@ class TestUnquotedDateKeywordPhrases:
|
||||
with time_machine.travel(FROZEN_NOW, tick=False):
|
||||
_matched_ids(backend, query)
|
||||
|
||||
def test_text_field_keyword_words_are_not_rewritten(
|
||||
def test_text_field_keyword_words_are_ordinary_text(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
period_documents: dict[str, int],
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
"""Pins the correctness gained by deleting the pre-parse
|
||||
_quote_date_keyword_phrases rewrite.
|
||||
|
||||
That rewrite matched date-keyword phrases (e.g. "previous month" after a
|
||||
date field) anywhere in the raw query string, including inside an
|
||||
unrelated quoted string, and inserted quotes mid-phrase there too — its
|
||||
own docstring gave ``title:"see added:previous month notes"`` as the
|
||||
example of what it corrupted. whoosh-compat's grammar accepts the same
|
||||
phrase vocabulary unquoted natively (see TestUnquotedDateKeywordPhrases
|
||||
in test_acceptance.py), so the rewrite was redundant everywhere it was
|
||||
safe and actively wrong everywhere it was not. This is the one case that
|
||||
tells the two apart: a literal title phrase that happens to contain
|
||||
"added:previous month" as running text.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
|
||||
from documents.models import Document
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from documents.search._backend import TantivyBackend
|
||||
|
||||
pytestmark = [pytest.mark.search, pytest.mark.django_db]
|
||||
|
||||
|
||||
def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
|
||||
return set(backend.search_ids(query, user=None))
|
||||
|
||||
|
||||
def _index(backend: TantivyBackend, **kwargs: object) -> Document:
|
||||
doc = Document.objects.create(**kwargs)
|
||||
backend.add_or_update(doc)
|
||||
return doc
|
||||
|
||||
|
||||
class TestQuotedStringContainingDateKeywordText:
|
||||
"""A quoted title phrase containing the literal text
|
||||
"added:previous month" as running words must match on that literal
|
||||
text alone, never spill into an unfielded search for "previous" and
|
||||
"month" across the default search fields the way the deleted rewrite
|
||||
would have decomposed it into."""
|
||||
|
||||
def test_matches_only_the_literal_phrase(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
) -> None:
|
||||
literal = _index(
|
||||
backend,
|
||||
title="see added:previous month notes",
|
||||
content="quarterly filing",
|
||||
checksum="dkp-literal",
|
||||
archive_serial_number=920,
|
||||
)
|
||||
# Under the deleted rewrite, this decoy would incorrectly match:
|
||||
# its title contains the "see added:" and " notes" fragments the
|
||||
# corrupted parse required as title phrases, and its content
|
||||
# supplies "previous" and "month" as the decomposed word-match
|
||||
# clauses the rewrite turned the middle of the phrase into.
|
||||
decoy = _index(
|
||||
backend,
|
||||
title="see added: quarterly report notes",
|
||||
content="we reviewed the previous statement about month end",
|
||||
checksum="dkp-decoy",
|
||||
archive_serial_number=921,
|
||||
)
|
||||
query = 'title:"see added:previous month notes"'
|
||||
assert _matched_ids(backend, query) == {literal.pk}
|
||||
assert decoy.pk not in _matched_ids(backend, query)
|
||||
Reference in New Issue
Block a user