mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-08-30 22:47:15 +00:00
fix: skip fuzzy search blend when raw query isn't tantivy-parseable
The fuzzy blend clause in parse_user_query() fed the raw, whoosh-syntax query string directly to tantivy's own query parser. Since the whoosh-compat migration, raw_query still contains whoosh grammar (date keywords, whoosh-style ranges, bracket-class wildcards) that tantivy's parser rejects with ValueError, which escaped parse_user_query and turned into a generic HTTP 400 for the entire query whenever ADVANCED_FUZZY_SEARCH_THRESHOLD was configured. Deriving a clean plain-text-only extraction for the fuzzy clause was ruled out: wc.parse() already expands unfielded terms into per-default- field copies in the AST, so there's no "still unfielded" marker left to walk without duplicating whoosh-compat's own expansion logic. Instead, scope a narrow try/except ValueError around exactly the index.parse_query() call and skip the fuzzy clause (logged at debug) when it can't parse, leaving the exact/CJK clauses unaffected.
This commit is contained in:
@@ -111,6 +111,40 @@ def _build_cjk_query(
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _try_parse_fuzzy_query(
|
||||||
|
index: tantivy.Index,
|
||||||
|
raw_query: str,
|
||||||
|
) -> tantivy.Query | None:
|
||||||
|
"""Build the fuzzy blend clause from ``raw_query``, or None if it can't.
|
||||||
|
|
||||||
|
The fuzzy blend hands ``raw_query`` directly to tantivy's own query
|
||||||
|
parser (there's no clean AST-level fuzzy equivalent to whoosh-compat's
|
||||||
|
parse tree, and fuzzy matching was always an approximate, secondary,
|
||||||
|
0.1-boosted clause). But raw_query is whoosh grammar, not tantivy
|
||||||
|
grammar: it can contain date keywords (``today``), whoosh ranges
|
||||||
|
(``[2005 to 2009]``), or bracket-class wildcards (``202[0-1]*``) that
|
||||||
|
tantivy's parser rejects with a ValueError. Rather than let that escape
|
||||||
|
parse_user_query and fail the query's EXACT clause too (see paperless-
|
||||||
|
ngx's whoosh-compat migration regression), degrade gracefully: skip the
|
||||||
|
fuzzy clause and keep the exact/CJK clauses. Only ValueError is caught
|
||||||
|
— a broad except here would also hide real bugs.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
return index.parse_query(
|
||||||
|
raw_query,
|
||||||
|
DEFAULT_SEARCH_FIELDS,
|
||||||
|
field_boosts=_FIELD_BOOSTS,
|
||||||
|
fuzzy_fields={f: (True, 1, True) for f in DEFAULT_SEARCH_FIELDS},
|
||||||
|
)
|
||||||
|
except ValueError:
|
||||||
|
logger.debug(
|
||||||
|
"Skipping fuzzy search clause: raw query is not valid tantivy "
|
||||||
|
"query syntax: %r",
|
||||||
|
raw_query,
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def build_permission_filter(
|
def build_permission_filter(
|
||||||
schema: tantivy.Schema,
|
schema: tantivy.Schema,
|
||||||
user: AbstractBaseUser,
|
user: AbstractBaseUser,
|
||||||
@@ -233,9 +267,13 @@ def parse_user_query(
|
|||||||
4. Optional fuzzy blend (ADVANCED_FUZZY_SEARCH_THRESHOLD) re-parses
|
4. Optional fuzzy blend (ADVANCED_FUZZY_SEARCH_THRESHOLD) re-parses
|
||||||
raw_query directly via index.parse_query — there's no clean AST-level
|
raw_query directly via index.parse_query — there's no clean AST-level
|
||||||
fuzzy equivalent, and fuzzy matching was always an approximate,
|
fuzzy equivalent, and fuzzy matching was always an approximate,
|
||||||
secondary clause.
|
secondary clause. raw_query still carries whoosh grammar (date
|
||||||
|
keywords, bracket-class wildcards, etc.) that tantivy's own parser
|
||||||
|
cannot parse; when that happens the fuzzy clause is skipped rather
|
||||||
|
than letting the ValueError escape and fail the whole query (see
|
||||||
|
_try_parse_fuzzy_query).
|
||||||
5. Optional CJK bigram clause — unchanged from before this migration,
|
5. Optional CJK bigram clause — unchanged from before this migration,
|
||||||
never went through the old translate_query() either.
|
never went through the pre-whoosh-compat translation layer either.
|
||||||
"""
|
"""
|
||||||
registry = get_field_registry(settings.SEARCH_LANGUAGE)
|
registry = get_field_registry(settings.SEARCH_LANGUAGE)
|
||||||
result = wc.parse(
|
result = wc.parse(
|
||||||
@@ -265,13 +303,11 @@ def parse_user_query(
|
|||||||
|
|
||||||
threshold = settings.ADVANCED_FUZZY_SEARCH_THRESHOLD
|
threshold = settings.ADVANCED_FUZZY_SEARCH_THRESHOLD
|
||||||
if threshold is not None:
|
if threshold is not None:
|
||||||
fuzzy = index.parse_query(
|
fuzzy = _try_parse_fuzzy_query(index, raw_query)
|
||||||
raw_query,
|
if fuzzy is not None:
|
||||||
DEFAULT_SEARCH_FIELDS,
|
clauses.append(
|
||||||
field_boosts=_FIELD_BOOSTS,
|
(tantivy.Occur.Should, tantivy.Query.boost_query(fuzzy, 0.1)),
|
||||||
fuzzy_fields={f: (True, 1, True) for f in DEFAULT_SEARCH_FIELDS},
|
)
|
||||||
)
|
|
||||||
clauses.append((tantivy.Occur.Should, tantivy.Query.boost_query(fuzzy, 0.1)))
|
|
||||||
|
|
||||||
if cjk_query is not None:
|
if cjk_query is not None:
|
||||||
clauses.append((tantivy.Occur.Should, cjk_query))
|
clauses.append((tantivy.Occur.Should, cjk_query))
|
||||||
|
|||||||
@@ -36,13 +36,28 @@ class TestParseUserQuery:
|
|||||||
def test_returns_tantivy_query(self, query_index: tantivy.Index) -> None:
|
def test_returns_tantivy_query(self, query_index: tantivy.Index) -> None:
|
||||||
assert isinstance(parse_user_query(query_index, "invoice", UTC), tantivy.Query)
|
assert isinstance(parse_user_query(query_index, "invoice", UTC), tantivy.Query)
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"raw_query",
|
||||||
|
[
|
||||||
|
pytest.param("invoice", id="plain_text"),
|
||||||
|
pytest.param("created:today", id="date_keyword"),
|
||||||
|
pytest.param("created:[2005 to 2009]", id="whoosh_date_range"),
|
||||||
|
pytest.param('added:"previous month"', id="quoted_date_phrase"),
|
||||||
|
pytest.param("title:202[0-1]*", id="bracket_class_wildcard"),
|
||||||
|
],
|
||||||
|
)
|
||||||
def test_fuzzy_mode_does_not_raise(
|
def test_fuzzy_mode_does_not_raise(
|
||||||
self,
|
self,
|
||||||
query_index: tantivy.Index,
|
query_index: tantivy.Index,
|
||||||
settings,
|
settings,
|
||||||
|
raw_query: str,
|
||||||
) -> None:
|
) -> None:
|
||||||
|
# These are all valid whoosh grammar that tantivy's own query parser
|
||||||
|
# (used only by the fuzzy blend clause) cannot parse; the fuzzy
|
||||||
|
# clause must degrade gracefully instead of raising and failing the
|
||||||
|
# whole query. See _try_parse_fuzzy_query.
|
||||||
settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.5
|
settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.5
|
||||||
assert isinstance(parse_user_query(query_index, "invoice", UTC), tantivy.Query)
|
assert isinstance(parse_user_query(query_index, raw_query, UTC), tantivy.Query)
|
||||||
|
|
||||||
def test_date_rewriting_applied_before_tantivy_parse(
|
def test_date_rewriting_applied_before_tantivy_parse(
|
||||||
self,
|
self,
|
||||||
|
|||||||
Reference in New Issue
Block a user