mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-08-19 09:13:24 +00:00
fix: skip fuzzy search blend when raw query isn't tantivy-parseable
The fuzzy blend clause in parse_user_query() fed the raw, whoosh-syntax query string directly to tantivy's own query parser. Since the whoosh-compat migration, raw_query still contains whoosh grammar (date keywords, whoosh-style ranges, bracket-class wildcards) that tantivy's parser rejects with ValueError, which escaped parse_user_query and turned into a generic HTTP 400 for the entire query whenever ADVANCED_FUZZY_SEARCH_THRESHOLD was configured. Deriving a clean plain-text-only extraction for the fuzzy clause was ruled out: wc.parse() already expands unfielded terms into per-default- field copies in the AST, so there's no "still unfielded" marker left to walk without duplicating whoosh-compat's own expansion logic. Instead, scope a narrow try/except ValueError around exactly the index.parse_query() call and skip the fuzzy clause (logged at debug) when it can't parse, leaving the exact/CJK clauses unaffected.
This commit is contained in:
@@ -111,6 +111,40 @@ def _build_cjk_query(
|
||||
return None
|
||||
|
||||
|
||||
def _try_parse_fuzzy_query(
|
||||
index: tantivy.Index,
|
||||
raw_query: str,
|
||||
) -> tantivy.Query | None:
|
||||
"""Build the fuzzy blend clause from ``raw_query``, or None if it can't.
|
||||
|
||||
The fuzzy blend hands ``raw_query`` directly to tantivy's own query
|
||||
parser (there's no clean AST-level fuzzy equivalent to whoosh-compat's
|
||||
parse tree, and fuzzy matching was always an approximate, secondary,
|
||||
0.1-boosted clause). But raw_query is whoosh grammar, not tantivy
|
||||
grammar: it can contain date keywords (``today``), whoosh ranges
|
||||
(``[2005 to 2009]``), or bracket-class wildcards (``202[0-1]*``) that
|
||||
tantivy's parser rejects with a ValueError. Rather than let that escape
|
||||
parse_user_query and fail the query's EXACT clause too (see paperless-
|
||||
ngx's whoosh-compat migration regression), degrade gracefully: skip the
|
||||
fuzzy clause and keep the exact/CJK clauses. Only ValueError is caught
|
||||
— a broad except here would also hide real bugs.
|
||||
"""
|
||||
try:
|
||||
return index.parse_query(
|
||||
raw_query,
|
||||
DEFAULT_SEARCH_FIELDS,
|
||||
field_boosts=_FIELD_BOOSTS,
|
||||
fuzzy_fields={f: (True, 1, True) for f in DEFAULT_SEARCH_FIELDS},
|
||||
)
|
||||
except ValueError:
|
||||
logger.debug(
|
||||
"Skipping fuzzy search clause: raw query is not valid tantivy "
|
||||
"query syntax: %r",
|
||||
raw_query,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def build_permission_filter(
|
||||
schema: tantivy.Schema,
|
||||
user: AbstractBaseUser,
|
||||
@@ -233,9 +267,13 @@ def parse_user_query(
|
||||
4. Optional fuzzy blend (ADVANCED_FUZZY_SEARCH_THRESHOLD) re-parses
|
||||
raw_query directly via index.parse_query — there's no clean AST-level
|
||||
fuzzy equivalent, and fuzzy matching was always an approximate,
|
||||
secondary clause.
|
||||
secondary clause. raw_query still carries whoosh grammar (date
|
||||
keywords, bracket-class wildcards, etc.) that tantivy's own parser
|
||||
cannot parse; when that happens the fuzzy clause is skipped rather
|
||||
than letting the ValueError escape and fail the whole query (see
|
||||
_try_parse_fuzzy_query).
|
||||
5. Optional CJK bigram clause — unchanged from before this migration,
|
||||
never went through the old translate_query() either.
|
||||
never went through the pre-whoosh-compat translation layer either.
|
||||
"""
|
||||
registry = get_field_registry(settings.SEARCH_LANGUAGE)
|
||||
result = wc.parse(
|
||||
@@ -265,13 +303,11 @@ def parse_user_query(
|
||||
|
||||
threshold = settings.ADVANCED_FUZZY_SEARCH_THRESHOLD
|
||||
if threshold is not None:
|
||||
fuzzy = index.parse_query(
|
||||
raw_query,
|
||||
DEFAULT_SEARCH_FIELDS,
|
||||
field_boosts=_FIELD_BOOSTS,
|
||||
fuzzy_fields={f: (True, 1, True) for f in DEFAULT_SEARCH_FIELDS},
|
||||
)
|
||||
clauses.append((tantivy.Occur.Should, tantivy.Query.boost_query(fuzzy, 0.1)))
|
||||
fuzzy = _try_parse_fuzzy_query(index, raw_query)
|
||||
if fuzzy is not None:
|
||||
clauses.append(
|
||||
(tantivy.Occur.Should, tantivy.Query.boost_query(fuzzy, 0.1)),
|
||||
)
|
||||
|
||||
if cjk_query is not None:
|
||||
clauses.append((tantivy.Occur.Should, cjk_query))
|
||||
|
||||
@@ -36,13 +36,28 @@ class TestParseUserQuery:
|
||||
def test_returns_tantivy_query(self, query_index: tantivy.Index) -> None:
|
||||
assert isinstance(parse_user_query(query_index, "invoice", UTC), tantivy.Query)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"raw_query",
|
||||
[
|
||||
pytest.param("invoice", id="plain_text"),
|
||||
pytest.param("created:today", id="date_keyword"),
|
||||
pytest.param("created:[2005 to 2009]", id="whoosh_date_range"),
|
||||
pytest.param('added:"previous month"', id="quoted_date_phrase"),
|
||||
pytest.param("title:202[0-1]*", id="bracket_class_wildcard"),
|
||||
],
|
||||
)
|
||||
def test_fuzzy_mode_does_not_raise(
|
||||
self,
|
||||
query_index: tantivy.Index,
|
||||
settings,
|
||||
raw_query: str,
|
||||
) -> None:
|
||||
# These are all valid whoosh grammar that tantivy's own query parser
|
||||
# (used only by the fuzzy blend clause) cannot parse; the fuzzy
|
||||
# clause must degrade gracefully instead of raising and failing the
|
||||
# whole query. See _try_parse_fuzzy_query.
|
||||
settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.5
|
||||
assert isinstance(parse_user_query(query_index, "invoice", UTC), tantivy.Query)
|
||||
assert isinstance(parse_user_query(query_index, raw_query, UTC), tantivy.Query)
|
||||
|
||||
def test_date_rewriting_applied_before_tantivy_parse(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user