From eaa6dc1eedb7f89fa47bdfd56aef37157ccf95a0 Mon Sep 17 00:00:00 2001 From: Trenton Holmes <797416+stumpylog@users.noreply.github.com> Date: Mon, 17 Aug 2026 20:26:16 -0700 Subject: [PATCH] fix: skip fuzzy search blend when raw query isn't tantivy-parseable The fuzzy blend clause in parse_user_query() fed the raw, whoosh-syntax query string directly to tantivy's own query parser. Since the whoosh-compat migration, raw_query still contains whoosh grammar (date keywords, whoosh-style ranges, bracket-class wildcards) that tantivy's parser rejects with ValueError, which escaped parse_user_query and turned into a generic HTTP 400 for the entire query whenever ADVANCED_FUZZY_SEARCH_THRESHOLD was configured. Deriving a clean plain-text-only extraction for the fuzzy clause was ruled out: wc.parse() already expands unfielded terms into per-default- field copies in the AST, so there's no "still unfielded" marker left to walk without duplicating whoosh-compat's own expansion logic. Instead, scope a narrow try/except ValueError around exactly the index.parse_query() call and skip the fuzzy clause (logged at debug) when it can't parse, leaving the exact/CJK clauses unaffected. --- src/documents/search/_query.py | 54 ++++++++++++++++++++---- src/documents/tests/search/test_query.py | 17 +++++++- 2 files changed, 61 insertions(+), 10 deletions(-) diff --git a/src/documents/search/_query.py b/src/documents/search/_query.py index 52cd55454..95ec38164 100644 --- a/src/documents/search/_query.py +++ b/src/documents/search/_query.py @@ -111,6 +111,40 @@ def _build_cjk_query( return None +def _try_parse_fuzzy_query( + index: tantivy.Index, + raw_query: str, +) -> tantivy.Query | None: + """Build the fuzzy blend clause from ``raw_query``, or None if it can't. + + The fuzzy blend hands ``raw_query`` directly to tantivy's own query + parser (there's no clean AST-level fuzzy equivalent to whoosh-compat's + parse tree, and fuzzy matching was always an approximate, secondary, + 0.1-boosted clause). But raw_query is whoosh grammar, not tantivy + grammar: it can contain date keywords (``today``), whoosh ranges + (``[2005 to 2009]``), or bracket-class wildcards (``202[0-1]*``) that + tantivy's parser rejects with a ValueError. Rather than let that escape + parse_user_query and fail the query's EXACT clause too (see paperless- + ngx's whoosh-compat migration regression), degrade gracefully: skip the + fuzzy clause and keep the exact/CJK clauses. Only ValueError is caught + — a broad except here would also hide real bugs. + """ + try: + return index.parse_query( + raw_query, + DEFAULT_SEARCH_FIELDS, + field_boosts=_FIELD_BOOSTS, + fuzzy_fields={f: (True, 1, True) for f in DEFAULT_SEARCH_FIELDS}, + ) + except ValueError: + logger.debug( + "Skipping fuzzy search clause: raw query is not valid tantivy " + "query syntax: %r", + raw_query, + ) + return None + + def build_permission_filter( schema: tantivy.Schema, user: AbstractBaseUser, @@ -233,9 +267,13 @@ def parse_user_query( 4. Optional fuzzy blend (ADVANCED_FUZZY_SEARCH_THRESHOLD) re-parses raw_query directly via index.parse_query — there's no clean AST-level fuzzy equivalent, and fuzzy matching was always an approximate, - secondary clause. + secondary clause. raw_query still carries whoosh grammar (date + keywords, bracket-class wildcards, etc.) that tantivy's own parser + cannot parse; when that happens the fuzzy clause is skipped rather + than letting the ValueError escape and fail the whole query (see + _try_parse_fuzzy_query). 5. Optional CJK bigram clause — unchanged from before this migration, - never went through the old translate_query() either. + never went through the pre-whoosh-compat translation layer either. """ registry = get_field_registry(settings.SEARCH_LANGUAGE) result = wc.parse( @@ -265,13 +303,11 @@ def parse_user_query( threshold = settings.ADVANCED_FUZZY_SEARCH_THRESHOLD if threshold is not None: - fuzzy = index.parse_query( - raw_query, - DEFAULT_SEARCH_FIELDS, - field_boosts=_FIELD_BOOSTS, - fuzzy_fields={f: (True, 1, True) for f in DEFAULT_SEARCH_FIELDS}, - ) - clauses.append((tantivy.Occur.Should, tantivy.Query.boost_query(fuzzy, 0.1))) + fuzzy = _try_parse_fuzzy_query(index, raw_query) + if fuzzy is not None: + clauses.append( + (tantivy.Occur.Should, tantivy.Query.boost_query(fuzzy, 0.1)), + ) if cjk_query is not None: clauses.append((tantivy.Occur.Should, cjk_query)) diff --git a/src/documents/tests/search/test_query.py b/src/documents/tests/search/test_query.py index d31364770..a1c63bd6e 100644 --- a/src/documents/tests/search/test_query.py +++ b/src/documents/tests/search/test_query.py @@ -36,13 +36,28 @@ class TestParseUserQuery: def test_returns_tantivy_query(self, query_index: tantivy.Index) -> None: assert isinstance(parse_user_query(query_index, "invoice", UTC), tantivy.Query) + @pytest.mark.parametrize( + "raw_query", + [ + pytest.param("invoice", id="plain_text"), + pytest.param("created:today", id="date_keyword"), + pytest.param("created:[2005 to 2009]", id="whoosh_date_range"), + pytest.param('added:"previous month"', id="quoted_date_phrase"), + pytest.param("title:202[0-1]*", id="bracket_class_wildcard"), + ], + ) def test_fuzzy_mode_does_not_raise( self, query_index: tantivy.Index, settings, + raw_query: str, ) -> None: + # These are all valid whoosh grammar that tantivy's own query parser + # (used only by the fuzzy blend clause) cannot parse; the fuzzy + # clause must degrade gracefully instead of raising and failing the + # whole query. See _try_parse_fuzzy_query. settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.5 - assert isinstance(parse_user_query(query_index, "invoice", UTC), tantivy.Query) + assert isinstance(parse_user_query(query_index, raw_query, UTC), tantivy.Query) def test_date_rewriting_applied_before_tantivy_parse( self,