diff --git a/src/documents/search/_query.py b/src/documents/search/_query.py index 52cd55454..95ec38164 100644 --- a/src/documents/search/_query.py +++ b/src/documents/search/_query.py @@ -111,6 +111,40 @@ def _build_cjk_query( return None +def _try_parse_fuzzy_query( + index: tantivy.Index, + raw_query: str, +) -> tantivy.Query | None: + """Build the fuzzy blend clause from ``raw_query``, or None if it can't. + + The fuzzy blend hands ``raw_query`` directly to tantivy's own query + parser (there's no clean AST-level fuzzy equivalent to whoosh-compat's + parse tree, and fuzzy matching was always an approximate, secondary, + 0.1-boosted clause). But raw_query is whoosh grammar, not tantivy + grammar: it can contain date keywords (``today``), whoosh ranges + (``[2005 to 2009]``), or bracket-class wildcards (``202[0-1]*``) that + tantivy's parser rejects with a ValueError. Rather than let that escape + parse_user_query and fail the query's EXACT clause too (see paperless- + ngx's whoosh-compat migration regression), degrade gracefully: skip the + fuzzy clause and keep the exact/CJK clauses. Only ValueError is caught + — a broad except here would also hide real bugs. + """ + try: + return index.parse_query( + raw_query, + DEFAULT_SEARCH_FIELDS, + field_boosts=_FIELD_BOOSTS, + fuzzy_fields={f: (True, 1, True) for f in DEFAULT_SEARCH_FIELDS}, + ) + except ValueError: + logger.debug( + "Skipping fuzzy search clause: raw query is not valid tantivy " + "query syntax: %r", + raw_query, + ) + return None + + def build_permission_filter( schema: tantivy.Schema, user: AbstractBaseUser, @@ -233,9 +267,13 @@ def parse_user_query( 4. Optional fuzzy blend (ADVANCED_FUZZY_SEARCH_THRESHOLD) re-parses raw_query directly via index.parse_query — there's no clean AST-level fuzzy equivalent, and fuzzy matching was always an approximate, - secondary clause. + secondary clause. raw_query still carries whoosh grammar (date + keywords, bracket-class wildcards, etc.) that tantivy's own parser + cannot parse; when that happens the fuzzy clause is skipped rather + than letting the ValueError escape and fail the whole query (see + _try_parse_fuzzy_query). 5. Optional CJK bigram clause — unchanged from before this migration, - never went through the old translate_query() either. + never went through the pre-whoosh-compat translation layer either. """ registry = get_field_registry(settings.SEARCH_LANGUAGE) result = wc.parse( @@ -265,13 +303,11 @@ def parse_user_query( threshold = settings.ADVANCED_FUZZY_SEARCH_THRESHOLD if threshold is not None: - fuzzy = index.parse_query( - raw_query, - DEFAULT_SEARCH_FIELDS, - field_boosts=_FIELD_BOOSTS, - fuzzy_fields={f: (True, 1, True) for f in DEFAULT_SEARCH_FIELDS}, - ) - clauses.append((tantivy.Occur.Should, tantivy.Query.boost_query(fuzzy, 0.1))) + fuzzy = _try_parse_fuzzy_query(index, raw_query) + if fuzzy is not None: + clauses.append( + (tantivy.Occur.Should, tantivy.Query.boost_query(fuzzy, 0.1)), + ) if cjk_query is not None: clauses.append((tantivy.Occur.Should, cjk_query)) diff --git a/src/documents/tests/search/test_query.py b/src/documents/tests/search/test_query.py index d31364770..a1c63bd6e 100644 --- a/src/documents/tests/search/test_query.py +++ b/src/documents/tests/search/test_query.py @@ -36,13 +36,28 @@ class TestParseUserQuery: def test_returns_tantivy_query(self, query_index: tantivy.Index) -> None: assert isinstance(parse_user_query(query_index, "invoice", UTC), tantivy.Query) + @pytest.mark.parametrize( + "raw_query", + [ + pytest.param("invoice", id="plain_text"), + pytest.param("created:today", id="date_keyword"), + pytest.param("created:[2005 to 2009]", id="whoosh_date_range"), + pytest.param('added:"previous month"', id="quoted_date_phrase"), + pytest.param("title:202[0-1]*", id="bracket_class_wildcard"), + ], + ) def test_fuzzy_mode_does_not_raise( self, query_index: tantivy.Index, settings, + raw_query: str, ) -> None: + # These are all valid whoosh grammar that tantivy's own query parser + # (used only by the fuzzy blend clause) cannot parse; the fuzzy + # clause must degrade gracefully instead of raising and failing the + # whole query. See _try_parse_fuzzy_query. settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.5 - assert isinstance(parse_user_query(query_index, "invoice", UTC), tantivy.Query) + assert isinstance(parse_user_query(query_index, raw_query, UTC), tantivy.Query) def test_date_rewriting_applied_before_tantivy_parse( self,