From cfa1d3b058157a60f71cb1286099e5d0c55f0891 Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Mon, 3 Aug 2026 08:03:03 -0700 Subject: [PATCH] Fix: correct multi-search non-adjacent queries (#13504) --- src/documents/search/_query.py | 78 +++++++++++++++------- src/documents/tests/search/test_backend.py | 68 ++++++++++++++++--- 2 files changed, 110 insertions(+), 36 deletions(-) diff --git a/src/documents/search/_query.py b/src/documents/search/_query.py index c45e37f80..3584767bc 100644 --- a/src/documents/search/_query.py +++ b/src/documents/search/_query.py @@ -139,31 +139,38 @@ def _simple_query_tokens(raw_query: str) -> list[str]: return simple_search_tokens(raw_query) -def _build_simple_field_query( +def _build_simple_token_query( index: tantivy.Index, - field: str, - tokens: list[str], + fields: list[str], + token: str, + *, + allow_infix: bool, ) -> tantivy.Query: - patterns = [] - for idx, token in enumerate(tokens): - escaped = regex.escape(token) - # For multi-token substring search, only the first token can begin mid-word. - # Later tokens follow a whitespace boundary in the original query, so anchor - # them to the start of the next indexed token to reduce false positives like - # matching "Z-Berichte 16" for the query "Z-Berichte 6". - if idx == 0: - patterns.append(f".*{escaped}.*") - else: - patterns.append(f"{escaped}.*") - if len(patterns) == 1: - query = tantivy.Query.regex_query(index.schema, field, patterns[0]) - else: - query = tantivy.Query.regex_phrase_query(index.schema, field, patterns) + escaped = regex.escape(token) + # The simple analyzer keeps punctuation inside whitespace-delimited terms. + # Boundary-constrained query tokens may therefore begin either at the indexed + # term boundary or after punctuation within a term (for example, + # ``medical-history``). This avoids matching a numeric token such as ``6`` + # in the middle of ``16``. + pattern = ( + f".*{escaped}.*" + if allow_infix + else ( + f"({escaped}.*|" + rf".*[\x20-\x2f\x3a-\x40\x5b-\x60\x7b-\x7e]{escaped}.*)" + ) + ) + field_queries: list[tuple[tantivy.Occur, tantivy.Query]] = [] + for field in fields: + query = tantivy.Query.regex_query(index.schema, field, pattern) + boost = _SIMPLE_FIELD_BOOSTS.get(field, 1.0) + if boost > 1.0: + query = tantivy.Query.boost_query(query, boost) + field_queries.append((tantivy.Occur.Should, query)) - boost = _SIMPLE_FIELD_BOOSTS.get(field, 1.0) - if boost > 1.0: - return tantivy.Query.boost_query(query, boost) - return query + if len(field_queries) == 1: + return field_queries[0][1] + return tantivy.Query.boolean_query(field_queries) def parse_user_query( @@ -265,10 +272,31 @@ def parse_simple_query( clauses: list[tuple[tantivy.Occur, tantivy.Query]] = [] if tokens: - clauses = [ - (tantivy.Occur.Should, _build_simple_field_query(index, field, tokens)) - for field in fields + # Match every query token, regardless of its position in the document. + # Each token may occur in any of the requested fields, so text mode also + # finds documents whose matches are split between title and content. + token_queries = [ + ( + tantivy.Occur.Must, + _build_simple_token_query( + index, + fields, + token, + # Preserve historical infix matching for single-token + # searches. In multi-token searches, constrain numeric + # tokens to boundaries to avoid partial-number overlap. + # This depends on token content, not query order. + allow_infix=len(tokens) == 1 or not token.isdecimal(), + ), + ) + for token in tokens ] + simple_query = ( + token_queries[0][1] + if len(token_queries) == 1 + else tantivy.Query.boolean_query(token_queries) + ) + clauses.append((tantivy.Occur.Should, simple_query)) if cjk_fields and _has_cjk(raw_query): cjk_q = _build_cjk_query(index, raw_query, cjk_fields) diff --git a/src/documents/tests/search/test_backend.py b/src/documents/tests/search/test_backend.py index e31708506..8615dc0ee 100644 --- a/src/documents/tests/search/test_backend.py +++ b/src/documents/tests/search/test_backend.py @@ -163,10 +163,55 @@ class TestSearch: assert ( len(backend.search_ids("sswo", user=None, search_mode=SearchMode.TEXT)) == 1 ) - assert ( - len(backend.search_ids("sswo re", user=None, search_mode=SearchMode.TEXT)) - == 1 + for query in ["sswo re", "re sswo"]: + assert ( + len(backend.search_ids(query, user=None, search_mode=SearchMode.TEXT)) + == 1 + ), query + + def test_text_mode_matches_all_terms_without_requiring_adjacency( + self, + backend: TantivyBackend, + ) -> None: + """Simple text mode should match all terms in any order or field.""" + doc = Document.objects.create( + title="complete-medical-history", + content="Samsung Odyssey curved monitor", + checksum="TXT13", + pk=19, ) + backend.add_or_update(doc) + + for query in [ + "complete history", + "history complete", + "Samsung curved", + "curved Samsung", + ]: + assert backend.search_ids( + query, + user=None, + search_mode=SearchMode.TEXT, + ) == [doc.pk], query + + def test_text_mode_matches_terms_across_title_and_content( + self, + backend: TantivyBackend, + ) -> None: + """Each simple-search term may match either title or content.""" + doc = Document.objects.create( + title="Complete record", + content="Patient history", + checksum="TXT14", + pk=20, + ) + backend.add_or_update(doc) + + assert backend.search_ids( + "complete history", + user=None, + search_mode=SearchMode.TEXT, + ) == [doc.pk] def test_text_mode_does_not_match_on_partial_term_overlap( self, @@ -186,11 +231,11 @@ class TestSearch: == 0 ) - def test_text_mode_anchors_later_query_tokens_to_token_starts( + def test_text_mode_anchors_numeric_tokens_regardless_of_query_order( self, backend: TantivyBackend, ) -> None: - """Multi-token simple search should not match later tokens in the middle of a word.""" + """Numeric tokens must not match in the middle of a larger number.""" exact_doc = Document.objects.create( title="Z-Berichte 6", content="monthly report", @@ -213,13 +258,14 @@ class TestSearch: backend.add_or_update(prefix_doc) backend.add_or_update(false_positive) - result_ids = set( - backend.search_ids("Z-Berichte 6", user=None, search_mode=SearchMode.TEXT), - ) + for query in ["Z-Berichte 6", "6 Z-Berichte"]: + result_ids = set( + backend.search_ids(query, user=None, search_mode=SearchMode.TEXT), + ) - assert exact_doc.id in result_ids - assert prefix_doc.id in result_ids - assert false_positive.id not in result_ids + assert exact_doc.id in result_ids, query + assert prefix_doc.id in result_ids, query + assert false_positive.id not in result_ids, query def test_text_mode_ignores_queries_without_searchable_tokens( self,