make allow_infix order-stable

This commit is contained in:
shamoon
2026-08-02 17:50:45 -07:00
parent 6faa6368f6
commit 83ad96b1b3
2 changed files with 24 additions and 18 deletions
+10 -6
View File
@@ -148,10 +148,10 @@ def _build_simple_token_query(
) -> tantivy.Query:
escaped = regex.escape(token)
# The simple analyzer keeps punctuation inside whitespace-delimited terms.
# Later query tokens may therefore begin either at the indexed term boundary
# or after punctuation within a term (for example, ``medical-history``).
# Do not allow an arbitrary infix for later tokens: a query ending in ``6``
# must not match the middle of ``16``.
# Boundary-constrained query tokens may therefore begin either at the indexed
# term boundary or after punctuation within a term (for example,
# ``medical-history``). This avoids matching a numeric token such as ``6``
# in the middle of ``16``.
pattern = (
f".*{escaped}.*"
if allow_infix
@@ -282,10 +282,14 @@ def parse_simple_query(
index,
fields,
token,
allow_infix=idx == 0,
# Preserve historical infix matching for single-token
# searches. In multi-token searches, constrain numeric
# tokens to boundaries to avoid partial-number overlap.
# This depends on token content, not query order.
allow_infix=len(tokens) == 1 or not token.isdecimal(),
),
)
for idx, token in enumerate(tokens)
for token in tokens
]
simple_query = (
token_queries[0][1]
+14 -12
View File
@@ -163,10 +163,11 @@ class TestSearch:
assert (
len(backend.search_ids("sswo", user=None, search_mode=SearchMode.TEXT)) == 1
)
assert (
len(backend.search_ids("sswo re", user=None, search_mode=SearchMode.TEXT))
== 1
)
for query in ["sswo re", "re sswo"]:
assert (
len(backend.search_ids(query, user=None, search_mode=SearchMode.TEXT))
== 1
), query
def test_text_mode_matches_all_terms_without_requiring_adjacency(
self,
@@ -230,11 +231,11 @@ class TestSearch:
== 0
)
def test_text_mode_anchors_later_query_tokens_to_token_starts(
def test_text_mode_anchors_numeric_tokens_regardless_of_query_order(
self,
backend: TantivyBackend,
) -> None:
"""Multi-token simple search should not match later tokens in the middle of a word."""
"""Numeric tokens must not match in the middle of a larger number."""
exact_doc = Document.objects.create(
title="Z-Berichte 6",
content="monthly report",
@@ -257,13 +258,14 @@ class TestSearch:
backend.add_or_update(prefix_doc)
backend.add_or_update(false_positive)
result_ids = set(
backend.search_ids("Z-Berichte 6", user=None, search_mode=SearchMode.TEXT),
)
for query in ["Z-Berichte 6", "6 Z-Berichte"]:
result_ids = set(
backend.search_ids(query, user=None, search_mode=SearchMode.TEXT),
)
assert exact_doc.id in result_ids
assert prefix_doc.id in result_ids
assert false_positive.id not in result_ids
assert exact_doc.id in result_ids, query
assert prefix_doc.id in result_ids, query
assert false_positive.id not in result_ids, query
def test_text_mode_ignores_queries_without_searchable_tokens(
self,