mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-09-16 22:58:00 +00:00
Feature: parse advanced search with whoosh-compat and delete the hand-written translator
This commit is contained in:
stumpylog
+450
-145
@@ -6,22 +6,30 @@ from typing import Final
|
||||
|
||||
import regex
|
||||
import tantivy
|
||||
import whoosh_compat as wc
|
||||
from django.conf import settings
|
||||
from whoosh_compat.emitters.tantivy_ import emit as tantivy_emit
|
||||
from whoosh_compat.errors import Cause
|
||||
from whoosh_compat.errors import Diagnostic
|
||||
from whoosh_compat.errors import DiagnosticKind
|
||||
from whoosh_compat.errors import QueryError
|
||||
|
||||
from documents.search._errors import InvalidDateQuery
|
||||
from documents.search._errors import InvalidNumberQuery
|
||||
from documents.search._errors import MultipleSearchQueryErrors
|
||||
from documents.search._errors import SearchQueryError
|
||||
from documents.search._registry import get_field_registry
|
||||
from documents.search._tokenizer import simple_search_tokens
|
||||
from documents.search._translate import SearchQueryError
|
||||
from documents.search._translate import translate_query
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterable
|
||||
from datetime import tzinfo
|
||||
|
||||
from django.contrib.auth.base_user import AbstractBaseUser
|
||||
|
||||
logger = logging.getLogger("paperless.search")
|
||||
|
||||
# Maximum seconds any single regex substitution may run.
|
||||
# Prevents ReDoS on adversarial user-supplied query strings.
|
||||
# Maximum seconds any single regex substitution over user-supplied query text
|
||||
# may run. The one remaining use is a character class, which cannot backtrack,
|
||||
# so the bound is an upper limit on that substitution's cost, not the ReDoS
|
||||
# guard it was originally written as.
|
||||
_REGEX_TIMEOUT: Final[float] = 1.0
|
||||
|
||||
# Matches CJK/Hangul characters so queries can be routed to bigram fields.
|
||||
@@ -29,6 +37,64 @@ _REGEX_TIMEOUT: Final[float] = 1.0
|
||||
_CJK_RE: Final = regex.compile(r"[\p{Han}\p{Hiragana}\p{Katakana}\p{Hangul}]+")
|
||||
|
||||
|
||||
def _user_facing_emit_message(d: Diagnostic) -> str:
|
||||
"""A user-safe message for an emit-time QueryError's Diagnostic.
|
||||
|
||||
Built from the Diagnostic's structured fields (kind, field), never from
|
||||
d.message: whoosh-compat documents that as developer/log output with no
|
||||
stability guarantee, and PATTERN_TOO_COMPLEX embeds the raw backend
|
||||
error text in it.
|
||||
"""
|
||||
field = str(d.field) if d.field is not None else None
|
||||
if d.kind is DiagnosticKind.EXISTS_REQUIRES_FAST:
|
||||
return f"Existence searches (field:*) are not supported for field {field!r}."
|
||||
if d.kind is DiagnosticKind.TEXT_RANGE:
|
||||
return f"Range searches are not supported for field {field!r}."
|
||||
if d.kind is DiagnosticKind.PATTERN_TOO_COMPLEX:
|
||||
return f"The wildcard pattern for field {field!r} is too complex."
|
||||
if d.kind is DiagnosticKind.SCHEMA_FIELD_MISSING:
|
||||
return f"Field {field!r} is not available in the search index."
|
||||
logger.warning("Unmapped emit diagnostic %s: %s", d.kind, d.message)
|
||||
return "The search query could not be executed."
|
||||
|
||||
|
||||
def _map_emit_error(e: QueryError) -> SearchQueryError:
|
||||
"""Route an emit-time QueryError by its Diagnostic's Cause.
|
||||
|
||||
INVALID_INPUT/UNSUPPORTED are user-input errors, exactly like a parse
|
||||
diagnostic, and map to a 400. INTERNAL means a defect in whoosh-compat
|
||||
or in our own AST handling, never the user's query, so the QueryError is
|
||||
re-raised rather than converted, reaching the generic 500 handler instead
|
||||
of blaming the query. MISCONFIGURED is deliberately both: the registry and
|
||||
the index schema disagree, which only an operator can fix, so it is logged
|
||||
as an error, but a request is still waiting and the query cannot run
|
||||
either way, so it also returns a 400.
|
||||
|
||||
EXISTS_REQUIRES_FAST is the one MISCONFIGURED kind that is not a
|
||||
disagreement. whoosh-compat derives it from the registry's own FieldSpec
|
||||
(kind plus fast) without ever consulting the index schema, so it fires
|
||||
whenever a non-fast field of a kind that cannot answer "exists" is asked
|
||||
to: for us that is only the JSON fields, which field_descriptors() builds
|
||||
non-fast on purpose. "notes:*" and the five other spellings of it are
|
||||
ordinary user error that no operator action can clear, so they get the
|
||||
400 without the alert.
|
||||
"""
|
||||
d = e.diagnostic
|
||||
if d.cause is Cause.INTERNAL:
|
||||
raise e
|
||||
if (
|
||||
d.cause is Cause.MISCONFIGURED
|
||||
and d.kind is not DiagnosticKind.EXISTS_REQUIRES_FAST
|
||||
):
|
||||
logger.error(
|
||||
"Search index misconfiguration for field %s (%s): %s",
|
||||
d.field,
|
||||
d.kind.name,
|
||||
d.message,
|
||||
)
|
||||
return SearchQueryError(_user_facing_emit_message(d))
|
||||
|
||||
|
||||
def _has_cjk(text: str) -> bool:
|
||||
"""Return True if text contains any CJK characters."""
|
||||
return bool(_CJK_RE.search(text))
|
||||
@@ -37,14 +103,36 @@ def _has_cjk(text: str) -> bool:
|
||||
def extract_cjk_text(text: str) -> str:
|
||||
"""Join the CJK runs in ``text`` for indexing into bigram (char-ngram) fields.
|
||||
|
||||
Mirrors the query side (``_build_cjk_query``): only CJK runs are ever searched
|
||||
against the bigram fields, so only CJK runs are worth indexing there. Latin
|
||||
text fed to a character-bigram field is never matched and only bloats the
|
||||
Mirrors the query side, which extracts the CJK runs of whatever it is
|
||||
about to search for (the raw string in simple modes, the parsed query's
|
||||
free-text tokens in query mode): only CJK runs are ever searched against
|
||||
the bigram fields, so only CJK runs are worth indexing there. Latin text
|
||||
fed to a character-bigram field is never matched and only bloats the
|
||||
index and slows indexing/merge. Returns "" when there is no CJK text.
|
||||
"""
|
||||
return " ".join(_CJK_RE.findall(text))
|
||||
|
||||
|
||||
def _parse_cjk_text(
|
||||
index: tantivy.Index,
|
||||
cjk_text: str,
|
||||
fields: list[str],
|
||||
) -> tantivy.Query | None:
|
||||
"""Parse a plain CJK run string against ``fields``, or None if it won't parse."""
|
||||
try:
|
||||
return index.parse_query(cjk_text, fields)
|
||||
except Exception:
|
||||
# Broad on purpose, unlike _try_parse_fuzzy_query's narrower
|
||||
# ValueError: cjk_text isn't filtered to a guaranteed-safe token
|
||||
# set the way the fuzzy blend's word string is, so the exact
|
||||
# failure mode tantivy could raise here isn't pinned down.
|
||||
logger.debug(
|
||||
"Skipping CJK search clause: could not parse CJK text: %r",
|
||||
cjk_text,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _build_cjk_query(
|
||||
index: tantivy.Index,
|
||||
raw_query: str,
|
||||
@@ -52,91 +140,259 @@ def _build_cjk_query(
|
||||
) -> tantivy.Query | None:
|
||||
"""Build a bigram-field query from the CJK runs in ``raw_query``.
|
||||
|
||||
Only the CJK character runs are extracted and parsed; ASCII field prefixes,
|
||||
boolean operators and date keywords are discarded. This keeps the CJK clause
|
||||
plain-text and consistent across query/simple modes (no leaked ``field:``
|
||||
semantics, no parse failures from spaced ``-``/``+``), and avoids feeding
|
||||
Latin tokens into the character-bigram matcher (which would produce spurious
|
||||
matches against unrelated Latin text). Returns None when there is no CJK
|
||||
text or the parse fails.
|
||||
For the simple (TEXT/TITLE) modes, whose input is plain text and carries
|
||||
no query grammar to respect. Only the CJK character runs are extracted, so
|
||||
a stray ``field:`` prefix or ``-``/``+`` in the input can neither leak
|
||||
field semantics nor fail the parse, and no Latin token reaches the
|
||||
character-bigram matcher (where it would produce spurious matches against
|
||||
unrelated Latin text). Returns None when there is no CJK text or the parse
|
||||
fails.
|
||||
"""
|
||||
cjk_text = " ".join(_CJK_RE.findall(raw_query))
|
||||
cjk_text = extract_cjk_text(raw_query)
|
||||
if not cjk_text:
|
||||
return None
|
||||
return _parse_cjk_text(index, cjk_text, fields)
|
||||
|
||||
|
||||
def _build_ast_cjk_query(
|
||||
index: tantivy.Index,
|
||||
ast: wc.ast.Node,
|
||||
registry: wc.FieldRegistry,
|
||||
) -> tantivy.Query | None:
|
||||
"""Build the bigram clause of a QUERY-mode search from the parsed AST.
|
||||
|
||||
Same discipline as the fuzzy clause (see _try_parse_fuzzy_query): the CJK
|
||||
runs come from whoosh_compat's ``free_text_tokens`` over the parsed tree,
|
||||
never from the raw query string, so a term the user negated or restricted
|
||||
to a field outside the default search fields contributes nothing, instead
|
||||
of resurfacing as a top-level clause matching every bigram field.
|
||||
|
||||
``free_text_tokens`` reports no field of its own, so the tokens are
|
||||
collected one default field at a time: a bare term, which the parser has
|
||||
already copied onto every default field, is therefore searched across
|
||||
every bigram field, while ``title:東京`` reaches ``bigram_title`` alone.
|
||||
Fields whose CJK text is identical (the bare-term case) share a single
|
||||
parse over all of their bigram fields at once.
|
||||
|
||||
Raw (``analyzed=False``) tokens are used because the bigram fields have
|
||||
their own character-ngram analyzer: the default fields' word analyzers
|
||||
have no useful say over a CJK run, and running them first would only
|
||||
risk dropping it (remove_long) before the run is ever extracted.
|
||||
Returns None when the query has no CJK free text.
|
||||
"""
|
||||
fields_by_text: dict[str, list[str]] = {}
|
||||
for field, bigram_field in _CJK_BIGRAM_FIELDS.items():
|
||||
tokens = wc.free_text_tokens(
|
||||
ast,
|
||||
registry=registry,
|
||||
fields=[field],
|
||||
analyzed=False,
|
||||
)
|
||||
cjk_text = extract_cjk_text(" ".join(tokens))
|
||||
if cjk_text:
|
||||
fields_by_text.setdefault(cjk_text, []).append(bigram_field)
|
||||
|
||||
clauses: list[tuple[tantivy.Occur, tantivy.Query]] = [
|
||||
(tantivy.Occur.Should, query)
|
||||
for cjk_text, bigram_fields in fields_by_text.items()
|
||||
if (query := _parse_cjk_text(index, cjk_text, bigram_fields)) is not None
|
||||
]
|
||||
return _any_of(clauses) if clauses else None
|
||||
|
||||
|
||||
# A joined fuzzy word string must stay plain words: it goes back through
|
||||
# tantivy's own query parser, and the raw query text the clause collects
|
||||
# routinely carries characters that parser reads as grammar (a colon, a
|
||||
# bracket, a quote, a leading -). Each token is cut into its word runs and
|
||||
# only those are kept, so no field syntax, pattern, range or grouping can
|
||||
# reach the parser. Cutting rather than dropping the whole token is what
|
||||
# keeps ordinary hyphenated, dotted and quoted input ("COVID-19",
|
||||
# "hello@example.com", "tax reports") contributing to the clause at all.
|
||||
_WORD_RUN_RE = regex.compile(r"\w+")
|
||||
|
||||
# The one piece of tantivy grammar that survives the cut: its boolean
|
||||
# keywords are themselves word runs. Only these exact spellings are
|
||||
# grammar there ("And"/"and" are ordinary terms), so lowercasing exactly
|
||||
# these turns them back into the ordinary terms the field analyzer used to
|
||||
# make of them, before the clause switched to raw text. Left alone, a
|
||||
# quoted phrase would silently restructure the clause ("tax AND reports"
|
||||
# becoming a conjunction) or fail to parse and drop it entirely
|
||||
# ("tax AND", or "IN" anywhere).
|
||||
#
|
||||
# Only these words are touched: tantivy lowercases query terms with the
|
||||
# field's own analyzer, and doing it ourselves first is not always the
|
||||
# same operation (Python folds a final sigma to a different letter than
|
||||
# tantivy does, and turns Turkish 'İ' into a sequence tantivy then splits
|
||||
# in two), which would search for terms the index does not contain.
|
||||
_TANTIVY_KEYWORDS: Final[frozenset[str]] = frozenset({"AND", "OR", "NOT", "IN"})
|
||||
|
||||
|
||||
def _try_parse_fuzzy_query(
|
||||
index: tantivy.Index,
|
||||
ast: wc.ast.Node,
|
||||
registry: wc.FieldRegistry,
|
||||
) -> tantivy.Query | None:
|
||||
"""Build the fuzzy blend clause from the parsed query's free-text
|
||||
words, or None if it has none.
|
||||
|
||||
The clause is built by handing tantivy's own query parser a plain
|
||||
word string (there's no clean AST-level fuzzy equivalent to
|
||||
whoosh-compat's parse tree, and fuzzy matching was always an
|
||||
approximate, secondary, 0.1-boosted clause). The words come from
|
||||
whoosh_compat's ``free_text_tokens`` over the already-parsed AST,
|
||||
never from the raw query string: raw whoosh grammar (date keywords,
|
||||
``[2005 to 2009]`` ranges, bracket-class wildcards) is not tantivy
|
||||
syntax, and feeding it here used to knock the fuzzy clause out for
|
||||
the whole query the moment any such construct appeared alongside a
|
||||
typo'd word. The helper also keeps excluded terms out: a ``NOT``'d
|
||||
word must not resurface through the fuzzy clause.
|
||||
|
||||
Chosen trade-off: a term explicitly fielded on one of the default
|
||||
search fields (``correspondent:acme``) contributes its text to the
|
||||
word string UNFIELDED, so the fuzzy clause searches it across all
|
||||
default fields rather than just the one the user named. That is
|
||||
recall-only widening on a secondary 0.1-boosted clause the score
|
||||
threshold already disciplines, accepted in exchange for never feeding
|
||||
field syntax to tantivy's parser. What the word string guarantees is
|
||||
exactly that: no field prefix, pattern, range, grouping or quoting
|
||||
survives, and the boolean keywords that do survive (they are word
|
||||
runs) are lowercased into ordinary terms; see _TANTIVY_KEYWORDS.
|
||||
|
||||
The words are the query's RAW text, not the analyzer's output
|
||||
(``analyzed=False``), because ``index.parse_query`` analyzes whatever
|
||||
it is given and analysis is not idempotent: ``universities`` stems to
|
||||
``univers``, and handing that back stems it again to ``univ``, a term
|
||||
the index does not contain. ``prefix=True`` hid this as over-broad
|
||||
matching (``univ`` also prefixes ``unicycle``) rather than as no
|
||||
matches at all. Raw text is untokenized, which is why it is cut into
|
||||
word runs above rather than taken whole.
|
||||
|
||||
The ValueError guard stays as insurance (the word string is plain
|
||||
tokens, so tantivy accepting it is expected, not assumed): on a parse
|
||||
failure the fuzzy clause is skipped and the exact/CJK clauses stand,
|
||||
rather than the whole query failing.
|
||||
"""
|
||||
tokens = wc.free_text_tokens(
|
||||
ast,
|
||||
registry=registry,
|
||||
fields=_DEFAULT_SEARCH_FIELDS,
|
||||
analyzed=False,
|
||||
)
|
||||
words = list(
|
||||
dict.fromkeys(
|
||||
word.lower() if word in _TANTIVY_KEYWORDS else word
|
||||
for token in tokens
|
||||
for word in _WORD_RUN_RE.findall(token)
|
||||
),
|
||||
)
|
||||
if not words:
|
||||
return None
|
||||
fuzzy_text = " ".join(words)
|
||||
try:
|
||||
return index.parse_query(cjk_text, fields)
|
||||
except Exception:
|
||||
return index.parse_query(
|
||||
fuzzy_text,
|
||||
_DEFAULT_SEARCH_FIELDS,
|
||||
field_boosts=_FIELD_BOOSTS,
|
||||
fuzzy_fields={f: (True, 1, True) for f in _DEFAULT_SEARCH_FIELDS},
|
||||
)
|
||||
except ValueError:
|
||||
logger.debug(
|
||||
"Skipping fuzzy search clause: token string is not valid "
|
||||
"tantivy query syntax: %r",
|
||||
fuzzy_text,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def build_permission_filter(
|
||||
schema: tantivy.Schema,
|
||||
user: AbstractBaseUser,
|
||||
viewer_group_ids: Iterable[int] = (),
|
||||
) -> tantivy.Query:
|
||||
"""
|
||||
Build a query filter for user document permissions.
|
||||
|
||||
Creates a query that matches only documents visible to the specified user
|
||||
according to paperless-ngx permission rules:
|
||||
- Public documents (no owner) are visible to all users
|
||||
- Private documents are visible to their owner
|
||||
- Documents explicitly shared with the user are visible
|
||||
- Documents shared with one of the user's current groups are visible
|
||||
|
||||
Args:
|
||||
schema: Tantivy schema for field validation
|
||||
user: User to check permissions for
|
||||
viewer_group_ids: Current group memberships for the user
|
||||
|
||||
Returns:
|
||||
Tantivy query that filters results to visible documents
|
||||
"""
|
||||
owner_any = tantivy.Query.exists_query("owner_id")
|
||||
no_owner = tantivy.Query.boolean_query(
|
||||
[
|
||||
(tantivy.Occur.Must, tantivy.Query.all_query()),
|
||||
(tantivy.Occur.MustNot, owner_any),
|
||||
],
|
||||
)
|
||||
owned = tantivy.Query.term_query(schema, "owner_id", user.pk)
|
||||
shared = tantivy.Query.term_query(schema, "viewer_id", user.pk)
|
||||
group_shared = [
|
||||
tantivy.Query.term_query(schema, "viewer_group_id", group_id)
|
||||
for group_id in viewer_group_ids
|
||||
]
|
||||
return tantivy.Query.disjunction_max_query(
|
||||
[no_owner, owned, shared, *group_shared],
|
||||
)
|
||||
|
||||
|
||||
DEFAULT_SEARCH_FIELDS = [
|
||||
_DEFAULT_SEARCH_FIELDS: Final[list[str]] = [
|
||||
"title",
|
||||
"content",
|
||||
"correspondent",
|
||||
"document_type",
|
||||
"tag",
|
||||
]
|
||||
SIMPLE_SEARCH_FIELDS = ["simple_title", "simple_content"]
|
||||
TITLE_SEARCH_FIELDS = ["simple_title"]
|
||||
_CJK_ALL_FIELDS: Final[list[str]] = [
|
||||
"bigram_content",
|
||||
"bigram_title",
|
||||
"bigram_correspondent",
|
||||
"bigram_document_type",
|
||||
"bigram_tag",
|
||||
]
|
||||
_SIMPLE_SEARCH_FIELDS: Final[list[str]] = ["simple_title", "simple_content"]
|
||||
_TITLE_SEARCH_FIELDS: Final[list[str]] = ["simple_title"]
|
||||
# The bigram (character-ngram) companion of each default search field.
|
||||
_CJK_BIGRAM_FIELDS: Final[dict[str, str]] = {
|
||||
field: f"bigram_{field}" for field in _DEFAULT_SEARCH_FIELDS
|
||||
}
|
||||
_CJK_CONTENT_FIELDS: Final[list[str]] = ["bigram_content"]
|
||||
_CJK_TITLE_FIELDS: Final[list[str]] = ["bigram_title"]
|
||||
_FIELD_BOOSTS = {"title": 2.0}
|
||||
_SIMPLE_FIELD_BOOSTS = {"simple_title": 2.0}
|
||||
|
||||
|
||||
def _simple_query_tokens(raw_query: str) -> list[str]:
|
||||
# Tokenize and fold via the same analyzer used to index simple_title /
|
||||
# simple_content, so query terms fold identically to the indexed terms
|
||||
# (single source of truth for ASCII folding).
|
||||
return simple_search_tokens(raw_query)
|
||||
class _ConjunctiveNegations(wc.ast.Visitor[tuple["wc.ast.Node", ...]]):
|
||||
"""Collect the subtrees an AST excludes from every document it matches.
|
||||
|
||||
A negation reached through ``And``/``AndNot``/``Require`` (and through
|
||||
the required half of an ``AndMaybe``) constrains the whole query, so it
|
||||
can be re-stated above the blend. ``Or`` is deliberately not descended
|
||||
into: in ``invoice OR NOT secret`` the negation is one branch's own
|
||||
condition, and hoisting it would throw away documents the other branch
|
||||
matches. Nor is a collected subtree descended into, since a negation
|
||||
inside a negation is not an exclusion.
|
||||
|
||||
Node types with no negation to contribute (every leaf, ``Or``) fall
|
||||
through to ``generic_visit``.
|
||||
"""
|
||||
|
||||
def generic_visit(self, node: wc.ast.Node) -> tuple[wc.ast.Node, ...]:
|
||||
return ()
|
||||
|
||||
def visit_not(self, node: wc.ast.Not) -> tuple[wc.ast.Node, ...]:
|
||||
return (node.child,)
|
||||
|
||||
def visit_andnot(self, node: wc.ast.AndNot) -> tuple[wc.ast.Node, ...]:
|
||||
return (*self.visit(node.positive), node.negative)
|
||||
|
||||
def visit_and(self, node: wc.ast.And) -> tuple[wc.ast.Node, ...]:
|
||||
return tuple(
|
||||
negation for child in node.children for negation in self.visit(child)
|
||||
)
|
||||
|
||||
def visit_boosted(self, node: wc.ast.Boosted) -> tuple[wc.ast.Node, ...]:
|
||||
return self.visit(node.child)
|
||||
|
||||
def visit_andmaybe(self, node: wc.ast.AndMaybe) -> tuple[wc.ast.Node, ...]:
|
||||
return self.visit(node.required)
|
||||
|
||||
def visit_require(self, node: wc.ast.Require) -> tuple[wc.ast.Node, ...]:
|
||||
return (*self.visit(node.scored), *self.visit(node.filter_only))
|
||||
|
||||
|
||||
def _negation_clauses(
|
||||
index: tantivy.Index,
|
||||
ast: wc.ast.Node,
|
||||
registry: wc.FieldRegistry,
|
||||
) -> list[tuple[tantivy.Occur, tantivy.Query]]:
|
||||
"""MustNot clauses for everything ``ast`` excludes conjunctively.
|
||||
|
||||
Each excluded subtree is emitted as its own positive query and attached
|
||||
with ``MustNot``, rather than emitting a negative query and hoping
|
||||
tantivy accepts a bare one.
|
||||
"""
|
||||
try:
|
||||
return [
|
||||
(
|
||||
tantivy.Occur.MustNot,
|
||||
tantivy_emit(negation, index=index, registry=registry),
|
||||
)
|
||||
for negation in _ConjunctiveNegations().visit(ast)
|
||||
]
|
||||
except QueryError as e:
|
||||
raise _map_emit_error(e) from e
|
||||
|
||||
|
||||
def _any_of(clauses: list[tuple[tantivy.Occur, tantivy.Query]]) -> tantivy.Query:
|
||||
"""Collapse a clause list: none -> empty, one -> itself (no wasted
|
||||
single-clause boolean_query wrapping), many -> boolean_query(clauses)."""
|
||||
if not clauses:
|
||||
return tantivy.Query.empty_query()
|
||||
if len(clauses) == 1:
|
||||
return clauses[0][1]
|
||||
return tantivy.Query.boolean_query(clauses)
|
||||
|
||||
|
||||
def _build_simple_token_query(
|
||||
@@ -168,9 +424,7 @@ def _build_simple_token_query(
|
||||
query = tantivy.Query.boost_query(query, boost)
|
||||
field_queries.append((tantivy.Occur.Should, query))
|
||||
|
||||
if len(field_queries) == 1:
|
||||
return field_queries[0][1]
|
||||
return tantivy.Query.boolean_query(field_queries)
|
||||
return _any_of(field_queries)
|
||||
|
||||
|
||||
def parse_user_query(
|
||||
@@ -179,52 +433,53 @@ def parse_user_query(
|
||||
tz: tzinfo,
|
||||
) -> tantivy.Query:
|
||||
"""
|
||||
Parse user query through the complete preprocessing pipeline.
|
||||
Parse user query through whoosh-compat, then blend in fuzzy/CJK clauses.
|
||||
|
||||
Transforms the raw user query through multiple stages:
|
||||
1. Date keyword rewriting (today → ISO 8601 ranges)
|
||||
2. Query normalization (comma expansion, whitespace cleanup)
|
||||
3. Tantivy parsing with field boosts
|
||||
4. Optional fuzzy query blending (if ADVANCED_FUZZY_SEARCH_THRESHOLD set)
|
||||
|
||||
Args:
|
||||
index: Tantivy index with registered tokenizers
|
||||
raw_query: Original user query string
|
||||
tz: Timezone for date boundary calculations
|
||||
|
||||
Returns:
|
||||
Parsed Tantivy query ready for execution
|
||||
|
||||
Note:
|
||||
When ADVANCED_FUZZY_SEARCH_THRESHOLD is configured, adds a low-priority
|
||||
fuzzy query as a Should clause (0.1 boost) to catch approximate matches
|
||||
while keeping exact matches ranked higher. The threshold value is applied
|
||||
as a post-search score filter, not during query construction.
|
||||
1. wc.parse() against the shared FieldRegistry (whoosh grammar -> AST).
|
||||
Bare notes:/custom_fields: prefixes resolve to their default subpath
|
||||
(notes.note:/custom_fields.value:) directly in the registry, via
|
||||
each JSON field's SubpathSpec(default=True).
|
||||
2. Any diagnostics (bad dates/numbers) map to SearchQueryError subclasses
|
||||
and raise, the view returns HTTP 400 with every offending field
|
||||
listed, not just the first.
|
||||
3. emit() turns the AST into a tantivy.Query directly (no string
|
||||
round-trip). A QueryError is routed by its Diagnostic's Cause
|
||||
(_map_emit_error): a construct that parses but can't execute against
|
||||
tantivy (e.g. a text-field range) is a 400, a registry/schema
|
||||
mismatch is logged and a 400, and an INTERNAL defect is re-raised.
|
||||
4. Optional fuzzy blend (ADVANCED_FUZZY_SEARCH_THRESHOLD) builds a
|
||||
plain word string from the parsed AST's free-text tokens
|
||||
(whoosh_compat.free_text_tokens) and feeds THAT to
|
||||
index.parse_query, never raw_query, whose whoosh grammar (date
|
||||
keywords, bracket-class wildcards, etc.) tantivy's parser rejects,
|
||||
which used to silently knock the fuzzy clause out of any mixed
|
||||
query (see _try_parse_fuzzy_query).
|
||||
5. Optional CJK bigram clause, built from the same parsed AST for the
|
||||
same reason (see _build_ast_cjk_query): a CJK term the query negated
|
||||
or fielded must not resurface through it.
|
||||
6. When any optional clause was added, the query's conjunctive
|
||||
exclusions are restated as MustNot above the blend
|
||||
(_negation_clauses): a clause built from positive terms cannot
|
||||
express them, and as a bare Should it would undo them.
|
||||
"""
|
||||
registry = get_field_registry(settings.SEARCH_LANGUAGE)
|
||||
result = wc.parse(
|
||||
raw_query,
|
||||
registry=registry,
|
||||
default_fields=_DEFAULT_SEARCH_FIELDS,
|
||||
field_boosts=_FIELD_BOOSTS,
|
||||
tz=tz,
|
||||
)
|
||||
if result.diagnostics:
|
||||
raise _diagnostics_to_error(result.diagnostics)
|
||||
|
||||
try:
|
||||
query_str = translate_query(raw_query, tz)
|
||||
except SearchQueryError:
|
||||
# Intentional, user-fixable error (e.g. an unparsable date). Propagate so
|
||||
# the view can return a 400 with a helpful message rather than falling
|
||||
# back to the raw (still-invalid) query.
|
||||
raise
|
||||
except Exception: # pragma: no cover - defensive
|
||||
logger.warning("Query translation failed; using raw query", exc_info=True)
|
||||
query_str = raw_query
|
||||
exact = tantivy_emit(result.ast, index=index, registry=registry)
|
||||
except QueryError as e:
|
||||
raise _map_emit_error(e) from e
|
||||
|
||||
exact = index.parse_query(
|
||||
query_str,
|
||||
DEFAULT_SEARCH_FIELDS,
|
||||
field_boosts=_FIELD_BOOSTS,
|
||||
)
|
||||
|
||||
# The standard analyzer keeps a whitespace-free CJK run as a single token,
|
||||
# so substring queries can't match content/title (and long runs are dropped
|
||||
# by remove_long). Route CJK queries to the bigram fields, whose ngram
|
||||
# tokenizer indexes overlapping 2-grams for substring matching.
|
||||
cjk_query = (
|
||||
_build_cjk_query(index, raw_query, _CJK_ALL_FIELDS)
|
||||
_build_ast_cjk_query(index, result.ast, registry)
|
||||
if _has_cjk(raw_query)
|
||||
else None
|
||||
)
|
||||
@@ -235,22 +490,73 @@ def parse_user_query(
|
||||
|
||||
threshold = settings.ADVANCED_FUZZY_SEARCH_THRESHOLD
|
||||
if threshold is not None:
|
||||
fuzzy = index.parse_query(
|
||||
query_str,
|
||||
DEFAULT_SEARCH_FIELDS,
|
||||
field_boosts=_FIELD_BOOSTS,
|
||||
# (prefix=True, distance=1, transposition_cost_one=True) — edit-distance fuzziness
|
||||
fuzzy_fields={f: (True, 1, True) for f in DEFAULT_SEARCH_FIELDS},
|
||||
)
|
||||
# 0.1 boost keeps fuzzy hits ranked below exact matches (intentional)
|
||||
clauses.append((tantivy.Occur.Should, tantivy.Query.boost_query(fuzzy, 0.1)))
|
||||
fuzzy = _try_parse_fuzzy_query(index, result.ast, registry)
|
||||
if fuzzy is not None:
|
||||
clauses.append(
|
||||
(tantivy.Occur.Should, tantivy.Query.boost_query(fuzzy, 0.1)),
|
||||
)
|
||||
|
||||
if cjk_query is not None:
|
||||
clauses.append((tantivy.Occur.Should, cjk_query))
|
||||
|
||||
if len(clauses) == 1:
|
||||
return exact
|
||||
return tantivy.Query.boolean_query(clauses)
|
||||
# The fuzzy and CJK clauses are built from positive terms only, so as
|
||||
# plain Shoulds beside the exact clause they re-admit exactly the
|
||||
# documents the query excluded. Restate the exclusions once, above the
|
||||
# whole blend. Redundant against the exact clause, which already
|
||||
# carries them, but idempotently so.
|
||||
negations = _negation_clauses(index, result.ast, registry)
|
||||
if not negations:
|
||||
return _any_of(clauses)
|
||||
return tantivy.Query.boolean_query(
|
||||
[(tantivy.Occur.Must, _any_of(clauses)), *negations],
|
||||
)
|
||||
|
||||
|
||||
# The three whoosh-compat kinds for a wildcard on a field that cannot
|
||||
# carry one. d.field_kind supplies the discriminator, so naming the field's
|
||||
# type needs no second trip through the registry.
|
||||
_PATTERN_ON_KINDS: Final = frozenset(
|
||||
{
|
||||
DiagnosticKind.PATTERN_ON_NUMERIC,
|
||||
DiagnosticKind.PATTERN_ON_BOOLEAN_EXISTS,
|
||||
DiagnosticKind.PATTERN_ON_SUBPATH,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _diagnostics_to_error(diagnostics: tuple[Diagnostic, ...]) -> SearchQueryError:
|
||||
errors = [_single_diagnostic_to_error(d) for d in diagnostics]
|
||||
return errors[0] if len(errors) == 1 else MultipleSearchQueryErrors(errors)
|
||||
|
||||
|
||||
def _single_diagnostic_to_error(d: Diagnostic) -> SearchQueryError:
|
||||
# d.field is a FieldRef, not a str: str(d.field) gives the canonical
|
||||
# dotted name (an aliased query, e.g. type:, reports document_type).
|
||||
field_name = str(d.field) if d.field is not None else None
|
||||
if d.kind is DiagnosticKind.BAD_DATE:
|
||||
return InvalidDateQuery(field_name, d.raw_value)
|
||||
if d.kind is DiagnosticKind.BAD_NUMBER:
|
||||
return InvalidNumberQuery(field_name, d.raw_value)
|
||||
if d.kind is DiagnosticKind.TOO_DEEP:
|
||||
return SearchQueryError("The search query is nested too deeply.")
|
||||
if d.kind in _PATTERN_ON_KINDS:
|
||||
kind_label = f" ({d.field_kind.name.lower()})" if d.field_kind else ""
|
||||
return SearchQueryError(
|
||||
f"Wildcard patterns are not supported for field "
|
||||
f"{field_name!r}{kind_label}.",
|
||||
)
|
||||
if d.kind is DiagnosticKind.SINGLE_CHAR_BRACKET_RANGE:
|
||||
field_label = f" for field {field_name!r}" if field_name else ""
|
||||
return SearchQueryError(
|
||||
f"{d.raw_value!r} looks like a bracket range{field_label}, but "
|
||||
"'[' is not a wildcard character on its own. Combine it with a "
|
||||
"wildcard, e.g. a trailing '*', or double-quote the value to "
|
||||
"search it as literal text.",
|
||||
)
|
||||
logger.warning("Unmapped parse diagnostic %s: %s", d.kind, d.message)
|
||||
return SearchQueryError("The search query could not be executed.")
|
||||
|
||||
|
||||
def parse_simple_query(
|
||||
@@ -268,7 +574,7 @@ def parse_simple_query(
|
||||
CJK substrings the simple analyzer can't (long whitespace-free runs are
|
||||
dropped by remove_long).
|
||||
"""
|
||||
tokens = _simple_query_tokens(raw_query)
|
||||
tokens = simple_search_tokens(raw_query)
|
||||
|
||||
clauses: list[tuple[tantivy.Occur, tantivy.Query]] = []
|
||||
if tokens:
|
||||
@@ -291,23 +597,14 @@ def parse_simple_query(
|
||||
)
|
||||
for token in tokens
|
||||
]
|
||||
simple_query = (
|
||||
token_queries[0][1]
|
||||
if len(token_queries) == 1
|
||||
else tantivy.Query.boolean_query(token_queries)
|
||||
)
|
||||
clauses.append((tantivy.Occur.Should, simple_query))
|
||||
clauses.append((tantivy.Occur.Should, _any_of(token_queries)))
|
||||
|
||||
if cjk_fields and _has_cjk(raw_query):
|
||||
cjk_q = _build_cjk_query(index, raw_query, cjk_fields)
|
||||
if cjk_q is not None:
|
||||
clauses.append((tantivy.Occur.Should, cjk_q))
|
||||
|
||||
if not clauses:
|
||||
return tantivy.Query.empty_query()
|
||||
if len(clauses) == 1:
|
||||
return clauses[0][1]
|
||||
return tantivy.Query.boolean_query(clauses)
|
||||
return _any_of(clauses)
|
||||
|
||||
|
||||
def parse_simple_text_highlight_query(
|
||||
@@ -322,13 +619,21 @@ def parse_simple_text_highlight_query(
|
||||
|
||||
# Strip Tantivy operator chars before tokenizing: this is a plain-text
|
||||
# highlight query, not a structured boolean query, so +/- are separators.
|
||||
tokens = _simple_query_tokens(
|
||||
tokens = simple_search_tokens(
|
||||
regex.sub(r"[-+]", " ", raw_query, timeout=_REGEX_TIMEOUT),
|
||||
)
|
||||
if not tokens:
|
||||
return tantivy.Query.empty_query()
|
||||
|
||||
return index.parse_query(" ".join(tokens), ["content"])
|
||||
# Quote each token as its own phrase, escaping backslashes and embedded
|
||||
# quotes. simple search tokens can carry arbitrary Tantivy syntax
|
||||
# characters (`"`, `:`, `(`, `[`, `/`, ...) that the query-string parser
|
||||
# would otherwise interpret as query grammar rather than literal text.
|
||||
quoted_tokens = [
|
||||
'"' + token.replace("\\", "\\\\").replace('"', '\\"') + '"' for token in tokens
|
||||
]
|
||||
|
||||
return index.parse_query(" ".join(quoted_tokens), ["content"])
|
||||
|
||||
|
||||
def parse_simple_text_query(
|
||||
@@ -342,7 +647,7 @@ def parse_simple_text_query(
|
||||
return parse_simple_query(
|
||||
index,
|
||||
raw_query,
|
||||
SIMPLE_SEARCH_FIELDS,
|
||||
_SIMPLE_SEARCH_FIELDS,
|
||||
cjk_fields=_CJK_CONTENT_FIELDS,
|
||||
)
|
||||
|
||||
@@ -358,6 +663,6 @@ def parse_simple_title_query(
|
||||
return parse_simple_query(
|
||||
index,
|
||||
raw_query,
|
||||
TITLE_SEARCH_FIELDS,
|
||||
_TITLE_SEARCH_FIELDS,
|
||||
cjk_fields=_CJK_TITLE_FIELDS,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user