amorfo77/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2026-09-11 20:28:01 +00:00
Code Issues Projects Releases Wiki Activity

Feature: parse advanced search with whoosh-compat and delete the hand-written translator

Browse Source
This commit is contained in:
stumpylog
2026-09-11 11:22:00 -07:00
parent 8c7d2d905c
commit a3723aa2ea
31 changed files with 4913 additions and 2477 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/documents/search/__init__.py
+10 -2
View File
@@ -6,13 +6,20 @@ from documents.search._backend import TantivyRelevanceList
from documents.search._backend import WriteBatch
from documents.search._backend import get_backend
from documents.search._backend import reset_backend
from documents.search._errors import InvalidDateQuery
from documents.search._errors import InvalidNumberQuery
from documents.search._errors import MultipleSearchQueryErrors
from documents.search._errors import QueryTooLongError
from documents.search._errors import SearchQueryError
from documents.search._errors import search_query_error_messages
from documents.search._schema import needs_rebuild
from documents.search._schema import wipe_index
from documents.search._translate import InvalidDateQuery
from documents.search._translate import SearchQueryError
__all__ = [
"InvalidDateQuery",
"InvalidNumberQuery",
"MultipleSearchQueryErrors",
"QueryTooLongError",
"SearchHit",
"SearchIndexLockError",
"SearchMode",
@@ -23,5 +30,6 @@ __all__ = [
"get_backend",
"needs_rebuild",
"reset_backend",
"search_query_error_messages",
"wipe_index",
]
src/documents/search/_backend.py
+59 -9
View File
@@ -22,7 +22,6 @@ import tantivy
from django.conf import settings
from django.utils.timezone import get_current_timezone
from documents.search._query import build_permission_filter
from documents.search._query import extract_cjk_text
from documents.search._query import parse_simple_text_highlight_query
from documents.search._query import parse_simple_text_query
@@ -40,6 +39,7 @@ from documents.utils import QuerySetStream
from documents.utils import identity
if TYPE_CHECKING:
from collections.abc import Iterable
from collections.abc import Iterator
from collections.abc import Sequence
from pathlib import Path
@@ -325,6 +325,47 @@ class WriteBatch:
self._writer.add_document(doc)
def build_permission_filter(
schema: tantivy.Schema,
user: AbstractUser,
viewer_group_ids: Iterable[int] = (),
) -> tantivy.Query:
"""
Build a query filter for user document permissions.
Creates a query that matches only documents visible to the specified user
according to paperless-ngx permission rules:
- Public documents (no owner) are visible to all users
- Private documents are visible to their owner
- Documents explicitly shared with the user are visible
- Documents shared with one of the user's current groups are visible
Args:
schema: Tantivy schema for field validation
user: User to check permissions for
viewer_group_ids: Current group memberships for the user
Returns:
Tantivy query that filters results to visible documents
"""
owner_any = tantivy.Query.exists_query("owner_id")
no_owner = tantivy.Query.boolean_query(
[
(tantivy.Occur.Must, tantivy.Query.all_query()),
(tantivy.Occur.MustNot, owner_any),
],
)
owned = tantivy.Query.term_query(schema, "owner_id", user.pk)
shared = tantivy.Query.term_query(schema, "viewer_id", user.pk)
group_shared = [
tantivy.Query.term_query(schema, "viewer_group_id", group_id)
for group_id in viewer_group_ids
]
return tantivy.Query.disjunction_max_query(
[no_owner, owned, shared, *group_shared],
)
class TantivyBackend:
"""
Tantivy search backend with explicit lifecycle management.
@@ -498,7 +539,6 @@ class TantivyBackend:
doc.add_text("correspondent_sort", document.correspondent.name)
if cjk_corr := extract_cjk_text(document.correspondent.name):
doc.add_text("bigram_correspondent", cjk_corr)
doc.add_unsigned("correspondent_id", document.correspondent_id)
# Document type
if document.document_type:
@@ -506,12 +546,10 @@ class TantivyBackend:
doc.add_text("type_sort", document.document_type.name)
if cjk_type := extract_cjk_text(document.document_type.name):
doc.add_text("bigram_document_type", cjk_type)
doc.add_unsigned("document_type_id", document.document_type_id)
# Storage path
if document.storage_path:
doc.add_text("storage_path", document.storage_path.name)
doc.add_unsigned("storage_path_id", document.storage_path_id)
# Tags — collect names for autocomplete in the same pass
tag_names: list[str] = []
@@ -519,12 +557,13 @@ class TantivyBackend:
doc.add_text("tag", tag.name)
if cjk_tag := extract_cjk_text(tag.name):
doc.add_text("bigram_tag", cjk_tag)
doc.add_unsigned("tag_id", tag.pk)
tag_names.append(tag.name)
# Notes — JSON for structured queries (notes.user:alice, notes.note:text).
# notes_text is a plain-text companion for snippet/highlight generation;
# tantivy's SnippetGenerator does not support JSON fields.
# tantivy's SnippetGenerator does not support JSON fields. It is not in
# _DEFAULT_SEARCH_FIELDS, so an unqualified query never searches it: a
# note matches through the JSON field or not at all.
num_notes = 0
note_texts: list[str] = []
for note in document.notes.all():
@@ -540,8 +579,9 @@ class TantivyBackend:
if note_texts:
doc.add_text("notes_text", " ".join(note_texts))
# Custom fields — JSON for structured queries (custom_fields.name:x, custom_fields.value:y),
# companion text field for default full-text search.
# Custom fields: JSON for structured queries (custom_fields.name:x,
# custom_fields.value:y). There is no companion text field here, unlike
# notes: custom field values are reachable only through the JSON field.
for cfi in document.custom_fields.all():
search_value = cfi.value_for_search
# Skip fields where there is no value yet
@@ -708,7 +748,17 @@ class TantivyBackend:
user_query = self._parse_query(query, search_mode)
highlight_query = user_query
if search_mode is SearchMode.TEXT:
highlight_query = parse_simple_text_highlight_query(self._index, query)
try:
highlight_query = parse_simple_text_highlight_query(
self._index,
query,
)
except ValueError:
logger.debug(
"Skipping simple text highlight query: token string is not "
"valid tantivy query syntax: %r",
query,
)
# For notes_text snippet generation, we need a query that targets the
# notes_text field directly. user_query may contain JSON-field terms
src/documents/search/_dates.py
-171
View File
@@ -1,171 +0,0 @@
from __future__ import annotations
from datetime import UTC
from datetime import date
from datetime import datetime
from datetime import timedelta
from typing import TYPE_CHECKING
from typing import Final
from dateutil.relativedelta import relativedelta
if TYPE_CHECKING:
from datetime import tzinfo
_DATE_ONLY_FIELDS = frozenset({"created"})
_TODAY: Final[str] = "today"
_YESTERDAY: Final[str] = "yesterday"
_PREVIOUS_WEEK: Final[str] = "previous week"
_THIS_MONTH: Final[str] = "this month"
_PREVIOUS_MONTH: Final[str] = "previous month"
_THIS_YEAR: Final[str] = "this year"
_PREVIOUS_YEAR: Final[str] = "previous year"
_PREVIOUS_QUARTER: Final[str] = "previous quarter"
_DATE_KEYWORDS = frozenset(
{
_TODAY,
_YESTERDAY,
_PREVIOUS_WEEK,
_THIS_MONTH,
_PREVIOUS_MONTH,
_THIS_YEAR,
_PREVIOUS_YEAR,
_PREVIOUS_QUARTER,
},
)
def _fmt(dt: datetime) -> str:
"""Format a datetime as an ISO 8601 UTC string for use in Tantivy range queries."""
return dt.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
def _iso_range(lo: datetime, hi: datetime) -> str:
"""
Format a half-open ``[lo TO hi)`` range in ISO 8601 for Tantivy query syntax.
``hi`` is always the exclusive ceiling of a computed period (the start of
the *next* day/week/month/quarter/year), so the closing bracket must be
the Tantivy exclusive-range brace ``}`` rather than ``]`` — otherwise the
first instant of the following period (e.g. the 1st of next month) is
incorrectly included in the match.
"""
return f"[{_fmt(lo)} TO {_fmt(hi)}}}"
def _quarter_start(d: date) -> date:
"""Return the first day of the calendar quarter containing ``d``."""
return date(d.year, ((d.month - 1) // 3) * 3 + 1, 1)
def _midnight(d: date, tz: tzinfo) -> datetime:
"""Convert a calendar date at local-timezone midnight to a UTC datetime."""
return datetime(d.year, d.month, d.day, tzinfo=tz).astimezone(UTC)
def _keyword_bounds(keyword: str, tz: tzinfo) -> tuple[date, date]:
"""
Map a relative date keyword to ``(start, exclusive_end)`` calendar dates.
``tz`` only determines what "today" is; the caller decides how the returned
dates become UTC datetime boundaries (date-only vs. local-midnight offset).
"""
today = datetime.now(tz).date()
if keyword == _TODAY:
return today, today + timedelta(days=1)
if keyword == _YESTERDAY:
return today - timedelta(days=1), today
if keyword == _PREVIOUS_WEEK:
this_monday = today - timedelta(days=today.weekday())
return this_monday - timedelta(weeks=1), this_monday
if keyword == _THIS_MONTH:
first = today.replace(day=1)
return first, first + relativedelta(months=1)
if keyword == _PREVIOUS_MONTH:
this_first = today.replace(day=1)
return this_first - relativedelta(months=1), this_first
if keyword == _THIS_YEAR:
return date(today.year, 1, 1), date(today.year + 1, 1, 1)
if keyword == _PREVIOUS_YEAR:
return date(today.year - 1, 1, 1), date(today.year, 1, 1)
if keyword == _PREVIOUS_QUARTER:
this_quarter = _quarter_start(today)
return this_quarter - relativedelta(months=3), this_quarter
raise ValueError(f"Unknown keyword: {keyword}")
def _date_only_range(keyword: str, tz: tzinfo) -> str:
"""
For `created` (DateField): use the local calendar date, converted to
midnight UTC boundaries. No offset arithmetic — date only.
"""
start, end = _keyword_bounds(keyword, tz)
lo = datetime(start.year, start.month, start.day, tzinfo=UTC)
hi = datetime(end.year, end.month, end.day, tzinfo=UTC)
return _iso_range(lo, hi)
def _datetime_range(keyword: str, tz: tzinfo) -> str:
"""
For `added` / `modified` (DateTimeField, stored as UTC): convert local day
boundaries to UTC — full offset arithmetic required.
"""
start, end = _keyword_bounds(keyword, tz)
return _iso_range(_midnight(start, tz), _midnight(end, tz))
def _precision_bounds(digits: str) -> tuple[date, date] | None:
"""
Map a 4/6/8-digit date token to (start, exclusive_end) calendar dates.
YYYY -> whole year, YYYYMM -> whole month, YYYYMMDD -> single day.
Returns None for any unparsable or out-of-range value (e.g. month 23),
so callers can emit a no-match clause instead of erroring (Whoosh parity).
"""
try:
if len(digits) == 4:
year = int(digits)
return date(year, 1, 1), date(year + 1, 1, 1)
if len(digits) == 6:
year, month = int(digits[:4]), int(digits[4:6])
start = date(year, month, 1)
end = date(year + 1, 1, 1) if month == 12 else date(year, month + 1, 1)
return start, end
if len(digits) == 8:
start = date(int(digits[:4]), int(digits[4:6]), int(digits[6:8]))
return start, start + timedelta(days=1)
except ValueError:
return None
return None
def _utc_bounds_for_field(
field: str,
start: date,
end: date,
tz: tzinfo,
) -> tuple[datetime, datetime]:
"""
Convert calendar-date bounds to UTC datetimes per the field's storage type.
For DateField (``created``) the bounds are UTC midnight (no offset). For
DateTimeField (``added``/``modified``) the bounds are local-tz midnight
converted to UTC, matching how each field is indexed.
"""
if field in _DATE_ONLY_FIELDS:
return (
datetime(start.year, start.month, start.day, tzinfo=UTC),
datetime(end.year, end.month, end.day, tzinfo=UTC),
)
return (
datetime(start.year, start.month, start.day, tzinfo=tz).astimezone(UTC),
datetime(end.year, end.month, end.day, tzinfo=tz).astimezone(UTC),
)
def _field_range_from_dates(field: str, start: date, end: date, tz: tzinfo) -> str:
"""Build a Tantivy ``field:[lo TO hi]`` ISO range from calendar-date bounds."""
lo, hi = _utc_bounds_for_field(field, start, end, tz)
return f"{field}:{_iso_range(lo, hi)}"
src/documents/search/_errors.py
+71
View File
@@ -0,0 +1,71 @@
from __future__ import annotations
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from collections.abc import Sequence
class SearchQueryError(ValueError):
"""
Base for user-fixable search query errors.
Carries a message safe to surface to the user (no internal details). The
view layer catches this and returns an HTTP 400, so any future subclass
gets the same treatment.
"""
class InvalidDateQuery(SearchQueryError):
"""Raised when a date field value or range bound cannot be parsed."""
def __init__(self, field: str | None, value: str | None) -> None:
self.field = field
self.value = value
super().__init__(f"Invalid date value {value!r} for field {field!r}.")
class InvalidNumberQuery(SearchQueryError):
"""Raised when a numeric field value or range bound cannot be parsed."""
def __init__(self, field: str | None, value: str | None) -> None:
self.field = field
self.value = value
super().__init__(f"Invalid numeric value {value!r} for field {field!r}.")
class QueryTooLongError(SearchQueryError):
"""Raised when a query string exceeds the maximum allowed length.
whoosh-compat's fieldname tagger is O(n^2) in plain word characters, so an
unbounded query is a CPU-exhaustion vector against a single request
handler. This is a hard boundary, not a validation nicety.
"""
def __init__(self, length: int, limit: int) -> None:
self.length = length
self.limit = limit
super().__init__(
f"The search query is too long ({length} characters). "
f"The maximum allowed length is {limit} characters.",
)
class MultipleSearchQueryErrors(SearchQueryError):
"""Aggregates every user-fixable error from one parse, not just the first."""
def __init__(self, errors: Sequence[SearchQueryError]) -> None:
self.errors = tuple(errors)
super().__init__("; ".join(str(e) for e in self.errors))
def search_query_error_messages(e: SearchQueryError) -> list[str]:
"""The user-facing message list for a SearchQueryError.
Every offending value's message, not just the first, so the user can
fix them all in one round-trip. Shared by every view that maps
SearchQueryError to an HTTP 400.
"""
if isinstance(e, MultipleSearchQueryErrors):
return [str(sub) for sub in e.errors]
return [str(e)]
src/documents/search/_query.py
+450 -145
View File
@@ -6,22 +6,30 @@ from typing import Final
import regex
import tantivy
import whoosh_compat as wc
from django.conf import settings
from whoosh_compat.emitters.tantivy_ import emit as tantivy_emit
from whoosh_compat.errors import Cause
from whoosh_compat.errors import Diagnostic
from whoosh_compat.errors import DiagnosticKind
from whoosh_compat.errors import QueryError
from documents.search._errors import InvalidDateQuery
from documents.search._errors import InvalidNumberQuery
from documents.search._errors import MultipleSearchQueryErrors
from documents.search._errors import SearchQueryError
from documents.search._registry import get_field_registry
from documents.search._tokenizer import simple_search_tokens
from documents.search._translate import SearchQueryError
from documents.search._translate import translate_query
if TYPE_CHECKING:
from collections.abc import Iterable
from datetime import tzinfo
from django.contrib.auth.base_user import AbstractBaseUser
logger = logging.getLogger("paperless.search")
# Maximum seconds any single regex substitution may run.
# Prevents ReDoS on adversarial user-supplied query strings.
# Maximum seconds any single regex substitution over user-supplied query text
# may run. The one remaining use is a character class, which cannot backtrack,
# so the bound is an upper limit on that substitution's cost, not the ReDoS
# guard it was originally written as.
_REGEX_TIMEOUT: Final[float] = 1.0
# Matches CJK/Hangul characters so queries can be routed to bigram fields.
@@ -29,6 +37,64 @@ _REGEX_TIMEOUT: Final[float] = 1.0
_CJK_RE: Final = regex.compile(r"[\p{Han}\p{Hiragana}\p{Katakana}\p{Hangul}]+")
def _user_facing_emit_message(d: Diagnostic) -> str:
"""A user-safe message for an emit-time QueryError's Diagnostic.
Built from the Diagnostic's structured fields (kind, field), never from
d.message: whoosh-compat documents that as developer/log output with no
stability guarantee, and PATTERN_TOO_COMPLEX embeds the raw backend
error text in it.
"""
field = str(d.field) if d.field is not None else None
if d.kind is DiagnosticKind.EXISTS_REQUIRES_FAST:
return f"Existence searches (field:*) are not supported for field {field!r}."
if d.kind is DiagnosticKind.TEXT_RANGE:
return f"Range searches are not supported for field {field!r}."
if d.kind is DiagnosticKind.PATTERN_TOO_COMPLEX:
return f"The wildcard pattern for field {field!r} is too complex."
if d.kind is DiagnosticKind.SCHEMA_FIELD_MISSING:
return f"Field {field!r} is not available in the search index."
logger.warning("Unmapped emit diagnostic %s: %s", d.kind, d.message)
return "The search query could not be executed."
def _map_emit_error(e: QueryError) -> SearchQueryError:
"""Route an emit-time QueryError by its Diagnostic's Cause.
INVALID_INPUT/UNSUPPORTED are user-input errors, exactly like a parse
diagnostic, and map to a 400. INTERNAL means a defect in whoosh-compat
or in our own AST handling, never the user's query, so the QueryError is
re-raised rather than converted, reaching the generic 500 handler instead
of blaming the query. MISCONFIGURED is deliberately both: the registry and
the index schema disagree, which only an operator can fix, so it is logged
as an error, but a request is still waiting and the query cannot run
either way, so it also returns a 400.
EXISTS_REQUIRES_FAST is the one MISCONFIGURED kind that is not a
disagreement. whoosh-compat derives it from the registry's own FieldSpec
(kind plus fast) without ever consulting the index schema, so it fires
whenever a non-fast field of a kind that cannot answer "exists" is asked
to: for us that is only the JSON fields, which field_descriptors() builds
non-fast on purpose. "notes:*" and the five other spellings of it are
ordinary user error that no operator action can clear, so they get the
400 without the alert.
"""
d = e.diagnostic
if d.cause is Cause.INTERNAL:
raise e
if (
d.cause is Cause.MISCONFIGURED
and d.kind is not DiagnosticKind.EXISTS_REQUIRES_FAST
):
logger.error(
"Search index misconfiguration for field %s (%s): %s",
d.field,
d.kind.name,
d.message,
)
return SearchQueryError(_user_facing_emit_message(d))
def _has_cjk(text: str) -> bool:
"""Return True if text contains any CJK characters."""
return bool(_CJK_RE.search(text))
@@ -37,14 +103,36 @@ def _has_cjk(text: str) -> bool:
def extract_cjk_text(text: str) -> str:
"""Join the CJK runs in ``text`` for indexing into bigram (char-ngram) fields.
Mirrors the query side (``_build_cjk_query``): only CJK runs are ever searched
against the bigram fields, so only CJK runs are worth indexing there. Latin
text fed to a character-bigram field is never matched and only bloats the
Mirrors the query side, which extracts the CJK runs of whatever it is
about to search for (the raw string in simple modes, the parsed query's
free-text tokens in query mode): only CJK runs are ever searched against
the bigram fields, so only CJK runs are worth indexing there. Latin text
fed to a character-bigram field is never matched and only bloats the
index and slows indexing/merge. Returns "" when there is no CJK text.
"""
return " ".join(_CJK_RE.findall(text))
def _parse_cjk_text(
index: tantivy.Index,
cjk_text: str,
fields: list[str],
) -> tantivy.Query | None:
"""Parse a plain CJK run string against ``fields``, or None if it won't parse."""
try:
return index.parse_query(cjk_text, fields)
except Exception:
# Broad on purpose, unlike _try_parse_fuzzy_query's narrower
# ValueError: cjk_text isn't filtered to a guaranteed-safe token
# set the way the fuzzy blend's word string is, so the exact
# failure mode tantivy could raise here isn't pinned down.
logger.debug(
"Skipping CJK search clause: could not parse CJK text: %r",
cjk_text,
)
return None
def _build_cjk_query(
index: tantivy.Index,
raw_query: str,
@@ -52,91 +140,259 @@ def _build_cjk_query(
) -> tantivy.Query | None:
"""Build a bigram-field query from the CJK runs in ``raw_query``.
Only the CJK character runs are extracted and parsed; ASCII field prefixes,
boolean operators and date keywords are discarded. This keeps the CJK clause
plain-text and consistent across query/simple modes (no leaked ``field:``
semantics, no parse failures from spaced ``-``/``+``), and avoids feeding
Latin tokens into the character-bigram matcher (which would produce spurious
matches against unrelated Latin text). Returns None when there is no CJK
text or the parse fails.
For the simple (TEXT/TITLE) modes, whose input is plain text and carries
no query grammar to respect. Only the CJK character runs are extracted, so
a stray ``field:`` prefix or ``-``/``+`` in the input can neither leak
field semantics nor fail the parse, and no Latin token reaches the
character-bigram matcher (where it would produce spurious matches against
unrelated Latin text). Returns None when there is no CJK text or the parse
fails.
"""
cjk_text = " ".join(_CJK_RE.findall(raw_query))
cjk_text = extract_cjk_text(raw_query)
if not cjk_text:
return None
return _parse_cjk_text(index, cjk_text, fields)
def _build_ast_cjk_query(
index: tantivy.Index,
ast: wc.ast.Node,
registry: wc.FieldRegistry,
) -> tantivy.Query | None:
"""Build the bigram clause of a QUERY-mode search from the parsed AST.
Same discipline as the fuzzy clause (see _try_parse_fuzzy_query): the CJK
runs come from whoosh_compat's ``free_text_tokens`` over the parsed tree,
never from the raw query string, so a term the user negated or restricted
to a field outside the default search fields contributes nothing, instead
of resurfacing as a top-level clause matching every bigram field.
``free_text_tokens`` reports no field of its own, so the tokens are
collected one default field at a time: a bare term, which the parser has
already copied onto every default field, is therefore searched across
every bigram field, while ``title:東京`` reaches ``bigram_title`` alone.
Fields whose CJK text is identical (the bare-term case) share a single
parse over all of their bigram fields at once.
Raw (``analyzed=False``) tokens are used because the bigram fields have
their own character-ngram analyzer: the default fields' word analyzers
have no useful say over a CJK run, and running them first would only
risk dropping it (remove_long) before the run is ever extracted.
Returns None when the query has no CJK free text.
"""
fields_by_text: dict[str, list[str]] = {}
for field, bigram_field in _CJK_BIGRAM_FIELDS.items():
tokens = wc.free_text_tokens(
ast,
registry=registry,
fields=[field],
analyzed=False,
)
cjk_text = extract_cjk_text(" ".join(tokens))
if cjk_text:
fields_by_text.setdefault(cjk_text, []).append(bigram_field)
clauses: list[tuple[tantivy.Occur, tantivy.Query]] = [
(tantivy.Occur.Should, query)
for cjk_text, bigram_fields in fields_by_text.items()
if (query := _parse_cjk_text(index, cjk_text, bigram_fields)) is not None
]
return _any_of(clauses) if clauses else None
# A joined fuzzy word string must stay plain words: it goes back through
# tantivy's own query parser, and the raw query text the clause collects
# routinely carries characters that parser reads as grammar (a colon, a
# bracket, a quote, a leading -). Each token is cut into its word runs and
# only those are kept, so no field syntax, pattern, range or grouping can
# reach the parser. Cutting rather than dropping the whole token is what
# keeps ordinary hyphenated, dotted and quoted input ("COVID-19",
# "hello@example.com", "tax reports") contributing to the clause at all.
_WORD_RUN_RE = regex.compile(r"\w+")
# The one piece of tantivy grammar that survives the cut: its boolean
# keywords are themselves word runs. Only these exact spellings are
# grammar there ("And"/"and" are ordinary terms), so lowercasing exactly
# these turns them back into the ordinary terms the field analyzer used to
# make of them, before the clause switched to raw text. Left alone, a
# quoted phrase would silently restructure the clause ("tax AND reports"
# becoming a conjunction) or fail to parse and drop it entirely
# ("tax AND", or "IN" anywhere).
#
# Only these words are touched: tantivy lowercases query terms with the
# field's own analyzer, and doing it ourselves first is not always the
# same operation (Python folds a final sigma to a different letter than
# tantivy does, and turns Turkish 'İ' into a sequence tantivy then splits
# in two), which would search for terms the index does not contain.
_TANTIVY_KEYWORDS: Final[frozenset[str]] = frozenset({"AND", "OR", "NOT", "IN"})
def _try_parse_fuzzy_query(
index: tantivy.Index,
ast: wc.ast.Node,
registry: wc.FieldRegistry,
) -> tantivy.Query | None:
"""Build the fuzzy blend clause from the parsed query's free-text
words, or None if it has none.
The clause is built by handing tantivy's own query parser a plain
word string (there's no clean AST-level fuzzy equivalent to
whoosh-compat's parse tree, and fuzzy matching was always an
approximate, secondary, 0.1-boosted clause). The words come from
whoosh_compat's ``free_text_tokens`` over the already-parsed AST,
never from the raw query string: raw whoosh grammar (date keywords,
``[2005 to 2009]`` ranges, bracket-class wildcards) is not tantivy
syntax, and feeding it here used to knock the fuzzy clause out for
the whole query the moment any such construct appeared alongside a
typo'd word. The helper also keeps excluded terms out: a ``NOT``'d
word must not resurface through the fuzzy clause.
Chosen trade-off: a term explicitly fielded on one of the default
search fields (``correspondent:acme``) contributes its text to the
word string UNFIELDED, so the fuzzy clause searches it across all
default fields rather than just the one the user named. That is
recall-only widening on a secondary 0.1-boosted clause the score
threshold already disciplines, accepted in exchange for never feeding
field syntax to tantivy's parser. What the word string guarantees is
exactly that: no field prefix, pattern, range, grouping or quoting
survives, and the boolean keywords that do survive (they are word
runs) are lowercased into ordinary terms; see _TANTIVY_KEYWORDS.
The words are the query's RAW text, not the analyzer's output
(``analyzed=False``), because ``index.parse_query`` analyzes whatever
it is given and analysis is not idempotent: ``universities`` stems to
``univers``, and handing that back stems it again to ``univ``, a term
the index does not contain. ``prefix=True`` hid this as over-broad
matching (``univ`` also prefixes ``unicycle``) rather than as no
matches at all. Raw text is untokenized, which is why it is cut into
word runs above rather than taken whole.
The ValueError guard stays as insurance (the word string is plain
tokens, so tantivy accepting it is expected, not assumed): on a parse
failure the fuzzy clause is skipped and the exact/CJK clauses stand,
rather than the whole query failing.
"""
tokens = wc.free_text_tokens(
ast,
registry=registry,
fields=_DEFAULT_SEARCH_FIELDS,
analyzed=False,
)
words = list(
dict.fromkeys(
word.lower() if word in _TANTIVY_KEYWORDS else word
for token in tokens
for word in _WORD_RUN_RE.findall(token)
),
)
if not words:
return None
fuzzy_text = " ".join(words)
try:
return index.parse_query(cjk_text, fields)
except Exception:
return index.parse_query(
fuzzy_text,
_DEFAULT_SEARCH_FIELDS,
field_boosts=_FIELD_BOOSTS,
fuzzy_fields={f: (True, 1, True) for f in _DEFAULT_SEARCH_FIELDS},
)
except ValueError:
logger.debug(
"Skipping fuzzy search clause: token string is not valid "
"tantivy query syntax: %r",
fuzzy_text,
)
return None
def build_permission_filter(
schema: tantivy.Schema,
user: AbstractBaseUser,
viewer_group_ids: Iterable[int] = (),
) -> tantivy.Query:
"""
Build a query filter for user document permissions.
Creates a query that matches only documents visible to the specified user
according to paperless-ngx permission rules:
- Public documents (no owner) are visible to all users
- Private documents are visible to their owner
- Documents explicitly shared with the user are visible
- Documents shared with one of the user's current groups are visible
Args:
schema: Tantivy schema for field validation
user: User to check permissions for
viewer_group_ids: Current group memberships for the user
Returns:
Tantivy query that filters results to visible documents
"""
owner_any = tantivy.Query.exists_query("owner_id")
no_owner = tantivy.Query.boolean_query(
[
(tantivy.Occur.Must, tantivy.Query.all_query()),
(tantivy.Occur.MustNot, owner_any),
],
)
owned = tantivy.Query.term_query(schema, "owner_id", user.pk)
shared = tantivy.Query.term_query(schema, "viewer_id", user.pk)
group_shared = [
tantivy.Query.term_query(schema, "viewer_group_id", group_id)
for group_id in viewer_group_ids
]
return tantivy.Query.disjunction_max_query(
[no_owner, owned, shared, *group_shared],
)
DEFAULT_SEARCH_FIELDS = [
_DEFAULT_SEARCH_FIELDS: Final[list[str]] = [
"title",
"content",
"correspondent",
"document_type",
"tag",
]
SIMPLE_SEARCH_FIELDS = ["simple_title", "simple_content"]
TITLE_SEARCH_FIELDS = ["simple_title"]
_CJK_ALL_FIELDS: Final[list[str]] = [
"bigram_content",
"bigram_title",
"bigram_correspondent",
"bigram_document_type",
"bigram_tag",
]
_SIMPLE_SEARCH_FIELDS: Final[list[str]] = ["simple_title", "simple_content"]
_TITLE_SEARCH_FIELDS: Final[list[str]] = ["simple_title"]
# The bigram (character-ngram) companion of each default search field.
_CJK_BIGRAM_FIELDS: Final[dict[str, str]] = {
field: f"bigram_{field}" for field in _DEFAULT_SEARCH_FIELDS
}
_CJK_CONTENT_FIELDS: Final[list[str]] = ["bigram_content"]
_CJK_TITLE_FIELDS: Final[list[str]] = ["bigram_title"]
_FIELD_BOOSTS = {"title": 2.0}
_SIMPLE_FIELD_BOOSTS = {"simple_title": 2.0}
def _simple_query_tokens(raw_query: str) -> list[str]:
# Tokenize and fold via the same analyzer used to index simple_title /
# simple_content, so query terms fold identically to the indexed terms
# (single source of truth for ASCII folding).
return simple_search_tokens(raw_query)
class _ConjunctiveNegations(wc.ast.Visitor[tuple["wc.ast.Node", ...]]):
"""Collect the subtrees an AST excludes from every document it matches.
A negation reached through ``And``/``AndNot``/``Require`` (and through
the required half of an ``AndMaybe``) constrains the whole query, so it
can be re-stated above the blend. ``Or`` is deliberately not descended
into: in ``invoice OR NOT secret`` the negation is one branch's own
condition, and hoisting it would throw away documents the other branch
matches. Nor is a collected subtree descended into, since a negation
inside a negation is not an exclusion.
Node types with no negation to contribute (every leaf, ``Or``) fall
through to ``generic_visit``.
"""
def generic_visit(self, node: wc.ast.Node) -> tuple[wc.ast.Node, ...]:
return ()
def visit_not(self, node: wc.ast.Not) -> tuple[wc.ast.Node, ...]:
return (node.child,)
def visit_andnot(self, node: wc.ast.AndNot) -> tuple[wc.ast.Node, ...]:
return (*self.visit(node.positive), node.negative)
def visit_and(self, node: wc.ast.And) -> tuple[wc.ast.Node, ...]:
return tuple(
negation for child in node.children for negation in self.visit(child)
)
def visit_boosted(self, node: wc.ast.Boosted) -> tuple[wc.ast.Node, ...]:
return self.visit(node.child)
def visit_andmaybe(self, node: wc.ast.AndMaybe) -> tuple[wc.ast.Node, ...]:
return self.visit(node.required)
def visit_require(self, node: wc.ast.Require) -> tuple[wc.ast.Node, ...]:
return (*self.visit(node.scored), *self.visit(node.filter_only))
def _negation_clauses(
index: tantivy.Index,
ast: wc.ast.Node,
registry: wc.FieldRegistry,
) -> list[tuple[tantivy.Occur, tantivy.Query]]:
"""MustNot clauses for everything ``ast`` excludes conjunctively.
Each excluded subtree is emitted as its own positive query and attached
with ``MustNot``, rather than emitting a negative query and hoping
tantivy accepts a bare one.
"""
try:
return [
(
tantivy.Occur.MustNot,
tantivy_emit(negation, index=index, registry=registry),
)
for negation in _ConjunctiveNegations().visit(ast)
]
except QueryError as e:
raise _map_emit_error(e) from e
def _any_of(clauses: list[tuple[tantivy.Occur, tantivy.Query]]) -> tantivy.Query:
"""Collapse a clause list: none -> empty, one -> itself (no wasted
single-clause boolean_query wrapping), many -> boolean_query(clauses)."""
if not clauses:
return tantivy.Query.empty_query()
if len(clauses) == 1:
return clauses[0][1]
return tantivy.Query.boolean_query(clauses)
def _build_simple_token_query(
@@ -168,9 +424,7 @@ def _build_simple_token_query(
query = tantivy.Query.boost_query(query, boost)
field_queries.append((tantivy.Occur.Should, query))
if len(field_queries) == 1:
return field_queries[0][1]
return tantivy.Query.boolean_query(field_queries)
return _any_of(field_queries)
def parse_user_query(
@@ -179,52 +433,53 @@ def parse_user_query(
tz: tzinfo,
) -> tantivy.Query:
"""
Parse user query through the complete preprocessing pipeline.
Parse user query through whoosh-compat, then blend in fuzzy/CJK clauses.
Transforms the raw user query through multiple stages:
1. Date keyword rewriting (today → ISO 8601 ranges)
2. Query normalization (comma expansion, whitespace cleanup)
3. Tantivy parsing with field boosts
4. Optional fuzzy query blending (if ADVANCED_FUZZY_SEARCH_THRESHOLD set)
Args:
index: Tantivy index with registered tokenizers
raw_query: Original user query string
tz: Timezone for date boundary calculations
Returns:
Parsed Tantivy query ready for execution
Note:
When ADVANCED_FUZZY_SEARCH_THRESHOLD is configured, adds a low-priority
fuzzy query as a Should clause (0.1 boost) to catch approximate matches
while keeping exact matches ranked higher. The threshold value is applied
as a post-search score filter, not during query construction.
1. wc.parse() against the shared FieldRegistry (whoosh grammar -> AST).
Bare notes:/custom_fields: prefixes resolve to their default subpath
(notes.note:/custom_fields.value:) directly in the registry, via
each JSON field's SubpathSpec(default=True).
2. Any diagnostics (bad dates/numbers) map to SearchQueryError subclasses
and raise, the view returns HTTP 400 with every offending field
listed, not just the first.
3. emit() turns the AST into a tantivy.Query directly (no string
round-trip). A QueryError is routed by its Diagnostic's Cause
(_map_emit_error): a construct that parses but can't execute against
tantivy (e.g. a text-field range) is a 400, a registry/schema
mismatch is logged and a 400, and an INTERNAL defect is re-raised.
4. Optional fuzzy blend (ADVANCED_FUZZY_SEARCH_THRESHOLD) builds a
plain word string from the parsed AST's free-text tokens
(whoosh_compat.free_text_tokens) and feeds THAT to
index.parse_query, never raw_query, whose whoosh grammar (date
keywords, bracket-class wildcards, etc.) tantivy's parser rejects,
which used to silently knock the fuzzy clause out of any mixed
query (see _try_parse_fuzzy_query).
5. Optional CJK bigram clause, built from the same parsed AST for the
same reason (see _build_ast_cjk_query): a CJK term the query negated
or fielded must not resurface through it.
6. When any optional clause was added, the query's conjunctive
exclusions are restated as MustNot above the blend
(_negation_clauses): a clause built from positive terms cannot
express them, and as a bare Should it would undo them.
"""
registry = get_field_registry(settings.SEARCH_LANGUAGE)
result = wc.parse(
raw_query,
registry=registry,
default_fields=_DEFAULT_SEARCH_FIELDS,
field_boosts=_FIELD_BOOSTS,
tz=tz,
)
if result.diagnostics:
raise _diagnostics_to_error(result.diagnostics)
try:
query_str = translate_query(raw_query, tz)
except SearchQueryError:
# Intentional, user-fixable error (e.g. an unparsable date). Propagate so
# the view can return a 400 with a helpful message rather than falling
# back to the raw (still-invalid) query.
raise
except Exception: # pragma: no cover - defensive
logger.warning("Query translation failed; using raw query", exc_info=True)
query_str = raw_query
exact = tantivy_emit(result.ast, index=index, registry=registry)
except QueryError as e:
raise _map_emit_error(e) from e
exact = index.parse_query(
query_str,
DEFAULT_SEARCH_FIELDS,
field_boosts=_FIELD_BOOSTS,
)
# The standard analyzer keeps a whitespace-free CJK run as a single token,
# so substring queries can't match content/title (and long runs are dropped
# by remove_long). Route CJK queries to the bigram fields, whose ngram
# tokenizer indexes overlapping 2-grams for substring matching.
cjk_query = (
_build_cjk_query(index, raw_query, _CJK_ALL_FIELDS)
_build_ast_cjk_query(index, result.ast, registry)
if _has_cjk(raw_query)
else None
)
@@ -235,22 +490,73 @@ def parse_user_query(
threshold = settings.ADVANCED_FUZZY_SEARCH_THRESHOLD
if threshold is not None:
fuzzy = index.parse_query(
query_str,
DEFAULT_SEARCH_FIELDS,
field_boosts=_FIELD_BOOSTS,
# (prefix=True, distance=1, transposition_cost_one=True) — edit-distance fuzziness
fuzzy_fields={f: (True, 1, True) for f in DEFAULT_SEARCH_FIELDS},
)
# 0.1 boost keeps fuzzy hits ranked below exact matches (intentional)
clauses.append((tantivy.Occur.Should, tantivy.Query.boost_query(fuzzy, 0.1)))
fuzzy = _try_parse_fuzzy_query(index, result.ast, registry)
if fuzzy is not None:
clauses.append(
(tantivy.Occur.Should, tantivy.Query.boost_query(fuzzy, 0.1)),
)
if cjk_query is not None:
clauses.append((tantivy.Occur.Should, cjk_query))
if len(clauses) == 1:
return exact
return tantivy.Query.boolean_query(clauses)
# The fuzzy and CJK clauses are built from positive terms only, so as
# plain Shoulds beside the exact clause they re-admit exactly the
# documents the query excluded. Restate the exclusions once, above the
# whole blend. Redundant against the exact clause, which already
# carries them, but idempotently so.
negations = _negation_clauses(index, result.ast, registry)
if not negations:
return _any_of(clauses)
return tantivy.Query.boolean_query(
[(tantivy.Occur.Must, _any_of(clauses)), *negations],
)
# The three whoosh-compat kinds for a wildcard on a field that cannot
# carry one. d.field_kind supplies the discriminator, so naming the field's
# type needs no second trip through the registry.
_PATTERN_ON_KINDS: Final = frozenset(
{
DiagnosticKind.PATTERN_ON_NUMERIC,
DiagnosticKind.PATTERN_ON_BOOLEAN_EXISTS,
DiagnosticKind.PATTERN_ON_SUBPATH,
},
)
def _diagnostics_to_error(diagnostics: tuple[Diagnostic, ...]) -> SearchQueryError:
errors = [_single_diagnostic_to_error(d) for d in diagnostics]
return errors[0] if len(errors) == 1 else MultipleSearchQueryErrors(errors)
def _single_diagnostic_to_error(d: Diagnostic) -> SearchQueryError:
# d.field is a FieldRef, not a str: str(d.field) gives the canonical
# dotted name (an aliased query, e.g. type:, reports document_type).
field_name = str(d.field) if d.field is not None else None
if d.kind is DiagnosticKind.BAD_DATE:
return InvalidDateQuery(field_name, d.raw_value)
if d.kind is DiagnosticKind.BAD_NUMBER:
return InvalidNumberQuery(field_name, d.raw_value)
if d.kind is DiagnosticKind.TOO_DEEP:
return SearchQueryError("The search query is nested too deeply.")
if d.kind in _PATTERN_ON_KINDS:
kind_label = f" ({d.field_kind.name.lower()})" if d.field_kind else ""
return SearchQueryError(
f"Wildcard patterns are not supported for field "
f"{field_name!r}{kind_label}.",
)
if d.kind is DiagnosticKind.SINGLE_CHAR_BRACKET_RANGE:
field_label = f" for field {field_name!r}" if field_name else ""
return SearchQueryError(
f"{d.raw_value!r} looks like a bracket range{field_label}, but "
"'[' is not a wildcard character on its own. Combine it with a "
"wildcard, e.g. a trailing '*', or double-quote the value to "
"search it as literal text.",
)
logger.warning("Unmapped parse diagnostic %s: %s", d.kind, d.message)
return SearchQueryError("The search query could not be executed.")
def parse_simple_query(
@@ -268,7 +574,7 @@ def parse_simple_query(
CJK substrings the simple analyzer can't (long whitespace-free runs are
dropped by remove_long).
"""
tokens = _simple_query_tokens(raw_query)
tokens = simple_search_tokens(raw_query)
clauses: list[tuple[tantivy.Occur, tantivy.Query]] = []
if tokens:
@@ -291,23 +597,14 @@ def parse_simple_query(
)
for token in tokens
]
simple_query = (
token_queries[0][1]
if len(token_queries) == 1
else tantivy.Query.boolean_query(token_queries)
)
clauses.append((tantivy.Occur.Should, simple_query))
clauses.append((tantivy.Occur.Should, _any_of(token_queries)))
if cjk_fields and _has_cjk(raw_query):
cjk_q = _build_cjk_query(index, raw_query, cjk_fields)
if cjk_q is not None:
clauses.append((tantivy.Occur.Should, cjk_q))
if not clauses:
return tantivy.Query.empty_query()
if len(clauses) == 1:
return clauses[0][1]
return tantivy.Query.boolean_query(clauses)
return _any_of(clauses)
def parse_simple_text_highlight_query(
@@ -322,13 +619,21 @@ def parse_simple_text_highlight_query(
# Strip Tantivy operator chars before tokenizing: this is a plain-text
# highlight query, not a structured boolean query, so +/- are separators.
tokens = _simple_query_tokens(
tokens = simple_search_tokens(
regex.sub(r"[-+]", " ", raw_query, timeout=_REGEX_TIMEOUT),
)
if not tokens:
return tantivy.Query.empty_query()
return index.parse_query(" ".join(tokens), ["content"])
# Quote each token as its own phrase, escaping backslashes and embedded
# quotes. simple search tokens can carry arbitrary Tantivy syntax
# characters (`"`, `:`, `(`, `[`, `/`, ...) that the query-string parser
# would otherwise interpret as query grammar rather than literal text.
quoted_tokens = [
'"' + token.replace("\\", "\\\\").replace('"', '\\"') + '"' for token in tokens
]
return index.parse_query(" ".join(quoted_tokens), ["content"])
def parse_simple_text_query(
@@ -342,7 +647,7 @@ def parse_simple_text_query(
return parse_simple_query(
index,
raw_query,
SIMPLE_SEARCH_FIELDS,
_SIMPLE_SEARCH_FIELDS,
cjk_fields=_CJK_CONTENT_FIELDS,
)
@@ -358,6 +663,6 @@ def parse_simple_title_query(
return parse_simple_query(
index,
raw_query,
TITLE_SEARCH_FIELDS,
_TITLE_SEARCH_FIELDS,
cjk_fields=_CJK_TITLE_FIELDS,
)
src/documents/search/_translate.py
-610
View File
@@ -1,610 +0,0 @@
from __future__ import annotations
from dataclasses import dataclass
from datetime import UTC
from datetime import datetime
from datetime import timedelta
from typing import TYPE_CHECKING
from typing import TypeAlias
import regex
from dateutil.relativedelta import relativedelta
from documents.search._dates import _DATE_KEYWORDS
from documents.search._dates import _DATE_ONLY_FIELDS
from documents.search._dates import _date_only_range
from documents.search._dates import _datetime_range
from documents.search._dates import _field_range_from_dates
from documents.search._dates import _fmt
from documents.search._dates import _precision_bounds
from documents.search._dates import _utc_bounds_for_field
# Compiled regex that matches any known multi-word (or single-word) date keyword
# at the start of a match position, longest alternatives first so "previous week"
# wins over a hypothetical shorter "previous".
_KEYWORD_VALUE_RE = regex.compile(
"|".join(sorted((regex.escape(k) for k in _DATE_KEYWORDS), key=len, reverse=True)),
regex.IGNORECASE,
)
if TYPE_CHECKING:
from datetime import tzinfo
# TODO: this module translates date queries into Tantivy *string* syntax, which
# forces a workaround for something Tantivy's string parser cannot express on
# date fields: open-ended ranges use far-past/far-future string sentinels
# (OPEN_LO/OPEN_HI). These can be replaced with a real tantivy.Query object
# (Query.range_query(..., None) for open bounds) once tantivy-py accepts Python
# datetimes in range_query/term_query on Date fields. That support exists on
# tantivy-py master (PRs #655 + #666) but postdates the pinned 0.26.0 wheel, so
# it is blocked only on a published release > 0.26.0 and a dependency bump.
# (Unparsable dates now raise InvalidDateQuery -> HTTP 400 rather than using a
# no-match string sentinel.)
# Fields that store exact, non-analyzed comma-joined tokens in the index and so
# need explicit comma->AND expansion (Whoosh KEYWORD(commas=True) set).
MULTI_VALUE_FIELDS = frozenset({"tag", "tag_id", "viewer_id"})
# Date fields whose values/ranges get rewritten to RFC3339 Tantivy ranges.
DATE_FIELDS = frozenset({"created", "modified", "added"})
# Field aliases: Whoosh (v2) field names that were renamed in the Tantivy schema.
# Preserved here so v2 queries using the old names continue to work without 400
# errors instead of silently failing. Applied by _render to non-date field tokens.
FIELD_ALIASES: dict[str, str] = {
"type": "document_type",
"type_id": "document_type_id",
"path": "storage_path",
"path_id": "storage_path_id",
}
# Known schema fields: a comma immediately followed by ``<known>:`` is a clause
# separator. Restricting to known fields prevents URL-like ``http:`` misfires.
KNOWN_FIELDS = frozenset(
{
"title",
"content",
"correspondent",
"document_type",
"type", # v2 alias -> document_type
"storage_path",
"path", # v2 alias -> storage_path
"tag",
"tag_id",
"correspondent_id",
"document_type_id",
"type_id", # v2 alias -> document_type_id
"storage_path_id",
"path_id", # v2 alias -> storage_path_id
"owner_id",
"viewer_id",
"asn",
"page_count",
"num_notes",
"created",
"modified",
"added",
"original_filename",
"checksum",
"notes",
"custom_fields",
},
)
_FIELD_RE = regex.compile(r"(?P<field>\w+):")
# Matches the TO separator inside a range bracket. Handles three forms:
# middle: "lo TO hi" (either lo or hi may be empty)
# trailing: "lo TO" (open upper bound)
# leading: "TO hi" (open lower bound)
# Bounds MAY contain internal spaces (e.g. "-7 days"), so we use .*? / .+?
# and split on the whitespace-delimited " TO " / " to " separator.
_RANGE_RE = regex.compile(
r"^\s*(?P<lo>.*?)\s+[Tt][Oo]\s+(?P<hi>.+?)\s*$"
r"|"
r"^\s*(?P<lo2>.+?)\s+[Tt][Oo]\s*$"
r"|"
r"^\s*[Tt][Oo]\s+(?P<hi2>.+?)\s*$",
)
@dataclass(frozen=True, slots=True)
class FieldValue:
field: str
value: str
# Produced by the comma-resolution pass (not by scan()).
@dataclass(frozen=True, slots=True)
class FieldValueList:
field: str
values: tuple[str, ...]
@dataclass(frozen=True, slots=True)
class FieldRange:
field: str
open: str
lo: str
hi: str
close: str
# Produced by the comma-resolution pass (not by scan()).
@dataclass(frozen=True, slots=True)
class Comma:
pass
@dataclass(frozen=True, slots=True)
class Passthrough:
raw: str
Token: TypeAlias = FieldValue | FieldValueList | FieldRange | Comma | Passthrough
_CLOSE: dict[str, str] = {"[": "]", "{": "}"}
def scan(query: str) -> list[Token]:
"""
Tokenize a raw query into date/comma-aware tokens, leaving everything else
as verbatim ``Passthrough`` runs. Non-recursive: finds the first matching
close bracket/quote. Nested brackets are not valid Tantivy range syntax and
pass through verbatim on mismatch.
"""
tokens: list[Token] = []
buf: list[str] = [] # accumulates passthrough chars
i, n = 0, len(query)
while i < n:
matched = _match_field_token(query, i)
if matched is None:
buf.append(query[i])
i += 1
continue
token, i = matched
if buf and buf[-1] == ",":
buf.pop()
_flush(buf, tokens)
tokens.append(Comma())
else:
_flush(buf, tokens)
tokens.append(token)
i = _maybe_comma(query, i, tokens)
_flush(buf, tokens)
return tokens
def _flush(buf: list[str], tokens: list[Token]) -> None:
"""Emit any accumulated passthrough characters as a single token."""
if buf:
tokens.append(Passthrough("".join(buf)))
buf.clear()
def _at_word_boundary(query: str, i: int) -> bool:
"""A field token may begin only at the start or after a non-word character."""
return i == 0 or not (query[i - 1].isalnum() or query[i - 1] == "_")
def _match_field_token(query: str, i: int) -> tuple[Token, int] | None:
"""
If a known ``field:`` token starts at ``i``, consume it and return
``(token, end_index)``; otherwise return None so the caller treats the
character as passthrough. Handles both ``field:[range]`` and ``field:value``,
and returns None when the range/value cannot be consumed.
"""
m = _FIELD_RE.match(query, i)
if m is None or m.group("field") not in KNOWN_FIELDS:
return None
if not _at_word_boundary(query, i):
return None
field = m.group("field")
j = m.end()
if j < len(query) and query[j] in "[{":
return _consume_range(query, j, field)
consumed = _consume_field_value(query, field, j)
if consumed is None:
return None
value, end = consumed
return FieldValue(field, value), end
def _consume_field_value(query: str, field: str, start: int) -> tuple[str, int] | None:
"""
Consume a field value starting at ``start``: a multi-word date keyword phrase
(date fields only), or a bare/quoted value, then absorb any comma-joined
continuation that is not a clause separator. ``resolve_commas`` later splits a
multi-value field's joined value into a ``FieldValueList``; for other fields
the comma stays literal.
"""
n = len(query)
consumed = None
if field in DATE_FIELDS:
km = _KEYWORD_VALUE_RE.match(query, start)
if km is not None and (km.end() >= n or query[km.end()] in " \t),"):
consumed = (km.group(0), km.end())
if consumed is None:
consumed = _consume_value(query, start)
if consumed is None:
return None
value, k = consumed
while k < n and query[k] == ",":
if _looks_like_known_field(query, k + 1):
break # clause separator: left for _maybe_comma to emit a Comma()
more = _consume_value(query, k + 1)
if more is None:
break
value = f"{value},{more[0]}"
k = more[1]
return value, k
def _consume_range(
query: str,
start: int,
field: str,
) -> tuple[FieldRange, int] | None:
"""Consume ``[lo TO hi]`` / ``{lo TO hi}`` from ``start`` (the bracket)."""
open_br = query[start]
close_br = _CLOSE[open_br]
end = query.find(close_br, start + 1)
if end == -1:
return None
inner = query[start + 1 : end]
m = _RANGE_RE.match(inner)
if m is not None:
if m.group("lo") is not None or m.group("hi") is not None:
# Middle form: "lo TO hi" (either may be empty string)
lo = (m.group("lo") or "").strip()
hi = (m.group("hi") or "").strip()
elif m.group("lo2") is not None:
# Trailing form: "lo TO"
lo = m.group("lo2").strip()
hi = ""
else:
# Leading form: "TO hi"
lo = ""
hi = (m.group("hi2") or "").strip()
else:
lo, hi = inner.strip(), ""
return FieldRange(field, open_br, lo, hi, close_br), end + 1
def _consume_value(query: str, start: int) -> tuple[str, int] | None:
"""Consume a bare or quoted field value from ``start``, stopping at comma."""
n = len(query)
if start >= n or query[start] in " \t":
return None
if query[start] in "\"'":
quote = query[start]
end = query.find(quote, start + 1)
if end == -1:
return None
return query[start : end + 1], end + 1
j = start
while j < n and query[j] not in " \t),":
j += 1
return query[start:j], j
def _looks_like_known_field(query: str, pos: int) -> bool:
"""True if a known ``field:`` token starts at ``pos``."""
m = _FIELD_RE.match(query, pos)
return bool(m and m.group("field") in KNOWN_FIELDS)
def _maybe_comma(query: str, i: int, tokens: list) -> int:
"""If a clause-separator comma follows at ``i``, emit ``Comma()`` and advance."""
if i < len(query) and query[i] == "," and _looks_like_known_field(query, i + 1):
tokens.append(Comma())
return i + 1
return i
def resolve_commas(tokens: list) -> list:
"""
Collapse value-list commas into ``FieldValueList`` and keep clause-separator
commas as ``Comma``. (Clause-sep commas are already emitted by ``scan`` via
the value-stop logic; this pass folds value-lists.)
"""
out: list = []
for tok in tokens:
if (
isinstance(tok, FieldValue)
and tok.field in MULTI_VALUE_FIELDS
and "," in tok.value
):
values = tuple(v for v in tok.value.split(",") if v)
out.append(FieldValueList(tok.field, values))
else:
out.append(tok)
return out
class SearchQueryError(ValueError):
"""
Base for user-fixable search query errors.
Carries a message safe to surface to the user (no internal details). The view
layer catches this and returns an HTTP 400, so any future subclass (unknown
field, malformed range, wrapped parser errors) gets the same treatment.
"""
class InvalidDateQuery(SearchQueryError):
"""Raised when a date field value or range bound cannot be parsed."""
def __init__(self, field: str, value: str) -> None:
self.field = field
self.value = value
super().__init__(f"Invalid date value {value!r} for field {field!r}.")
_DIGITS_RE = regex.compile(r"^\d{4}(?:\d{2}){0,2}$")
_ISO_RE = regex.compile(r"^\d{4}(?:-\d{2}(?:-\d{2})?)?$")
def translate_scalar(field: str, value: str, tz: tzinfo) -> str:
"""Translate a bare date-field value to a Tantivy range string."""
bare = value.strip("\"'").lower()
if bare in _DATE_KEYWORDS:
if field in _DATE_ONLY_FIELDS:
return f"{field}:{_date_only_range(bare, tz)}"
return f"{field}:{_datetime_range(bare, tz)}"
digits = value.replace("-", "")
if _DIGITS_RE.match(value) or _ISO_RE.match(value):
bounds = _precision_bounds(digits)
if bounds is None:
raise InvalidDateQuery(field, value)
return _field_range_from_dates(field, bounds[0], bounds[1], tz)
if regex.fullmatch(r"\d{14}", value):
try:
dt = datetime(
int(value[0:4]),
int(value[4:6]),
int(value[6:8]),
int(value[8:10]),
int(value[10:12]),
int(value[12:14]),
tzinfo=UTC,
)
except ValueError:
raise InvalidDateQuery(field, value) from None
iso = _fmt(dt)
return f"{field}:[{iso} TO {iso}]"
# Unrecognized shape -> tell the user their date is malformed rather than
# silently matching nothing or emitting invalid Tantivy syntax.
raise InvalidDateQuery(field, value)
# Open-bound sentinels for date ranges. These far-past/far-future strings allow
# open-ended ranges to be expressed as Tantivy string queries until tantivy-py
# exposes Query.range_query(..., None) on Date fields (see module TODO).
OPEN_LO = "0001-01-01T00:00:00Z"
OPEN_HI = "9999-12-31T23:59:59Z"
# Matches compact now-offset tokens like now-7d, now+1h, now-30m.
_NOW_COMPACT_RE = regex.compile(
r"^now(?P<sign>[+-])(?P<n>\d+)(?P<unit>[dhm])$",
regex.IGNORECASE,
)
# Matches "±N <unit>" Whoosh-style offsets (e.g. -7 days, -1 week, +3 hours).
# Whoosh's own date parser (qparser.dateparse.PlusMinus) additionally accepted
# abbreviated unit spellings (e.g. "yrs", "yr", "y", "mos", "wks", "hrs", "mins",
# "secs"); saved views/searches created under the old Whoosh backend can still
# contain those tokens (e.g. "-999yrs"), so they are accepted here too and
# normalized to a canonical unit via _UNIT_ALIASES below.
_NOW_SPACED_RE = regex.compile(
r"^(?P<sign>[+-])(?P<n>\d+)\s*"
r"(?P<unit>years|year|yrs|yr|ys|y"
r"|months|month|mons|mon|mos|mo"
r"|weeks|week|wks|wk|ws|w"
r"|days|day|dys|dy|ds|d"
r"|hours|hour|hrs|hr|hs|h"
r"|minutes|minute|mins|min|ms|m"
r"|seconds|second|secs|sec|s)$",
regex.IGNORECASE,
)
# Maps every accepted unit spelling (including Whoosh-era abbreviations) to the
# canonical unit name used as a key into the delta map in _resolve_relative_bound.
_UNIT_ALIASES: dict[str, str] = {
alias: canonical
for canonical, aliases in {
"year": ("years", "year", "yrs", "yr", "ys", "y"),
"month": ("months", "month", "mons", "mon", "mos", "mo"),
"week": ("weeks", "week", "wks", "wk", "ws", "w"),
"day": ("days", "day", "dys", "dy", "ds", "d"),
"hour": ("hours", "hour", "hrs", "hr", "hs", "h"),
"minute": ("minutes", "minute", "mins", "min", "ms", "m"),
"second": ("seconds", "second", "secs", "sec", "s"),
}.items()
for alias in aliases
}
def _resolve_relative_bound(token: str) -> datetime | None:
"""
Resolve a relative bound token to an exact UTC instant, or return None.
Supported forms:
- ``now`` -> current UTC instant
- ``now+/-<n>d/h/m`` -> now +/- timedelta (d=days, h=hours, m=minutes)
- ``±N <unit>`` -> now +/- delta; month/year use relativedelta;
unit also accepts Whoosh-era abbreviations
(e.g. "yrs", "mos", "wks", "hrs", "mins", "secs")
"""
stripped = token.strip()
low = stripped.lower()
now = datetime.now(UTC)
if low == "now":
return now
m = _NOW_COMPACT_RE.match(stripped)
if m:
sign = 1 if m.group("sign") == "+" else -1
n = int(m.group("n"))
unit = m.group("unit").lower()
delta = (
sign
* {
"d": timedelta(days=n),
"h": timedelta(hours=n),
"m": timedelta(minutes=n),
}[unit]
)
return now + delta
m = _NOW_SPACED_RE.match(stripped)
if m:
sign = 1 if m.group("sign") == "+" else -1
n = int(m.group("n"))
unit = _UNIT_ALIASES[m.group("unit").lower()]
delta_map: dict[str, timedelta | relativedelta] = {
"second": timedelta(seconds=n),
"minute": timedelta(minutes=n),
"hour": timedelta(hours=n),
"day": timedelta(days=n),
"week": timedelta(weeks=n),
"month": relativedelta(months=n),
"year": relativedelta(years=n),
}
return now - delta_map[unit] if sign == -1 else now + delta_map[unit]
return None
def _bound_datetimes(
field: str,
token: str,
tz: tzinfo,
) -> tuple[datetime, datetime] | None:
"""
Return (floor_dt, ceil_dt) UTC datetimes for a single range bound token, or
None if the token is unparsable. ``now`` and relative offsets resolve to the
current instant (floor == ceil == that instant; no day-flooring).
"""
token = token.strip()
# Try relative/now forms first (before stripping hyphens which would mangle them).
rel = _resolve_relative_bound(token)
if rel is not None:
return rel, rel
# Full ISO datetime token (contains "T"): parse directly and return an exact
# instant (floor == ceil). Python 3.11+ datetime.fromisoformat accepts trailing Z.
if "T" in token:
try:
dt = datetime.fromisoformat(token)
# Ensure timezone-aware UTC result.
dt = dt.replace(tzinfo=UTC) if dt.tzinfo is None else dt.astimezone(UTC)
return dt, dt
except ValueError:
return None
digits = token.replace("-", "")
bounds = _precision_bounds(digits)
if bounds is None:
return None
start, end = bounds
return _utc_bounds_for_field(field, start, end, tz)
def _render(tok: Token, tz: tzinfo) -> str:
"""Render a single token back to a Tantivy query string fragment."""
if isinstance(tok, Passthrough):
return tok.raw
if isinstance(tok, Comma):
return " AND "
if isinstance(tok, FieldValueList):
field = FIELD_ALIASES.get(tok.field, tok.field)
return " AND ".join(f"{field}:{v}" for v in tok.values)
if isinstance(tok, FieldValue):
field = FIELD_ALIASES.get(tok.field, tok.field)
if field in DATE_FIELDS:
return translate_scalar(field, tok.value, tz)
return f"{field}:{tok.value}"
if isinstance(tok, FieldRange):
field = FIELD_ALIASES.get(tok.field, tok.field)
if field in DATE_FIELDS:
return translate_range(field, tok.lo, tok.hi, tz)
return f"{field}:{tok.open}{tok.lo} TO {tok.hi}{tok.close}"
return "" # pragma: no cover
# Post-render operator normalization patterns: collapse repeated whitespace and
# strip spaced/trailing Tantivy boolean operators that would otherwise be invalid.
_MULTI_SPACE_RE = regex.compile(r" {2,}")
_TRAILING_OP_RE = regex.compile(r"\s+[-+]+\s*$")
_SPACED_OP_RE = regex.compile(r"\s+[-+]\s+")
def _normalize_operators(text: str) -> str:
"""
Collapse multiple spaces, strip trailing dangling operators, and replace
spaced operators (`` - `` / `` + ``) with a single space.
Applied only to Passthrough fragments (the rendered output is scanned for
operator artifacts outside bracketed ranges) via a post-render pass on the
full rendered string. This preserves date ranges (``[... TO ...]``) verbatim
while cleaning natural-language separators in the surrounding text.
"""
text = _MULTI_SPACE_RE.sub(" ", text)
text = _TRAILING_OP_RE.sub("", text).strip()
text = _SPACED_OP_RE.sub(" ", text).strip()
return text
def translate_query(raw: str, tz: tzinfo) -> str:
"""Translate a raw Whoosh-style query into Tantivy-compatible syntax."""
tokens = resolve_commas(scan(raw))
rendered = "".join(_render(t, tz) for t in tokens)
return _normalize_operators(rendered)
def translate_range(field: str, lo: str, hi: str, tz: tzinfo) -> str:
"""Translate a date-field ``[lo TO hi]`` range to a Tantivy ISO range string.
Handles partial-date bounds (YYYY, YYYYMM, YYYYMMDD, ISO dash variants),
open bounds (empty string -> OPEN_LO/OPEN_HI), ``now``, and reversed ranges
(swaps tokens before computing floor/ceil so the span is always correct).
"""
lo_s = lo.strip()
hi_s = hi.strip()
# Parse both bounds to (floor, ceil) pairs when present.
lo_pair: tuple[datetime, datetime] | None = None
hi_pair: tuple[datetime, datetime] | None = None
if lo_s:
lo_pair = _bound_datetimes(field, lo_s, tz)
if lo_pair is None:
raise InvalidDateQuery(field, lo_s)
if hi_s:
hi_pair = _bound_datetimes(field, hi_s, tz)
if hi_pair is None:
raise InvalidDateQuery(field, hi_s)
# Detect a reversed range: only swap when BOTH bounds are present.
if lo_pair is not None and hi_pair is not None and lo_pair[0] > hi_pair[0]:
lo_pair, hi_pair = hi_pair, lo_pair
lo_iso = _fmt(lo_pair[0]) if lo_pair is not None else OPEN_LO
# A bound resolves to (floor, ceil) where floor == ceil for an exact instant
# (a full ISO datetime, "now", or a "+/-N unit" offset) and floor != ceil for
# a coarser period token (year/month/day precision). Only the latter needs a
# half-open close: its ceil is the start of the *next* period and must be
# excluded, or that instant (e.g. the 1st of next month) wrongly matches.
if hi_pair is not None:
hi_iso = _fmt(hi_pair[1])
hi_close = "]" if hi_pair[0] == hi_pair[1] else "}"
else:
hi_iso = OPEN_HI
hi_close = "]"
return f"{field}:[{lo_iso} TO {hi_iso}{hi_close}"
src/documents/tests/search/conftest.py
-12
View File
@@ -1,15 +1,11 @@
from __future__ import annotations
import tempfile
from typing import TYPE_CHECKING
import pytest
import tantivy
from documents.search._backend import TantivyBackend
from documents.search._backend import reset_backend
from documents.search._schema import build_schema
from documents.search._tokenizer import register_tokenizers
if TYPE_CHECKING:
from collections.abc import Generator
@@ -35,11 +31,3 @@ def backend() -> Generator[TantivyBackend, None, None]:
finally:
b.close()
reset_backend()
@pytest.fixture(scope="module")
def index() -> tantivy.Index:
"""A real Tantivy index for parse-acceptance tests (module scope for speed)."""
idx = tantivy.Index(build_schema(), path=tempfile.mkdtemp())
register_tokenizers(idx, "english")
return idx
src/documents/tests/search/test_acceptance.py
+541
View File
@@ -0,0 +1,541 @@
"""Result-level acceptance corpus: real documents indexed via build_schema(),
real queries run through parse_user_query(), matched-document-ID sets
asserted, not intermediate ASTs or query strings. This is paperless-ngx's
analogue of whoosh-compat's own tests/emitter/test_acceptance_e2e.py.
Supersedes test_query.py's TestParseUserQuery result-level cases.
"""
from __future__ import annotations
from datetime import UTC
from datetime import datetime
from typing import TYPE_CHECKING
import pytest
import time_machine
from django.contrib.auth.models import User
from documents.models import CustomField
from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import DocumentType
from documents.models import Note
from documents.models import StoragePath
from documents.search._query import parse_user_query
if TYPE_CHECKING:
from documents.search._backend import TantivyBackend
pytestmark = [pytest.mark.search, pytest.mark.django_db]
FROZEN_NOW = datetime(2026, 6, 15, 12, 0, tzinfo=UTC)
def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
return set(backend.search_ids(query, user=None))
def _index(backend: TantivyBackend, **kwargs: object) -> Document:
"""Create a Document and index it in one step, for the common case
where nothing needs to happen between the two (no related Note/
CustomFieldInstance to attach first)."""
doc = Document.objects.create(**kwargs)
backend.add_or_update(doc)
return doc
@pytest.fixture
def indexed_documents(backend: TantivyBackend) -> dict[str, int]:
"""Index a small fixture set, return {label: doc_id} for corpus queries."""
docs = {
"invoice_2020": _index(
backend,
title="Invoice 2020",
content="invoice total due",
checksum="acc-invoice-2020",
archive_serial_number=100,
),
"invoice_2021": _index(
backend,
title="Invoice 2021",
content="invoice total due",
checksum="acc-invoice-2021",
archive_serial_number=101,
),
"invoice_2023": _index(
backend,
title="Invoice 2023",
content="invoice total due",
checksum="acc-invoice-2023",
archive_serial_number=102,
),
"receipt_2022": _index(
backend,
title="Receipt 2022",
content="receipt total due",
checksum="acc-receipt-2022",
archive_serial_number=103,
),
}
return {label: doc.pk for label, doc in docs.items()}
class TestIssue13568BracketWildcard:
"""paperless-ngx#13568: title:202[0-3]* must keep its character class,
not fold to a prefix query that silently drops it."""
def test_bracket_class_wildcard_matches_only_in_range_years(
self,
backend: TantivyBackend,
indexed_documents: dict[str, int],
) -> None:
"""
GIVEN:
- Four indexed documents titled Invoice 2020/2021/2023 and
Receipt 2022
WHEN:
- "title:202[0-1]*" is searched ([0-1], not [0-3], is
deliberate: the fixture's trailing digits are 0/1/2/3, so a
[0-3] class would match all four and pass even if the
character class were silently dropped and folded to an
unconstrained "202*" prefix; [0-1] partitions the fixture
into a genuine in-range/out-of-range split)
THEN:
- Only the 2020 and 2021 documents match, proving the bracket
character class survived (issue #13568's original bug)
"""
matched = _matched_ids(backend, "title:202[0-1]*")
expected = {
indexed_documents["invoice_2020"],
indexed_documents["invoice_2021"],
}
assert matched == expected, (
"title:202[0-1]* must match 2020/2021 titles and exclude 2022/2023 "
"- if this matches everything, the wildcard's character class was "
"silently dropped (issue #13568's original bug)"
)
class TestFieldBoosts:
def test_title_boost_ranks_title_match_above_content_only_match(
self,
backend: TantivyBackend,
) -> None:
"""
GIVEN:
- One document whose title contains the query word and another
whose content (not title) contains it
WHEN:
- The query word is searched unfielded
THEN:
- The title match ranks first, proving our title field boost
actually affects ranking
"""
title_match = _index(
backend,
title="urgent",
content="nothing else relevant",
checksum="acc-boost-title",
)
_index(
backend,
title="nothing",
content="urgent matter here",
checksum="acc-boost-content",
)
query = parse_user_query(backend._index, "urgent", UTC)
searcher = backend._index.searcher()
results = searcher.search(query, limit=10)
ranked_ids = [
searcher.doc(addr).to_dict()["id"][0] for _score, addr in results.hits
]
assert ranked_ids[0] == title_match.pk
class TestJsonSubpaths:
def test_notes_user_matches_document_with_that_note_author(
self,
backend: TantivyBackend,
) -> None:
"""
GIVEN:
- A document with a Note authored by "alice" and a second,
unrelated document with no note
WHEN:
- "notes.user:alice" is searched
THEN:
- Only the document with alice's note matches
"""
alice = User.objects.create_user(username="alice")
doc_with_note = Document.objects.create(
title="Has note",
content="x",
checksum="acc-note-with",
)
Note.objects.create(document=doc_with_note, user=alice, note="reminder")
backend.add_or_update(doc_with_note)
_index(backend, title="No note", content="x", checksum="acc-note-without")
matched = _matched_ids(backend, "notes.user:alice")
assert matched == {doc_with_note.pk}
def test_custom_fields_name_and_value_combine(
self,
backend: TantivyBackend,
) -> None:
"""
GIVEN:
- A document with a "Contract Number" custom field valued
"policy", and a second document with a differently-named
custom field also valued "policy"
WHEN:
- 'custom_fields.name:"Contract Number" custom_fields.value:policy'
is searched
THEN:
- Only the document whose field name AND value both match is
returned
"""
field = CustomField.objects.create(
name="Contract Number",
data_type=CustomField.FieldDataType.STRING,
)
other_field = CustomField.objects.create(
name="Other Field",
data_type=CustomField.FieldDataType.STRING,
)
matching = Document.objects.create(
title="Matching",
content="x",
checksum="acc-cf-matching",
)
CustomFieldInstance.objects.create(
document=matching,
field=field,
value_text="policy",
)
backend.add_or_update(matching)
non_matching = Document.objects.create(
title="Non-matching",
content="x",
checksum="acc-cf-nonmatching",
)
CustomFieldInstance.objects.create(
document=non_matching,
field=other_field,
value_text="policy",
)
backend.add_or_update(non_matching)
matched = _matched_ids(
backend,
'custom_fields.name:"Contract Number" custom_fields.value:policy',
)
assert matched == {matching.pk}
class TestUnregisteredIdFieldFoldsToLiteralText:
"""tag_id, owner_id, etc. are intentionally excluded from the
FieldRegistry - always internal index columns, never meant to be
query-addressable. Prove an unregistered field folds to a literal
text search that matches nothing, rather than erroring."""
def test_tag_id_query_matches_nothing(
self,
backend: TantivyBackend,
indexed_documents: dict[str, int],
) -> None:
"""
GIVEN:
- A real indexed corpus and "tag_id", a field intentionally
excluded from the FieldRegistry (an internal index column,
never meant to be query-addressable)
WHEN:
- "tag_id:5" is searched
THEN:
- It folds to a literal text search and matches nothing,
rather than erroring
"""
matched = _matched_ids(backend, "tag_id:5")
assert matched == set()
class TestFuzzyBlendSurvivesWhooshGrammar:
"""A query mixing whoosh-only grammar (a date keyword) with a typo'd
free-text word must still fuzzy-match the intended document when
ADVANCED_FUZZY_SEARCH_THRESHOLD is enabled. The fuzzy clause is built
from the parsed query's free-text tokens (whoosh_compat's
free_text_tokens), never from the raw query string, so whoosh grammar
that tantivy's own parser rejects cannot knock the fuzzy clause out."""
def test_typo_fuzzy_matches_alongside_date_keyword(
self,
backend: TantivyBackend,
settings,
) -> None:
"""
GIVEN:
- ADVANCED_FUZZY_SEARCH_THRESHOLD enabled, and a document
indexed with content "receipt total due"
WHEN:
- The query blends whoosh-only grammar tantivy's own parser
rejects ("added:today") with a one-transposition misspelling
of a word in the indexed content
THEN:
- The document still matches, because the fuzzy clause is
built from the parsed query's free-text tokens
(whoosh_compat's free_text_tokens), never from the raw
query string, so grammar tantivy's parser cannot handle
cannot knock the fuzzy clause out
"""
settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.5
with time_machine.travel(FROZEN_NOW, tick=False):
doc = _index(
backend,
title="Receipt March",
content="receipt total due",
checksum="fuzzy-blend-1",
archive_serial_number=900,
)
# Sanity: the exact spelling matches through the exact clause.
assert doc.pk in _matched_ids(backend, "added:today receipt")
# The regression: the misspelling (one transposition) only
# matches via the fuzzy clause, and "added:today" is
# whoosh-only grammar tantivy's parser rejects, so raw-string
# fuzzy parsing skips the clause entirely and this returns
# nothing. The typo is deliberate; keep codespell away from it.
typo_query = "added:today reciept" # codespell:ignore reciept
assert doc.pk in _matched_ids(backend, typo_query)
def test_negated_words_do_not_fuzzy_match(
self,
backend: TantivyBackend,
settings,
) -> None:
"""
GIVEN:
- ADVANCED_FUZZY_SEARCH_THRESHOLD enabled, and a document
containing the NOT'd word ("receipt") but not the positive
word ("total"), so nothing matches the exact clause -- the
shape a naive fuzzy string built from ALL words (including
the NOT'd one) would make this document the sole hit,
normalize its score to 1.0, and survive any threshold (a
shape with an exact-matching sibling document would NOT
discriminate: normalization would rank the resurfaced
document far below the exact match and the threshold would
cut it even for a naive implementation)
WHEN:
- "added:today total NOT receipt" is searched
THEN:
- The document does not match; a term the user excluded must
not resurface through the fuzzy clause
"""
settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.5
with time_machine.travel(FROZEN_NOW, tick=False):
_index(
backend,
title="Receipt Archive",
content="receipt archived stack",
checksum="fuzzy-blend-2",
archive_serial_number=901,
)
assert _matched_ids(backend, "added:today total NOT receipt") == set()
class TestUnquotedDateKeywordPhrases:
"""The unquoted spelling (added:previous month) is honored natively by
whoosh-compat's own grammar for this closed phrase vocabulary, no
app-level rewrite is involved. Pins that the historically supported
spelling keeps working now that paperless no longer pre-quotes it."""
@pytest.fixture
def period_documents(self, backend: TantivyBackend) -> dict[str, int]:
with time_machine.travel(FROZEN_NOW, tick=False):
in_may = _index(
backend,
title="May Doc",
content="statement",
checksum="kw-may",
archive_serial_number=910,
added=datetime(2026, 5, 20, 12, 0, tzinfo=UTC),
)
in_june = _index(
backend,
title="June Doc",
content="statement",
checksum="kw-june",
archive_serial_number=911,
added=datetime(2026, 6, 10, 12, 0, tzinfo=UTC),
)
return {"in_may": in_may.pk, "in_june": in_june.pk}
@pytest.mark.parametrize(
"query",
[
pytest.param("added:previous month", id="unquoted"),
pytest.param('added:"previous month"', id="quoted"),
pytest.param("added:Previous Month", id="unquoted-mixed-case"),
],
)
def test_unquoted_matches_the_same_documents_as_quoted(
self,
backend: TantivyBackend,
period_documents: dict[str, int],
query: str,
) -> None:
"""
GIVEN:
- Two documents added in different months, time frozen so
only one falls in "previous month"
WHEN:
- The same date-keyword phrase is spelled unquoted, quoted,
and unquoted with mixed case
THEN:
- All three spellings match the same document; paperless no
longer pre-quotes this phrase before parsing, relying on
whoosh-compat's own grammar to accept it unquoted natively
"""
with time_machine.travel(FROZEN_NOW, tick=False):
assert _matched_ids(backend, query) == {period_documents["in_may"]}
@pytest.mark.parametrize(
"query",
[
pytest.param("added:this month", id="this-month"),
pytest.param("added:this year", id="this-year"),
pytest.param("added:previous week", id="previous-week"),
pytest.param("added:previous quarter", id="previous-quarter"),
pytest.param("added:previous year", id="previous-year"),
pytest.param("created:previous month", id="created-field"),
pytest.param("modified:previous month", id="modified-field"),
],
)
def test_every_phrase_and_date_field_parses_without_error(
self,
backend: TantivyBackend,
period_documents: dict[str, int],
query: str,
) -> None:
"""
GIVEN:
- Our real schema and every date-keyword phrase in the
vocabulary, against every date field we expose (added,
created, modified)
WHEN:
- Each combination is searched
THEN:
- It parses and searches cleanly against our schema (no
SearchQueryError, so no HTTP 400); exact window semantics
are whoosh-compat's own and are pinned in its own suite
"""
with time_machine.travel(FROZEN_NOW, tick=False):
_matched_ids(backend, query)
def test_text_field_keyword_words_are_ordinary_text(
self,
backend: TantivyBackend,
period_documents: dict[str, int],
) -> None:
"""
GIVEN:
- period_documents (indexed by added-date) and a third
document whose title literally contains the words
"previous month"
WHEN:
- "title:previous month" is searched
THEN:
- Only the document whose title contains those words matches;
"previous month" after a TEXT field (or unfielded) is
ordinary text, not a date phrase, so the date-window
documents do not match
"""
with time_machine.travel(FROZEN_NOW, tick=False):
wordy = _index(
backend,
title="Notes from the previous month",
content="meeting notes",
checksum="kw-text",
archive_serial_number=912,
)
assert _matched_ids(backend, "title:previous month") == {wordy.pk}
class TestFieldAliases:
"""type:/path: are registry aliases for document_type:/storage_path:.
The only other alias coverage is parse-shape; these prove resolution
end-to-end against a real index."""
def test_type_alias_and_canonical_name_match_the_same_document(
self,
backend: TantivyBackend,
) -> None:
"""
GIVEN:
- A document with document_type "invoice", and a decoy
document with no type whose content merely mentions
"invoice" (document_type is itself a default search field,
so if alias resolution ever broke and "type:invoice"
demoted to unfielded text, the token would STILL match the
typed document through the field value; the decoy carrying
the query word in content is what makes a demoted search
distinguishable, since it would then match both documents
and fail the exact-set assertion -- the title avoids
stemming to "type": English stems Typed -> type)
WHEN:
- "type:invoice" and "document_type:invoice" are each
searched
THEN:
- Both resolve to the same document, proving the "type" alias
and its canonical field name agree end-to-end against a
real index
"""
invoice_type = DocumentType.objects.create(name="invoice")
typed = _index(
backend,
title="First",
content="quarterly statement",
checksum="alias-type-1",
document_type=invoice_type,
)
_index(
backend,
title="Second",
content="invoice mentioned in body",
checksum="alias-type-2",
)
assert _matched_ids(backend, "type:invoice") == {typed.pk}
assert _matched_ids(backend, "document_type:invoice") == {typed.pk}
def test_path_alias_and_canonical_name_match_the_same_document(
self,
backend: TantivyBackend,
) -> None:
"""
GIVEN:
- A document stored under storage_path "archive", and a decoy
document with no storage_path whose content merely mentions
"archive" (storage_path is NOT a default search field
today, so a demoted "path:archive" already matches nothing;
the content decoy keeps this test discriminating even if
storage_path ever joins the defaults)
WHEN:
- "path:archive" and "storage_path:archive" are each searched
THEN:
- Both resolve to the same document, proving the "path" alias
and its canonical field name agree end-to-end against a
real index
"""
archive = StoragePath.objects.create(name="archive", path="archive/{title}")
stored = _index(
backend,
title="Stored",
content="quarterly statement",
checksum="alias-path-1",
storage_path=archive,
)
_index(
backend,
title="Loose",
content="archive mentioned in body",
checksum="alias-path-2",
)
assert _matched_ids(backend, "path:archive") == {stored.pk}
assert _matched_ids(backend, "storage_path:archive") == {stored.pk}
src/documents/tests/search/test_checksum_prefix_queries.py
+82
View File
@@ -0,0 +1,82 @@
"""``checksum`` wildcard patterns stay literal end to end, once user queries
route through whoosh-compat.
The registry-level fact (the pattern normalizer folds a KEYWORD pattern
rather than stemming it) is pinned on its own in
``test_keyword_pattern_literal.py``. This proves it actually reaches a real
query: ``checksum:ceded*`` must match only the document whose checksum
starts with "ceded", not the one whose checksum stems to the same run.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import pytest
from documents.models import Document
if TYPE_CHECKING:
from documents.search._backend import TantivyBackend
pytestmark = [pytest.mark.search, pytest.mark.django_db]
CEDEF00D = "cedef00ddeadbeef0123456789abcdef01234567"
CEDEDEAD = "cededeadbeef567801234567" + "89abcdef01234567"
class TestChecksumPrefixQueries:
@pytest.fixture
def indexed(self, backend: TantivyBackend) -> None:
for i, checksum in enumerate((CEDEF00D, CEDEDEAD)):
doc = Document.objects.create(
title=f"Checksum doc {i}",
content="invoices for the quarter",
checksum=checksum,
archive_serial_number=940 + i,
)
backend.add_or_update(doc)
def _ids(self, backend: TantivyBackend, query: str) -> set[int]:
return set(backend.search_ids(query, user=None))
def test_prefix_matches_only_the_document_that_starts_with_it(
self,
backend: TantivyBackend,
indexed: None,
) -> None:
"""
GIVEN:
- Two documents indexed with checksums that share a stem when
run through the English stemmer ("cedef00d..." and
"cededead...") but only one literally starts with "ceded"
WHEN:
- "checksum:ceded*" is searched
THEN:
- Only the document whose checksum literally starts with
"ceded" matches; the pattern normalizer folds a KEYWORD
pattern rather than stemming it, so this reaches a real
query end to end
"""
matched = self._ids(backend, "checksum:ceded*")
expected = Document.objects.get(checksum=CEDEDEAD).pk
assert matched == {expected}
def test_text_prefix_still_reaches_the_stemmed_index(
self,
backend: TantivyBackend,
indexed: None,
) -> None:
"""
GIVEN:
- Two documents indexed with content "invoices for the
quarter"
WHEN:
- "invoice*" is searched against the TEXT content field
THEN:
- Both documents match, confirming the checksum field's
literal-pattern behavior is specific to KEYWORD fields and
does not affect TEXT field wildcard matching against
stemmed terms
"""
assert len(self._ids(backend, "invoice*")) == 2
src/documents/tests/search/test_cjk_clause.py
+169
View File
@@ -0,0 +1,169 @@
"""The CJK bigram clause blended into QUERY-mode searches.
The clause exists so CJK runs are matchable at all (the default analyzers
keep a whitespace-free CJK run as one indivisible token), but it must not
widen the query beyond what the user asked for: a CJK term the query
excludes, or restricts to one field, must not come back through it.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import pytest
from documents.models import Document
if TYPE_CHECKING:
from pytest_django.fixtures import SettingsWrapper
from documents.search._backend import TantivyBackend
pytestmark = [pytest.mark.search, pytest.mark.django_db]
def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
return set(backend.search_ids(query, user=None))
def _index(backend: TantivyBackend, **kwargs: object) -> Document:
doc = Document.objects.create(**kwargs)
backend.add_or_update(doc)
return doc
class TestCjkClauseFollowsTheParsedQuery:
def test_negated_cjk_term_is_excluded(self, backend: TantivyBackend) -> None:
"""
GIVEN:
- Two documents both matching "invoice", one whose content
also contains 漢字
WHEN:
- "invoice NOT 漢字" is searched
THEN:
- Only the document without 漢字 matches; 'invoice NOT 漢字'
must not return the document containing 漢字
"""
with_cjk = _index(
backend,
title="Invoice A",
content="invoice total 漢字",
checksum="cjk-neg-1",
)
without_cjk = _index(
backend,
title="Invoice B",
content="invoice total only",
checksum="cjk-neg-2",
)
assert _matched_ids(backend, "invoice") == {with_cjk.pk, without_cjk.pk}
assert _matched_ids(backend, "invoice NOT 漢字") == {without_cjk.pk}
@pytest.mark.parametrize(
("threshold", "expected"),
[
pytest.param(None, {"titled"}, id="fuzzy_off"),
pytest.param(0.0, {"titled", "content_only"}, id="fuzzy_on"),
],
)
def test_fielded_cjk_term_searches_only_that_field(
self,
backend: TantivyBackend,
settings: SettingsWrapper,
threshold: float | None,
expected: set[str],
) -> None:
"""
GIVEN:
- One document with 東京 in its title, another with 東京 only
in its content, and ADVANCED_FUZZY_SEARCH_THRESHOLD either
off or on
WHEN:
- "title:東京" is searched
THEN:
- With fuzzy off, only the titled document matches: the CJK
clause honours the field, so 'title:東京' must not match a
document whose 東京 is only in the content. With fuzzy on,
the content-only document is also readmitted, because the
fuzzy clause contributes every free-text term UNFIELDED by
design (see _try_parse_fuzzy_query) on its own
0.1-boosted terms -- a documented trade-off, pinned here so
it stays deliberate
"""
settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = threshold
content_only = _index(
backend,
title="Tokyo report",
content="東京都の人口は約1400万人です",
checksum="cjk-field-1",
)
titled = _index(
backend,
title="東京都の報告書",
content="an english summary",
checksum="cjk-field-2",
)
pks = {"titled": titled.pk, "content_only": content_only.pk}
assert _matched_ids(backend, "東京") == set(pks.values())
assert _matched_ids(backend, "title:東京") == {pks[label] for label in expected}
def test_cjk_on_a_non_default_field_builds_no_clause(
self,
backend: TantivyBackend,
) -> None:
"""
GIVEN:
- A document with 東京 in its content
WHEN:
- "notes:東京" is searched (a field outside the default
search fields)
THEN:
- Nothing matches; a CJK term restricted to a field outside
the default search fields has nothing to contribute to the
bigram clause, so it must not fall back to matching 東京 in
the content
"""
_index(
backend,
title="Tokyo report",
content="東京都の人口は約1400万人です",
checksum="cjk-notes-1",
)
assert _matched_ids(backend, "notes:東京") == set()
def test_bare_cjk_term_still_matches_every_default_field(
self,
backend: TantivyBackend,
) -> None:
"""
GIVEN:
- One document with 重要 in its content, another with 重要 in
its title
WHEN:
- "重要" and "重要 OR report" are each searched unfielded
THEN:
- Both documents match either way; the clause's reason for
existing is that an unfielded CJK run matches wherever it
is indexed, and does so alongside a latin term
"""
in_content = _index(
backend,
title="report",
content="本文に重要な情報",
checksum="cjk-bare-1",
)
in_title = _index(
backend,
title="重要な報告書",
content="english only",
checksum="cjk-bare-2",
)
assert _matched_ids(backend, "重要") == {in_content.pk, in_title.pk}
assert _matched_ids(backend, "重要 OR report") == {
in_content.pk,
in_title.pk,
}
src/documents/tests/search/test_compact_date_forms.py
+86
View File
@@ -0,0 +1,86 @@
"""Whoosh's compact, separator-free date spelling, resolved end to end.
whoosh-compat owns both widths of this spelling and asserts both forms'
bounds directly in its own test suite: the 8-digit form as a whole calendar
day (lower bound, upper bound and exclusivity), and the 14-digit form as a
single instant. The 14-digit form is kept here as the single representative
because it is the one that exercises paperless's ``added`` DATETIME fast
field at full precision: the corpus separates a document at the named
instant from one on the same calendar day at another hour and one on the
next day at the same hour, so a query that degrades into a whole-day
window, or drops the time of day, matches the wrong set rather than passing
on a corpus that could not tell the difference.
"""
from __future__ import annotations
from datetime import UTC
from datetime import datetime
from typing import TYPE_CHECKING
import pytest
from documents.models import Document
if TYPE_CHECKING:
from documents.search._backend import TantivyBackend
pytestmark = [pytest.mark.search, pytest.mark.django_db]
def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
return set(backend.search_ids(query, user=None))
def _index(backend: TantivyBackend, **kwargs: object) -> Document:
doc = Document.objects.create(**kwargs)
backend.add_or_update(doc)
return doc
@pytest.fixture
def docs(backend: TantivyBackend) -> dict[str, int]:
return {
"instant": _index(
backend,
title="On the instant",
content="x",
checksum="compact-date-instant",
added=datetime(2005, 3, 4, 15, 30, tzinfo=UTC),
).pk,
"same_day": _index(
backend,
title="Same day, other hour",
content="x",
checksum="compact-date-same-day",
added=datetime(2005, 3, 4, 9, 0, tzinfo=UTC),
).pk,
"next_day": _index(
backend,
title="Next day, same hour",
content="x",
checksum="compact-date-next-day",
added=datetime(2005, 3, 5, 15, 30, tzinfo=UTC),
).pk,
}
def test_fourteen_digits_is_a_single_instant(
backend: TantivyBackend,
docs: dict[str, int],
) -> None:
"""
GIVEN:
- Three documents indexed on the ``added`` DATETIME fast field:
one at 2005-03-04T15:30:00, one on the same calendar day at a
different hour, and one on the next day at the same hour
WHEN:
- Searching with the 14-digit compact date form
``added:20050304153000``
THEN:
- Only the document at that exact instant matches; the same-day
document is what tells this apart from the 8-digit day-window
form, and the next-day document from a form that ignored the
time of day altogether
"""
assert _matched_ids(backend, "added:20050304153000") == {docs["instant"]}
src/documents/tests/search/test_date_keyword_phrase_removal.py
+91
View File
@@ -0,0 +1,91 @@
"""Pins the correctness gained by deleting the pre-parse
_quote_date_keyword_phrases rewrite.
That rewrite matched date-keyword phrases (e.g. "previous month" after a
date field) anywhere in the raw query string, including inside an
unrelated quoted string, and inserted quotes mid-phrase there too. Its
own docstring gave ``title:"see added:previous month notes"`` as the
example of what it corrupted. whoosh-compat's grammar accepts the same
phrase vocabulary unquoted natively (see TestUnquotedDateKeywordPhrases
in test_acceptance.py), so the rewrite was redundant everywhere it was
safe and actively wrong everywhere it was not. This is the one case that
tells the two apart: a literal title phrase that happens to contain
"added:previous month" as running text.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import pytest
from documents.models import Document
if TYPE_CHECKING:
from documents.search._backend import TantivyBackend
pytestmark = [pytest.mark.search, pytest.mark.django_db]
def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
return set(backend.search_ids(query, user=None))
def _index(backend: TantivyBackend, **kwargs: object) -> Document:
doc = Document.objects.create(**kwargs)
backend.add_or_update(doc)
return doc
class TestQuotedStringContainingDateKeywordText:
"""A quoted title phrase containing the literal text
"added:previous month" as running words must match on that literal
text alone, never spill into an unfielded search for "previous" and
"month" across the default search fields the way the deleted rewrite
would have decomposed it into."""
def test_matches_only_the_literal_phrase(
self,
backend: TantivyBackend,
) -> None:
"""
GIVEN:
- A document whose title literally contains "see
added:previous month notes", and a decoy document whose
title/content carry the individual fragments the deleted
_quote_date_keyword_phrases rewrite would have decomposed
the phrase into (the decoy would incorrectly match under
the deleted rewrite: its title contains the "see added:"
and " notes" fragments the corrupted parse required as
title phrases, and its content supplies "previous" and
"month" as the decomposed word-match clauses the rewrite
turned the middle of the phrase into)
WHEN:
- 'title:"see added:previous month notes"' is searched
THEN:
- Only the document with the literal phrase matches; it must
never spill into an unfielded search for "previous" and
"month" across the default search fields
"""
literal = _index(
backend,
title="see added:previous month notes",
content="quarterly filing",
checksum="dkp-literal",
archive_serial_number=920,
)
# Under the deleted rewrite, this decoy would incorrectly match:
# its title contains the "see added:" and " notes" fragments the
# corrupted parse required as title phrases, and its content
# supplies "previous" and "month" as the decomposed word-match
# clauses the rewrite turned the middle of the phrase into.
decoy = _index(
backend,
title="see added: quarterly report notes",
content="we reviewed the previous statement about month end",
checksum="dkp-decoy",
archive_serial_number=921,
)
query = 'title:"see added:previous month notes"'
assert _matched_ids(backend, query) == {literal.pk}
assert decoy.pk not in _matched_ids(backend, query)
src/documents/tests/search/test_date_keyword_timezone.py
+100
View File
@@ -0,0 +1,100 @@
"""Date keyword phrases (``today``, etc.) resolved in a non-UTC timezone,
end to end.
paperless's own ``tz=get_current_timezone()`` plumbing
(``TantivyBackend._parse_query``) is exercised elsewhere only for
relative *ranges* (``added:[-1 week to now]``, in
documents/tests/test_api_search.py). This covers a date *keyword*
(``today``), whose day boundary depends on the active timezone the same
way but goes through whoosh-compat's DateParserPlugin resolution instead
of an explicit range.
Discriminating shape: frozen at 2026-06-15T02:00 UTC, which is
2026-06-14T22:00 in America/New_York -- still "today" (06-14) there, but
already "today" (06-15) in UTC. Two documents pin both directions of the
mistake a hardcoded-UTC bug would make:
- ``in_ny_today`` (added 2026-06-14T20:00 UTC = 2026-06-14T16:00 NY) is
inside New York's "today" window and outside a naive UTC-calendar-day
window. A ``tz``-ignoring bug would miss it.
- ``in_utc_calendar_day_only`` (added 2026-06-15T10:00 UTC =
2026-06-15T06:00 NY) is inside a naive UTC-calendar-day window but
outside New York's actual "today" window. A ``tz``-ignoring bug would
wrongly match it.
"""
from __future__ import annotations
from datetime import UTC
from datetime import datetime
from typing import TYPE_CHECKING
import pytest
import time_machine
from documents.models import Document
if TYPE_CHECKING:
from pytest_django.fixtures import SettingsWrapper
from documents.search._backend import TantivyBackend
pytestmark = [pytest.mark.search, pytest.mark.django_db]
FROZEN_NOW = datetime(2026, 6, 15, 2, 0, tzinfo=UTC)
def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
return set(backend.search_ids(query, user=None))
def _index(backend: TantivyBackend, **kwargs: object) -> Document:
doc = Document.objects.create(**kwargs)
backend.add_or_update(doc)
return doc
class TestDateKeywordUsesTheActiveTimezone:
def test_today_matches_the_new_york_calendar_day_not_the_utc_one(
self,
backend: TantivyBackend,
settings: SettingsWrapper,
) -> None:
"""
GIVEN:
- TIME_ZONE set to America/New_York, time frozen at
2026-06-15T02:00 UTC (2026-06-14T22:00 NY -- still "today"
there, but already "today" in UTC), and two documents: one
added inside New York's "today" window but outside a naive
UTC-calendar-day window, the other the reverse (inside a
naive UTC-calendar-day window but outside New York's actual
"today")
WHEN:
- "added:today" is searched
THEN:
- Only the document inside New York's actual "today" window
matches, proving our tz=get_current_timezone() plumbing
resolves the date keyword in the active timezone rather
than a hardcoded UTC calendar day
"""
settings.TIME_ZONE = "America/New_York"
with time_machine.travel(FROZEN_NOW, tick=False):
in_ny_today = _index(
backend,
title="NY today",
content="x",
checksum="tz-keyword-ny-today",
added=datetime(2026, 6, 14, 20, 0, tzinfo=UTC),
)
# Not captured: the exact-set assertion below already proves
# this document (inside a naive UTC-calendar-day window, but
# outside New York's actual "today") does not match.
_index(
backend,
title="UTC calendar day only",
content="x",
checksum="tz-keyword-utc-calendar-day-only",
added=datetime(2026, 6, 15, 10, 0, tzinfo=UTC),
)
assert _matched_ids(backend, "added:today") == {in_ny_today.pk}
src/documents/tests/search/test_default_search_fields_guard.py
+32
View File
@@ -0,0 +1,32 @@
"""``_DEFAULT_SEARCH_FIELDS`` must stay a subset of the registered public
field names.
Nothing enforced this before: a rename in PUBLIC_FIELDS not mirrored in
``_DEFAULT_SEARCH_FIELDS`` (documents/search/_query.py) would 400 every
unfielded search at request time, since ``index.parse_query`` and the
fuzzy/CJK clause builders are handed a field name the schema no longer
has.
"""
from __future__ import annotations
from documents.search._fields import PUBLIC_FIELDS
from documents.search._query import _DEFAULT_SEARCH_FIELDS
class TestDefaultSearchFieldsAreRegistered:
def test_every_default_search_field_is_a_public_field(self) -> None:
"""
GIVEN:
- PUBLIC_FIELDS and _DEFAULT_SEARCH_FIELDS, our own field
tables
WHEN:
- Every name in _DEFAULT_SEARCH_FIELDS is checked against the
registered public field names
THEN:
- Every one is present; a rename in PUBLIC_FIELDS not
mirrored here would 400 every unfielded search at request
time
"""
public_field_names = {f.name for f in PUBLIC_FIELDS}
assert set(_DEFAULT_SEARCH_FIELDS) <= public_field_names
src/documents/tests/search/test_documented_syntax.py
+474
View File
@@ -0,0 +1,474 @@
"""Pins the search syntax that ``docs/usage.md`` promises users.
Every query here appears verbatim, or as a direct paraphrase, in the
"Document searches" section of ``docs/usage.md``. Each case indexes real
documents and asserts on matched document IDs rather than on the parsed
query, because a query that parses cleanly is not necessarily a query that
means what the documentation says it means: ``added:now`` parses without a
single diagnostic and then matches nothing, because it resolves to an
instant rather than to a span.
The negative cases matter as much as the positive ones. They pin the
behaviours the docs explicitly warn about, so that if any of them ever
starts working the warning can be removed deliberately rather than being
left standing as a lie.
"""
from __future__ import annotations
from datetime import UTC
from datetime import datetime
from typing import TYPE_CHECKING
import pytest
import time_machine
from documents.models import Document
from documents.models import Note
from documents.models import Tag
from documents.search._errors import InvalidDateQuery
if TYPE_CHECKING:
from collections.abc import Generator
from django.contrib.auth.models import User
from documents.search._backend import TantivyBackend
pytestmark = [pytest.mark.search, pytest.mark.django_db]
# A Monday, so that "next monday"/"last monday" land a clean week either side.
FROZEN_NOW = datetime(2026, 6, 15, 12, 0, tzinfo=UTC)
# The checksum used in the docs' `checksum:` example.
DOC_CHECKSUM = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"
def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
return set(backend.search_ids(query, user=None))
def _index(backend: TantivyBackend, **kwargs: object) -> Document:
doc = Document.objects.create(**kwargs)
backend.add_or_update(doc)
return doc
class TestLogicalExpressions:
@pytest.fixture
def docs(self, backend: TantivyBackend) -> dict[str, int]:
return {
"secret": _index(
backend,
title="Invoice one",
content="invoice secret contents",
checksum="doc-syntax-secret",
).pk,
"plain": _index(
backend,
title="Invoice two",
content="invoice ordinary contents",
checksum="doc-syntax-plain",
).pk,
}
def test_not_excludes_a_term(
self,
backend: TantivyBackend,
docs: dict[str, int],
) -> None:
"""
GIVEN:
- Two indexed documents, one containing "secret" and one not
WHEN:
- "invoice NOT secret" is searched, as docs/usage.md documents
THEN:
- Only the document without "secret" matches
"""
assert _matched_ids(backend, "invoice NOT secret") == {docs["plain"]}
def test_leading_hyphen_requires_the_term_instead_of_excluding_it(
self,
backend: TantivyBackend,
docs: dict[str, int],
) -> None:
"""
GIVEN:
- Two indexed documents, one containing "secret" and one not
WHEN:
- "invoice -secret" is searched (a leading hyphen, not "NOT")
THEN:
- Only the document containing "secret" matches, because
separators are stripped at index time, so "-secret" is
indexed as the plain term "secret" and the query becomes an
AND rather than an exclusion, exactly as the docs warn
"""
assert _matched_ids(backend, "invoice -secret") == {docs["secret"]}
def test_or_inside_parentheses_matches_either_branch(
self,
backend: TantivyBackend,
docs: dict[str, int],
) -> None:
"""
GIVEN:
- Two indexed documents, one containing "secret" and one
containing "ordinary"
WHEN:
- "invoice AND (secret OR ordinary)" is searched
THEN:
- Both documents match
"""
matched = _matched_ids(backend, "invoice AND (secret OR ordinary)")
assert matched == {docs["secret"], docs["plain"]}
class TestPhraseSearch:
def test_quoted_phrase_requires_the_words_in_order(
self,
backend: TantivyBackend,
) -> None:
"""
GIVEN:
- A document whose content contains "the quick brown fox jumps"
WHEN:
- A quoted phrase is searched, in order and out of order
THEN:
- The in-order phrase matches, and the same words reordered do
not
"""
doc = _index(
backend,
title="Phrase",
content="the quick brown fox jumps",
checksum="doc-syntax-phrase",
)
assert _matched_ids(backend, '"quick brown fox"') == {doc.pk}
assert _matched_ids(backend, '"brown quick fox"') == set()
class TestTagCommaList:
"""``tag:bills,unpaid`` is published syntax (docs/usage.md), so this checks
that the documented spelling still returns what the docs promise: only the
document carrying every listed tag.
It is deliberately not proof of paperless's field configuration, and must
not be read as such. Removing ``comma_values`` from the ``tag`` FieldSpec
leaves this test passing, because paperless's analyzer splits the literal
value "bills,unpaid" into the same two tokens the value-list reading
produces, so the two readings select the same documents. The registry fact
-- that ``tag`` opts in and no other field does -- is observable only at
the registry, and is owned by test_registry.py's
``test_tag_is_comma_values``/``test_correspondent_is_not_comma_values``.
"""
def test_comma_list_requires_every_listed_tag(
self,
backend: TantivyBackend,
) -> None:
"""
GIVEN:
- A document carrying both "bills" and "unpaid" tags, and a
second document carrying only "bills" (plus "archived")
WHEN:
- "tag:bills,unpaid" is searched
THEN:
- Only the document carrying every listed tag matches, and a
single-tag "tag:bills" search still matches both documents
"""
bills = Tag.objects.create(name="bills")
unpaid = Tag.objects.create(name="unpaid")
archived = Tag.objects.create(name="archived")
both = Document.objects.create(
title="Both tags",
content="body",
checksum="doc-syntax-tag-both",
)
both.tags.add(bills, unpaid)
backend.add_or_update(both)
one = Document.objects.create(
title="One tag",
content="body",
checksum="doc-syntax-tag-one",
)
one.tags.add(bills, archived)
backend.add_or_update(one)
assert _matched_ids(backend, "tag:bills,unpaid") == {both.pk}
assert _matched_ids(backend, "tag:bills") == {both.pk, one.pk}
class TestArchiveMetadataFields:
@pytest.fixture
def doc(self, backend: TantivyBackend, admin_user: User) -> Document:
doc = Document.objects.create(
title="Metadata",
content="body",
checksum=DOC_CHECKSUM,
archive_serial_number=100,
page_count=12,
original_filename="invoice.pdf",
)
Note.objects.create(document=doc, user=admin_user, note="a note")
backend.add_or_update(doc)
return doc
@pytest.mark.parametrize(
"query",
[
"asn:100",
"asn:[50 to 150]",
"page_count:12",
"page_count:[10 to 20]",
"num_notes:1",
"num_notes:[1 to 5]",
"original_filename:invoice.pdf",
f"checksum:{DOC_CHECKSUM}",
"checksum:9f86d081*",
# A checksum term is stored verbatim, but a checksum *pattern* is
# lowercased before it is matched, which the docs now say outright
# next to the "only a complete, lowercase checksum matches" rule
# that the uppercase term in the negative list below pins.
"checksum:9F86D081*",
],
)
def test_documented_metadata_query_matches(
self,
backend: TantivyBackend,
doc: Document,
query: str,
) -> None:
"""
GIVEN:
- A document with an ASN, page count, a note, an original
filename and a known checksum
WHEN:
- Every documented metadata-field spelling (exact value,
range, and, for checksum, a lowercase prefix pattern
regardless of the case the pattern itself is typed in) is
searched
THEN:
- Each one matches the document
"""
assert _matched_ids(backend, query) == {doc.pk}
@pytest.mark.parametrize(
"query",
[
# The docs say only a complete, lowercase checksum matches.
"checksum:9f86d081",
f"checksum:{DOC_CHECKSUM.upper()}",
],
)
def test_partial_or_uppercase_checksum_matches_nothing(
self,
backend: TantivyBackend,
doc: Document,
query: str,
) -> None:
"""
GIVEN:
- A document with a known, complete, lowercase checksum
WHEN:
- An exact-value search is run with a partial or uppercase
spelling of that checksum
THEN:
- Nothing matches, as the docs say only a complete, lowercase
checksum matches as an exact value
"""
assert _matched_ids(backend, query) == set()
class TestDocumentedDateForms:
@pytest.fixture(autouse=True)
def frozen_now(self) -> Generator[None, None, None]:
with time_machine.travel(FROZEN_NOW, tick=False):
yield
@pytest.fixture
def dated(self, backend: TantivyBackend) -> dict[str, int]:
stamps = {
"today": datetime(2026, 6, 15, 9, 0, tzinfo=UTC),
"yesterday": datetime(2026, 6, 14, 9, 0, tzinfo=UTC),
"tomorrow": datetime(2026, 6, 16, 9, 0, tzinfo=UTC),
"next_monday": datetime(2026, 6, 22, 10, 0, tzinfo=UTC),
"last_monday": datetime(2026, 6, 8, 10, 0, tzinfo=UTC),
"january": datetime(2026, 1, 10, 10, 0, tzinfo=UTC),
"old": datetime(2005, 3, 4, 15, 30, tzinfo=UTC),
}
return {
label: _index(
backend,
title=label,
content="dated body",
checksum=f"doc-syntax-date-{label}",
added=stamp,
).pk
for label, stamp in stamps.items()
}
@pytest.mark.parametrize(
("query", "label"),
[
("added:today", "today"),
("added:yesterday", "yesterday"),
("added:tomorrow", "tomorrow"),
('added:"next monday"', "next_monday"),
('added:"last monday"', "last_monday"),
("added:january", "january"),
("added:2005-03-04", "old"),
("added:2005-03", "old"),
("added:[2005-01-01 to 2005-12-31]", "old"),
("added:[2005 to 2009]", "old"),
# A full timestamp works, but only quoted when it stands alone,
# and only unquoted when it is a range bound. The bare standalone
# spelling is pinned as a non-match below.
('added:"2005-03-04T15:30:00Z"', "old"),
("added:[2005-03-04T09:00:00Z to 2005-03-04T17:00:00Z]", "old"),
# A quoted range bound works when the quotes are single ones; the
# double-quoted spelling is pinned as an error below.
("added:['2005-03-04' to 2005-03-05]", "old"),
],
)
def test_documented_date_form_matches_its_day_or_month(
self,
backend: TantivyBackend,
dated: dict[str, int],
query: str,
label: str,
) -> None:
"""
GIVEN:
- Documents dated today, yesterday, tomorrow, next/last
Monday, in January, and on an old fixed date, indexed
against a frozen "now" (a Monday)
WHEN:
- Every documented date-form spelling is searched: relative
keywords, quoted multi-word phrases, a bare year-month, an
explicit range, a quoted full timestamp standing alone, an
unquoted full timestamp as a range bound, and a
single-quoted range bound
THEN:
- Each form matches exactly the document dated on its day or
within its month
"""
assert _matched_ids(backend, query) == {dated[label]}
@pytest.mark.parametrize(
"query",
[
# Zero-width: these resolve to a single instant, not a span, so
# nothing in a realistic corpus lands on them. The docs warn
# about them rather than presenting them as usable.
"added:now",
"added:noon",
"added:midnight",
# Quoting is what rescues the other multi-word date expressions,
# so pin that it does not rescue these: the problem is the width
# of the resulting range, not the way the value is delimited.
# One quoted spelling is enough for that; which keyword sits
# inside the quotes is grammar whoosh-compat owns.
'added:"now"',
# A relative offset, which the warning in the docs names by this
# exact spelling. Standing alone it is an instant like the rest of
# this list; the same offset used as a range bound is a real
# window, pinned by the test below.
'added:"-1 week"',
],
)
def test_forms_the_docs_warn_about_match_nothing(
self,
backend: TantivyBackend,
dated: dict[str, int],
query: str,
) -> None:
"""
GIVEN:
- A realistic dated corpus (see the `dated` fixture)
WHEN:
- A zero-width date form ("now", "noon", "midnight", a quoted
"now") or a standalone relative offset ("-1 week") is
searched: each resolves to a single instant rather than a
span, and quoting does not rescue them the way it rescues
other multi-word date expressions, since the problem is the
width of the resulting range, not how the value is
delimited
THEN:
- Nothing matches, exactly as the docs warn, rather than
presenting these as usable spellings
"""
assert _matched_ids(backend, query) == set()
def test_bare_timestamp_is_rejected_rather_than_matching_nothing(
self,
backend: TantivyBackend,
dated: dict[str, int],
) -> None:
"""
GIVEN:
- A realistic dated corpus, including a document dated at a
known full timestamp
WHEN:
- The bare, unquoted spelling of that full timestamp is
searched (the quoted and range-bound spellings pinned above
do work and match this fixture's document)
THEN:
- `InvalidDateQuery` is raised rather than the query silently
matching nothing, since this is a user-fixable error the
docs tell the user to quote, and the reported value is the
whole contiguous fragment the user typed, not just the
prefix the date grammar's tokenizer first split on
"""
with pytest.raises(InvalidDateQuery) as exc_info:
_matched_ids(backend, "added:2005-03-04T15:30:00Z")
assert exc_info.value.field == "added"
assert exc_info.value.value == "2005-03-04T15:30:00Z"
def test_relative_offset_as_a_range_bound_is_a_real_window(
self,
backend: TantivyBackend,
dated: dict[str, int],
) -> None:
"""
GIVEN:
- A realistic dated corpus, including a document dated two
hours before a "last Monday to now" window opens, and
documents dated today and yesterday, inside that window
WHEN:
- "added:['-1 week' to now]" is searched: the same offset
that matches nothing standing alone (see the test above),
used here as a range bound instead
THEN:
- The window matches today and yesterday but excludes the
document two hours before it opens, showing the bound is
the offset itself and not a whole-day rounding of it, as
the docs say next to the warning about the standalone form
"""
assert _matched_ids(backend, "added:['-1 week' to now]") == {
dated["today"],
dated["yesterday"],
}
def test_double_quoted_range_bound_is_rejected(
self,
backend: TantivyBackend,
dated: dict[str, int],
) -> None:
"""
GIVEN:
- A realistic dated corpus
WHEN:
- A range bound is double-quoted rather than single-quoted
("added:[\"2005-03-04\" to 2005-03-05]")
THEN:
- `InvalidDateQuery` is raised, pinning which of the two
quote characters fails: quoting a range bound is allowed,
but only with single quotes, since the double-quoted
spelling reaches the date grammar with its quotes still
attached and is not a recognizable date
"""
with pytest.raises(InvalidDateQuery) as exc_info:
_matched_ids(backend, 'added:["2005-03-04" to 2005-03-05]')
assert exc_info.value.value == '"2005-03-04"'
src/documents/tests/search/test_error_routing.py
+357
View File
@@ -0,0 +1,357 @@
"""Diagnostics route by Cause, and user-facing messages are host-owned.
whoosh-compat documents ``Diagnostic.message`` as developer output with no
stability guarantee, so it must never reach an HTTP response body.
"""
from __future__ import annotations
import logging
from datetime import UTC
import pytest
import tantivy
from whoosh_compat.errors import Diagnostic
from whoosh_compat.errors import DiagnosticKind
from whoosh_compat.errors import QueryError
from whoosh_compat.errors import cause_for
from whoosh_compat.fields import FieldKind
from whoosh_compat.fields import FieldRef
from documents.search._errors import SearchQueryError
from documents.search._query import _map_emit_error
from documents.search._query import _single_diagnostic_to_error
from documents.search._query import parse_user_query
from documents.search._schema import build_schema
from documents.search._tokenizer import register_tokenizers
pytestmark = pytest.mark.search
_LIBRARY_PROSE = "INTERNAL LIBRARY WORDING WITH raw tantivy detail"
@pytest.fixture(scope="module")
def query_index() -> tantivy.Index:
"""An in-memory, unstemmed index; these tests only parse, never index."""
idx = tantivy.Index(build_schema(), path=None)
register_tokenizers(idx, "")
return idx
def _diagnostic(
kind: DiagnosticKind,
*,
field: FieldRef | None = FieldRef("title"),
field_kind: FieldKind | None = FieldKind.TEXT,
) -> Diagnostic:
"""A Diagnostic shaped like the emitter's, with the library's own
kind -> cause mapping rather than a hand-picked cause."""
return Diagnostic(
kind=kind,
cause=cause_for(kind),
message=_LIBRARY_PROSE,
field=field,
field_kind=field_kind,
)
class TestEmitErrorRouting:
"""Every Cause gets a distinguishable treatment, not just "a 400"."""
@pytest.mark.parametrize(
"kind",
[
DiagnosticKind.BACKEND_REJECTED,
DiagnosticKind.AST_INVALID_SHAPE,
DiagnosticKind.AST_UNKNOWN_FIELD,
],
)
def test_internal_cause_is_not_converted(self, kind: DiagnosticKind) -> None:
"""
GIVEN:
- A QueryError wrapping a Diagnostic whose Cause is INTERNAL
(BACKEND_REJECTED/AST_INVALID_SHAPE/AST_UNKNOWN_FIELD)
WHEN:
- _map_emit_error processes it
THEN:
- The original QueryError propagates unchanged, so it surfaces
as a 500 monitoring can see, never a 400 blaming the user
"""
error = QueryError(_diagnostic(kind))
with pytest.raises(QueryError) as excinfo:
_map_emit_error(error)
assert excinfo.value is error
def test_misconfigured_cause_is_logged_and_becomes_a_400(
self,
caplog: pytest.LogCaptureFixture,
) -> None:
"""
GIVEN:
- A QueryError for SCHEMA_FIELD_MISSING naming field "asn"
WHEN:
- _map_emit_error processes it
THEN:
- It becomes a SearchQueryError, and exactly one ERROR log
record is emitted naming the field and the diagnostic kind
"""
kind = DiagnosticKind.SCHEMA_FIELD_MISSING
with caplog.at_level(logging.ERROR, logger="paperless.search"):
error = _map_emit_error(
QueryError(_diagnostic(kind, field=FieldRef("asn"))),
)
assert isinstance(error, SearchQueryError)
errors = [r for r in caplog.records if r.levelno == logging.ERROR]
assert len(errors) == 1
assert "asn" in errors[0].getMessage()
assert kind.name in errors[0].getMessage()
@pytest.mark.parametrize(
"kind",
[
DiagnosticKind.TEXT_RANGE,
DiagnosticKind.PATTERN_TOO_COMPLEX,
DiagnosticKind.EXISTS_REQUIRES_FAST,
],
)
def test_unsupported_cause_is_a_400_with_no_operator_log(
self,
kind: DiagnosticKind,
caplog: pytest.LogCaptureFixture,
) -> None:
"""
GIVEN:
- A QueryError for a query tantivy cannot run
(TEXT_RANGE/PATTERN_TOO_COMPLEX/EXISTS_REQUIRES_FAST)
WHEN:
- _map_emit_error processes it
THEN:
- It becomes a SearchQueryError with no log record at WARNING
or above; a query tantivy cannot run is the user's to fix,
not an operator alert. EXISTS_REQUIRES_FAST is nominally
MISCONFIGURED but belongs here: it is decided from the
registry's own FieldSpec, so it never reports a disagreement
anyone could resolve
"""
with caplog.at_level(logging.WARNING, logger="paperless.search"):
error = _map_emit_error(QueryError(_diagnostic(kind)))
assert isinstance(error, SearchQueryError)
assert caplog.records == []
@pytest.mark.parametrize(
"kind",
[
DiagnosticKind.TEXT_RANGE,
DiagnosticKind.PATTERN_TOO_COMPLEX,
DiagnosticKind.EXISTS_REQUIRES_FAST,
DiagnosticKind.SCHEMA_FIELD_MISSING,
],
)
def test_user_facing_message_never_echoes_library_prose(
self,
kind: DiagnosticKind,
) -> None:
"""
GIVEN:
- A QueryError carrying whoosh-compat's own developer-facing
message text
WHEN:
- _map_emit_error processes it
THEN:
- The resulting error's string never contains that library
prose
"""
error = _map_emit_error(QueryError(_diagnostic(kind)))
assert _LIBRARY_PROSE not in str(error)
@pytest.mark.parametrize(
"kind",
[
DiagnosticKind.TEXT_RANGE,
DiagnosticKind.PATTERN_TOO_COMPLEX,
DiagnosticKind.EXISTS_REQUIRES_FAST,
DiagnosticKind.SCHEMA_FIELD_MISSING,
],
)
def test_user_facing_message_names_the_field(
self,
kind: DiagnosticKind,
) -> None:
"""
GIVEN:
- A QueryError for a JSON subpath field (custom_fields.value)
WHEN:
- _map_emit_error processes it
THEN:
- The resulting error names the field using its canonical
dotted form, including the subpath (FieldRef.__str__ yields
this dotted name, so every user-reachable emit kind can name
it)
"""
diagnostic = _diagnostic(
kind,
field=FieldRef("custom_fields", "value"),
field_kind=FieldKind.JSON,
)
error = _map_emit_error(QueryError(diagnostic))
assert "custom_fields.value" in str(error)
class TestParseDiagnosticMessages:
"""Parse-time diagnostics are host-worded too, off field_kind."""
def test_too_deep_is_a_400_without_library_prose(self) -> None:
"""
GIVEN:
- A parse-time Diagnostic for TOO_DEEP with no field
WHEN:
- _single_diagnostic_to_error processes it
THEN:
- It becomes a SearchQueryError with no library prose in its
message
"""
error = _single_diagnostic_to_error(
_diagnostic(DiagnosticKind.TOO_DEEP, field=None, field_kind=None),
)
assert isinstance(error, SearchQueryError)
assert _LIBRARY_PROSE not in str(error)
@pytest.mark.parametrize(
("kind", "field_kind"),
[
(DiagnosticKind.PATTERN_ON_NUMERIC, FieldKind.U64),
(DiagnosticKind.PATTERN_ON_BOOLEAN_EXISTS, FieldKind.BOOLEAN_EXISTS),
(DiagnosticKind.PATTERN_ON_SUBPATH, FieldKind.JSON),
],
)
def test_pattern_on_kinds_name_the_field_and_its_kind(
self,
kind: DiagnosticKind,
field_kind: FieldKind,
) -> None:
"""
GIVEN:
- A parse-time Diagnostic for a pattern used against a kind
that cannot take one
(PATTERN_ON_NUMERIC/PATTERN_ON_BOOLEAN_EXISTS/PATTERN_ON_SUBPATH)
WHEN:
- _single_diagnostic_to_error processes it
THEN:
- The message names both the field and its kind, with no
library prose
"""
error = _single_diagnostic_to_error(
_diagnostic(kind, field=FieldRef("asn"), field_kind=field_kind),
)
message = str(error)
assert _LIBRARY_PROSE not in message
assert "asn" in message
assert field_kind.name.lower() in message
def test_single_char_bracket_range_names_the_field_and_the_value(self) -> None:
"""
GIVEN:
- A SINGLE_CHAR_BRACKET_RANGE diagnostic for "title" with
raw_value "200[1-9]"
WHEN:
- _single_diagnostic_to_error processes it
THEN:
- The resulting SearchQueryError names both the field and the
offending value, with no library prose
"""
diagnostic = Diagnostic(
kind=DiagnosticKind.SINGLE_CHAR_BRACKET_RANGE,
cause=cause_for(DiagnosticKind.SINGLE_CHAR_BRACKET_RANGE),
message=_LIBRARY_PROSE,
field=FieldRef("title"),
field_kind=FieldKind.TEXT,
raw_value="200[1-9]",
)
error = _single_diagnostic_to_error(diagnostic)
message = str(error)
assert isinstance(error, SearchQueryError)
assert _LIBRARY_PROSE not in message
assert "title" in message
assert "200[1-9]" in message
class TestRealQueriesRouteCorrectly:
"""The routing table against diagnostics emit() really produces."""
def test_text_range_is_a_400_naming_the_field(
self,
query_index: tantivy.Index,
) -> None:
"""
GIVEN:
- A real query index
WHEN:
- parse_user_query is called with a text-range query
("title:[a to b]")
THEN:
- It raises SearchQueryError naming "title"
"""
with pytest.raises(SearchQueryError) as excinfo:
parse_user_query(query_index, "title:[a to b]", UTC)
assert "title" in str(excinfo.value)
def test_wildcard_on_a_numeric_field_is_a_400_naming_the_field(
self,
query_index: tantivy.Index,
) -> None:
"""
GIVEN:
- A real query index
WHEN:
- parse_user_query is called with a wildcard on a numeric
field ("asn:12*")
THEN:
- It raises SearchQueryError naming "asn"
"""
with pytest.raises(SearchQueryError) as excinfo:
parse_user_query(query_index, "asn:12*", UTC)
assert "asn" in str(excinfo.value)
def test_single_char_bracket_range_is_a_400_naming_field_and_value(
self,
query_index: tantivy.Index,
) -> None:
"""
GIVEN:
- A real query index
WHEN:
- parse_user_query is called with "title:200[1-9]"
THEN:
- It raises SearchQueryError naming both "title" and
"200[1-9]"
"""
with pytest.raises(SearchQueryError) as excinfo:
parse_user_query(query_index, "title:200[1-9]", UTC)
message = str(excinfo.value)
assert "title" in message
assert "200[1-9]" in message
def test_internal_diagnostic_escapes_as_a_query_error(
self,
query_index: tantivy.Index,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
GIVEN:
- tantivy_emit monkeypatched to raise a QueryError with an
INTERNAL-cause diagnostic (BACKEND_REJECTED), the one case
with no query text of its own involved
WHEN:
- parse_user_query runs a normal query ("invoice")
THEN:
- The QueryError propagates unconverted; emit() reporting a
defect in itself must not become a user-facing 400
"""
import documents.search._query as query_mod
def raise_internal(*args: object, **kwargs: object) -> None:
raise QueryError(_diagnostic(DiagnosticKind.BACKEND_REJECTED))
monkeypatch.setattr(query_mod, "tantivy_emit", raise_internal)
with pytest.raises(QueryError):
parse_user_query(query_index, "invoice", UTC)
src/documents/tests/search/test_exists_on_json_fields.py
+114
View File
@@ -0,0 +1,114 @@
"""``field:*`` on a JSON field is user error, not an operator alert.
whoosh-compat classifies EXISTS_REQUIRES_FAST as MISCONFIGURED, and
_map_emit_error used to route every MISCONFIGURED diagnostic to an ERROR log.
But the kind is decided from the registry's own FieldSpec (kind plus fast)
without consulting the index schema, and field_descriptors() builds the JSON
fields non-fast deliberately, so nothing is misconfigured and no operator
action can clear the condition. Any authenticated user could otherwise emit
ERROR lines in a loop by repeating ``notes:*``.
SCHEMA_FIELD_MISSING, the other MISCONFIGURED kind, does compare the registry
against the live schema, so it stays an ERROR.
"""
from __future__ import annotations
import logging
from datetime import UTC
import pytest
import tantivy
from whoosh_compat.errors import Diagnostic
from whoosh_compat.errors import DiagnosticKind
from whoosh_compat.errors import QueryError
from whoosh_compat.errors import cause_for
from whoosh_compat.fields import FieldKind
from whoosh_compat.fields import FieldRef
from documents.search._errors import SearchQueryError
from documents.search._query import _map_emit_error
from documents.search._query import parse_user_query
from documents.search._schema import build_schema
from documents.search._tokenizer import register_tokenizers
pytestmark = pytest.mark.search
# Every spelling of "does this JSON field have a value" a user can type.
EXISTS_QUERIES = [
"notes:*",
"notes.note:*",
"notes.user:*",
"custom_fields:*",
"custom_fields.name:*",
"custom_fields.value:*",
]
@pytest.fixture(scope="module")
def query_index() -> tantivy.Index:
idx = tantivy.Index(build_schema(), path=None)
register_tokenizers(idx, "")
return idx
class TestJsonExistsIsUserError:
@pytest.mark.parametrize("query", EXISTS_QUERIES)
def test_query_is_a_400_that_emits_no_error_log(
self,
query_index: tantivy.Index,
caplog: pytest.LogCaptureFixture,
query: str,
) -> None:
"""
GIVEN:
- A real query index, and every spelling of "does this JSON
field have a value" (notes:*, notes.note:*, custom_fields:*,
etc.)
WHEN:
- parse_user_query runs the query
THEN:
- It raises SearchQueryError naming the field, and no
ERROR-level log record is emitted; EXISTS_REQUIRES_FAST on a
JSON field is by design, not a misconfiguration an operator
could act on
"""
with caplog.at_level(logging.WARNING, logger="paperless.search"):
with pytest.raises(SearchQueryError) as excinfo:
parse_user_query(query_index, query, UTC)
assert query.split(":", maxsplit=1)[0] in str(excinfo.value)
assert [r for r in caplog.records if r.levelno >= logging.ERROR] == []
class TestGenuineMisconfigurationStillLogs:
def test_schema_field_missing_is_an_error_log(
self,
caplog: pytest.LogCaptureFixture,
) -> None:
"""
GIVEN:
- A QueryError for SCHEMA_FIELD_MISSING: the registry naming a
field the index schema does not have
WHEN:
- _map_emit_error processes it
THEN:
- It becomes a SearchQueryError and logs exactly one ERROR
record naming the diagnostic kind, since this is a real
mismatch an operator can fix and keeps the alert
"""
kind = DiagnosticKind.SCHEMA_FIELD_MISSING
error = QueryError(
Diagnostic(
kind=kind,
cause=cause_for(kind),
message="field 'asn' is not defined in the index schema",
field=FieldRef("asn"),
field_kind=FieldKind.U64,
),
)
with caplog.at_level(logging.ERROR, logger="paperless.search"):
mapped = _map_emit_error(error)
assert isinstance(mapped, SearchQueryError)
records = [r for r in caplog.records if r.levelno == logging.ERROR]
assert len(records) == 1
assert kind.name in records[0].getMessage()
src/documents/tests/search/test_fuzzy_tokenization.py
+225
View File
@@ -0,0 +1,225 @@
"""The words the fuzzy blend clause hands back to tantivy's parser.
The clause re-parses a word string through tantivy, which analyzes it
again, so the words must be the query's raw text rather than the analyzed
text (analysis is not idempotent), and must still be split into plain
words so that hyphenated, dotted and quoted terms keep contributing.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import pytest
from documents.models import Document
if TYPE_CHECKING:
from pytest_django.fixtures import SettingsWrapper
from documents.search._backend import TantivyBackend
pytestmark = [pytest.mark.search, pytest.mark.django_db]
def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
return set(backend.search_ids(query, user=None))
def _index(backend: TantivyBackend, **kwargs: object) -> Document:
doc = Document.objects.create(**kwargs)
backend.add_or_update(doc)
return doc
@pytest.fixture(autouse=True)
def fuzzy_enabled(settings: SettingsWrapper) -> None:
"""Enable the fuzzy blend clause. The threshold doubles as a minimum
score filter, so it is set to 0.0: every hit passes and the test sees
the clause's matching behaviour, not the filter's."""
settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.0
class TestFuzzyClauseWords:
def test_a_stemmed_word_is_not_stemmed_a_second_time(
self,
backend: TantivyBackend,
) -> None:
"""
GIVEN:
- Documents whose content contains "universities", a
one-transposition typo of it ("universties"), and two
unrelated words that share its stem prefix ("univalent",
"unicycle")
WHEN:
- Searching for "universities" with the fuzzy blend enabled
THEN:
- Only the correctly-spelled document and its typo match; the
clause does not widen far enough to reach the unrelated
words. 'universities' stems to 'univers'; feeding that back
to tantivy would stem it again to 'univ', whose fuzzy prefix
reaches unrelated words - the clause must stay wide enough
for a typo and no wider
"""
wanted = _index(
backend,
title="A",
content="universities of europe",
checksum="fuzz-stem-1",
)
typo = _index(
backend,
title="B",
content="universties of europe",
checksum="fuzz-stem-2",
)
_index(
backend,
title="C",
content="univalent chemical bonds",
checksum="fuzz-stem-3",
)
_index(
backend,
title="D",
content="unicycle repair manual",
checksum="fuzz-stem-4",
)
assert _matched_ids(backend, "universities") == {wanted.pk, typo.pk}
def test_a_hyphenated_term_still_reaches_the_clause(
self,
backend: TantivyBackend,
) -> None:
"""
GIVEN:
- A document whose content contains a near-miss of "COVID-19"
("covidx")
WHEN:
- Searching for "COVID-19" with the fuzzy blend enabled
THEN:
- The document matches; 'COVID-19' is one raw token, so unless
it is split into words, it carries characters the re-parse
would read as grammar, is dropped, and the whole query loses
its fuzzy clause
"""
misspelled = _index(
backend,
title="A",
content="covidx testing results",
checksum="fuzz-hyphen-1",
)
assert _matched_ids(backend, "COVID-19") == {misspelled.pk}
def test_a_phrase_still_reaches_the_clause(
self,
backend: TantivyBackend,
) -> None:
"""
GIVEN:
- A document whose content near-misses a quoted phrase
WHEN:
- Searching for the quoted phrase '"tax reports"' with the
fuzzy blend enabled
THEN:
- The document matches; a phrase is one raw token carrying a
space, and is the whole query's only free text here, so it
must still reach the clause
"""
near_miss = _index(
backend,
title="A",
content="taxation reportage weekly",
checksum="fuzz-phrase-1",
)
assert _matched_ids(backend, '"tax reports"') == {near_miss.pk}
class TestBooleanKeywordsInRawText:
"""Tantivy's boolean keywords are word runs, so they survive the cut
into words and its own parser reads them as grammar. Raw query text
reaches that parser with its case intact, so a quoted phrase can carry
them in."""
@pytest.fixture
def corpus(self, backend: TantivyBackend) -> dict[str, int]:
both = _index(
backend,
title="A",
content="taxation reportage weekly",
checksum="fuzz-kw-1",
)
tax_only = _index(
backend,
title="B",
content="taxation only here",
checksum="fuzz-kw-2",
)
report_only = _index(
backend,
title="C",
content="reportage only here",
checksum="fuzz-kw-3",
)
return {
"both": both.pk,
"tax_only": tax_only.pk,
"report_only": report_only.pk,
}
@pytest.mark.parametrize(
"query",
[
pytest.param('"tax AND reports"', id="and"),
pytest.param('"tax OR reports"', id="or"),
pytest.param('"tax NOT reports"', id="not"),
pytest.param('"tax IN reports"', id="in"),
],
)
def test_a_keyword_inside_a_phrase_stays_an_ordinary_word(
self,
backend: TantivyBackend,
corpus: dict[str, int],
query: str,
) -> None:
"""
GIVEN:
- Three documents: one with both "taxation" and "reportage",
one with only "taxation", one with only "reportage"
WHEN:
- Searching for a quoted phrase carrying a tantivy boolean
keyword as one of its words (e.g. '"tax AND reports"')
THEN:
- The keyword stays an ordinary word inside the phrase, and
the fuzzy clause matches all three documents, the same
disjunction as the plain '"tax reports"' phrase: AND must
not turn it into a conjunction, NOT must not give it its own
exclusion, IN must not fail the parse
"""
assert _matched_ids(backend, '"tax reports"') == set(corpus.values())
assert _matched_ids(backend, query) == set(corpus.values())
def test_a_trailing_keyword_does_not_drop_the_clause(
self,
backend: TantivyBackend,
corpus: dict[str, int],
) -> None:
"""
GIVEN:
- Three documents: one with both "taxation" and "reportage",
one with only "taxation", one with only "reportage"
WHEN:
- Searching for '"tax AND"', a phrase ending in a tantivy
syntax error
THEN:
- The fuzzy clause still matches on "tax"; 'tax AND' alone is
a syntax error to tantivy's parser, which would otherwise
cost the whole query its fuzzy clause
"""
assert _matched_ids(backend, '"tax AND"') == {
corpus["both"],
corpus["tax_only"],
}
src/documents/tests/search/test_highlight_query_guard.py
+249
View File
@@ -0,0 +1,249 @@
"""Regression coverage for the unguarded TEXT-mode highlight query.
parse_simple_text_highlight_query re-parses simple-search tokens through
Tantivy's query-string parser to build a SnippetGenerator-compatible query.
Simple-search tokens keep arbitrary punctuation (quotes, colons, brackets,
slashes), so any token carrying Tantivy query grammar raised an unguarded
ValueError. The search itself had already succeeded by the time this ran:
only the highlight step failed, and with the DocumentViewSet.list
exception handler narrowed elsewhere on this branch, that ValueError now
reaches the client as a bare 500 rather than a 400.
Covers three angles:
- the query builder itself: quoting each token as its own escaped phrase
should let it parse instead of raising, for every failure mode a plain-
text query can trigger (syntax error, unknown field, unsupported regex).
- highlight_hits: even when a token still can't be expressed as a
highlight query, the guard must fall back to a query that still
produces usable highlight HTML, not silently empty ones.
- the real API endpoint: pinning the previously-500 status to 200.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import pytest
import tantivy
from rest_framework import status
from documents.search._backend import SearchMode
from documents.search._query import parse_simple_text_highlight_query
from documents.search._schema import build_schema
from documents.search._tokenizer import register_tokenizers
from documents.tests.factories import DocumentFactory
if TYPE_CHECKING:
from rest_framework.test import APIClient
from documents.search._backend import TantivyBackend
pytestmark = [pytest.mark.search, pytest.mark.django_db]
# Each spelling below trips a different Tantivy parser failure mode:
# 'a"b' -> Syntax Error (unterminated quote)
# foo:bar -> unknown field
# (a -> Syntax Error (unbalanced group)
# [a -> Syntax Error (unbalanced range)
# /a/ -> Unsupported query (regex queries disallowed)
_MALFORMED_QUERIES = [
pytest.param('a"b', id="unterminated_quote"),
pytest.param("foo:bar", id="unknown_field"),
pytest.param("(a", id="unbalanced_group"),
pytest.param("[a", id="unbalanced_range"),
pytest.param("/a/", id="unsupported_regex"),
]
@pytest.fixture(scope="module")
def query_index() -> tantivy.Index:
"""An in-memory, unstemmed index for parse-only tests."""
schema = build_schema()
idx = tantivy.Index(schema, path=None)
register_tokenizers(idx, "")
return idx
class TestParseSimpleTextHighlightQueryDoesNotRaise:
"""The query builder itself must tolerate Tantivy syntax in its tokens."""
@pytest.mark.parametrize("raw_query", _MALFORMED_QUERIES)
def test_malformed_token_does_not_raise(
self,
query_index: tantivy.Index,
raw_query: str,
) -> None:
"""
GIVEN:
- A simple-search query token carrying Tantivy query grammar
(unterminated quote, unknown field, unbalanced group/range,
or unsupported regex)
WHEN:
- parse_simple_text_highlight_query builds a highlight query
from it
THEN:
- It returns a tantivy.Query instead of raising, since each
token is quoted as its own escaped phrase rather than fed
to the parser raw
"""
assert isinstance(
parse_simple_text_highlight_query(query_index, raw_query),
tantivy.Query,
)
class TestHighlightHitsProducesUsableHighlights:
"""highlight_hits must keep producing real <b>-wrapped snippet HTML for
these queries, not merely avoid raising."""
@pytest.mark.parametrize(
"raw_query",
[*_MALFORMED_QUERIES, pytest.param("plain text", id="plain_text_sanity")],
)
def test_highlight_still_contains_matched_text(
self,
backend: TantivyBackend,
raw_query: str,
) -> None:
"""
GIVEN:
- A document whose content contains the raw query text
verbatim
WHEN:
- backend.highlight_hits builds highlights for a TEXT-mode
search using that same (possibly Tantivy-grammar-carrying)
query text
THEN:
- The hit still carries a content highlight with real
<b>-wrapped matched-term markup, not an empty fallback
"""
doc = DocumentFactory.create(
title="probe",
content=f"needle content containing {raw_query} literally here",
)
backend.add_or_update(doc)
hits = backend.highlight_hits(
raw_query,
[doc.pk],
search_mode=SearchMode.TEXT,
)
assert len(hits) == 1
highlights = hits[0]["highlights"]
assert "content" in highlights, (
f"Expected a content highlight for {raw_query!r}, got: {highlights!r}"
)
assert "<b>" in highlights["content"], (
f"Highlight for {raw_query!r} carries no matched-term markup: "
f"{highlights['content']!r}"
)
class TestHighlightGuardDiscriminatesOnValueError:
"""The guard added to highlight_hits must catch exactly ValueError, the
same shape as the sibling notes_text guard, and let anything else
through -- so a real library defect is never mistaken for a harmless
syntax error."""
def test_non_value_error_is_not_swallowed(
self,
backend: TantivyBackend,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
GIVEN:
- parse_simple_text_highlight_query patched to raise
RuntimeError instead of a syntax-related ValueError
WHEN:
- backend.highlight_hits is called
THEN:
- The RuntimeError propagates unguarded; the highlight guard
must catch exactly ValueError, the same shape as the
sibling notes_text guard, never mistaking a real library
defect for a harmless syntax error
"""
import documents.search._backend as backend_mod
def raise_runtime_error(*args: object, **kwargs: object) -> object:
raise RuntimeError("synthetic bug, unrelated to query syntax")
monkeypatch.setattr(
backend_mod,
"parse_simple_text_highlight_query",
raise_runtime_error,
)
doc = DocumentFactory.create(title="probe", content="anything here")
backend.add_or_update(doc)
with pytest.raises(RuntimeError):
backend.highlight_hits(
"anything",
[doc.pk],
search_mode=SearchMode.TEXT,
)
@pytest.mark.usefixtures("_search_index")
class TestApiNoLongerReturns500:
"""Pins the actual regression: a matching TEXT-mode search whose query
string carries Tantivy syntax must return results, not a server error."""
@pytest.mark.parametrize("raw_query", _MALFORMED_QUERIES)
def test_malformed_text_query_returns_200(
self,
admin_client: APIClient,
raw_query: str,
) -> None:
"""
GIVEN:
- A matching document whose content contains the raw query
text, indexed via the real search index fixture
WHEN:
- A TEXT-mode search is issued through the real API with a
query string carrying Tantivy syntax
THEN:
- The response is 200 with the expected result count, not a
500 (the regression this file exists to pin)
"""
from documents.search import get_backend
doc = DocumentFactory.create(
title="probe",
content=f"needle content containing {raw_query} literally here",
)
get_backend().add_or_update(doc)
response = admin_client.get(f"/api/documents/?text={raw_query}")
assert response.status_code == status.HTTP_200_OK
assert response.data["count"] == 1
def test_plain_text_query_still_returns_200(
self,
admin_client: APIClient,
) -> None:
"""
GIVEN:
- A matching document indexed via the real search index
fixture
WHEN:
- An ordinary TEXT-mode search (no Tantivy syntax) is issued
THEN:
- The response is 200 with the expected result count; sanity
check that the guard does not mask a total failure of the
ordinary highlight path
"""
from documents.search import get_backend
doc = DocumentFactory.create(
title="probe",
content="needle content containing plain text literally here",
)
get_backend().add_or_update(doc)
response = admin_client.get("/api/documents/?text=plain text")
assert response.status_code == status.HTTP_200_OK
assert response.data["count"] == 1
src/documents/tests/search/test_json_field_prefixes.py
+206
View File
@@ -0,0 +1,206 @@
"""Bare notes:/custom_fields: prefix resolution.
"notes:foo"/"custom_fields:foo" were valid fielded searches before the
whoosh-compat migration. The registry only exposes them as JSON subpaths, so
each JSON FieldSpec declares a default subpath (SubpathSpec(default=True)):
notes: resolves to notes.note:, custom_fields: resolves to
custom_fields.value:. This replaced an earlier regex-based rewrite
(_rewrite_bare_json_field_prefixes) that ran on the raw query string before
parsing and was blind to quoting, so a phrase like
content:"payment notes: none" was silently corrupted into a notes-field
search and matched nothing. Resolving the default subpath inside the parser
instead means quoting is already understood by the time it happens.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import pytest
from django.contrib.auth.models import User
from documents.models import CustomField
from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import Note
if TYPE_CHECKING:
from documents.search._backend import TantivyBackend
pytestmark = [pytest.mark.search, pytest.mark.django_db]
def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
return set(backend.search_ids(query, user=None))
def _index(backend: TantivyBackend, **kwargs: object) -> Document:
doc = Document.objects.create(**kwargs)
backend.add_or_update(doc)
return doc
class TestBareJsonFieldPrefixes:
def test_bare_notes_prefix_searches_note_text(
self,
backend: TantivyBackend,
) -> None:
"""
GIVEN:
- A document with a note whose text contains a word, and a
decoy document whose content (not notes) contains the same
word
WHEN:
- A bare "notes:" prefix query is run (notes declares "note"
as its default subpath)
THEN:
- Only the document whose note matches is returned; the
decoy's content match does not resurface through a demoted
text search
"""
alice = User.objects.create_user(username="alice")
with_note = Document.objects.create(
title="Has note",
content="x",
checksum="bare-notes-with",
)
Note.objects.create(document=with_note, user=alice, note="crocodile")
backend.add_or_update(with_note)
# This document's CONTENT contains the words a demoted text search
# would match; it must NOT match once the prefix addresses notes.
_index(
backend,
title="Notes about things",
content="notes crocodile mention",
checksum="bare-notes-decoy",
)
assert _matched_ids(backend, "notes:crocodile") == {with_note.pk}
def test_bare_custom_fields_prefix_searches_values(
self,
backend: TantivyBackend,
) -> None:
"""
GIVEN:
- A document with a custom field instance whose value
contains a word, and a decoy document whose content (not a
custom field value) contains the same word
WHEN:
- A bare "custom_fields:" prefix query is run (custom_fields
declares "value" as its default subpath)
THEN:
- Only the document whose custom field value matches is
returned
"""
field = CustomField.objects.create(
name="Policy Number",
data_type=CustomField.FieldDataType.STRING,
)
with_value = Document.objects.create(
title="Has field",
content="x",
checksum="bare-cf-with",
)
CustomFieldInstance.objects.create(
document=with_value,
field=field,
value_text="crocodile",
)
backend.add_or_update(with_value)
_index(
backend,
title="Custom things",
content="custom fields crocodile",
checksum="bare-cf-decoy",
)
assert _matched_ids(backend, "custom_fields:crocodile") == {with_value.pk}
def test_subpath_spellings_are_untouched(
self,
backend: TantivyBackend,
) -> None:
"""
GIVEN:
- A document with a note carrying both an author and note text
WHEN:
- The explicit subpath spellings "notes.user:" and
"notes.note:" are queried
THEN:
- Both resolve to their intended subpath and match the
document; the default-subpath resolution for the bare
prefix does not interfere with explicit subpath addressing
"""
bob = User.objects.create_user(username="bob")
doc = Document.objects.create(
title="Bob note",
content="x",
checksum="bare-subpath",
)
Note.objects.create(document=doc, user=bob, note="remark")
backend.add_or_update(doc)
assert _matched_ids(backend, "notes.user:bob") == {doc.pk}
assert _matched_ids(backend, "notes.note:remark") == {doc.pk}
class TestQuotedPhraseContainingNotesColonIsNotCorrupted:
"""The regex rewrite this migration removes was blind to quoting: it
matched "notes:" anywhere in the raw query string, including inside an
already-quoted phrase on an unrelated field, silently turning
content:"payment notes: none" into a notes-field search that matched
nothing. Resolving the default subpath during parsing (which is
quote-aware) fixes this."""
def test_quoted_phrase_with_notes_colon_matches_by_content(
self,
backend: TantivyBackend,
) -> None:
"""
GIVEN:
- A document whose content literally contains the text
"payment notes: none" inside a quoted phrase
WHEN:
- A query quoting that exact phrase against the content
field is run
THEN:
- It matches by content, rather than the "notes:" substring
inside the quotes being corrupted into a notes-field search
that matches nothing (the bug the deleted regex rewrite
caused, since it was blind to quoting)
"""
target = _index(
backend,
title="Statement",
content="payment notes: none",
checksum="quoted-phrase-notes-colon",
)
assert _matched_ids(
backend,
'content:"payment notes: none"',
) == {target.pk}
def test_quoted_phrase_matches_the_same_document_unquoted(
self,
backend: TantivyBackend,
) -> None:
"""
GIVEN:
- A document whose content contains the same words as the
previous test's phrase, but without the colon
WHEN:
- A query quoting that phrase against the content field is
run
THEN:
- It matches by content, proving the earlier fix is about
quote-awareness specifically, not about the words
themselves being unsearchable
"""
target = _index(
backend,
title="Statement",
content="payment notes none",
checksum="quoted-phrase-no-colon",
)
assert _matched_ids(
backend,
'content:"payment notes none"',
) == {target.pk}
src/documents/tests/search/test_pattern_stemming.py
+220
View File
@@ -0,0 +1,220 @@
"""Wildcard patterns must match a stemmed index, end to end.
Query patterns are normalized but were not stemmed, while index terms are
stemmed, so the natural spelling of a prefix search matched nothing:
``invoice*`` found no document although ``invoic*`` did. v2's index was
UNSTEMMED (whoosh ``TEXT()`` defaults to ``StandardAnalyzer``), so this
regressed against both baselines.
These are end-to-end tests against a real indexed document and a real
query, proving the pattern normalizer's stem-alternates contract actually
reaches a stemmed index term. The pure unit tests against the normalizer
function itself live in ``test_pattern_normalizer.py``.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import pytest
from documents.models import Document
if TYPE_CHECKING:
from documents.search._backend import TantivyBackend
pytestmark = [pytest.mark.search, pytest.mark.django_db]
CONTENT = (
"invoice total due for electricity from both companies, "
"payments made to the university library, copies attached"
)
def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
return set(backend.search_ids(query, user=None))
@pytest.fixture
def indexed_doc(backend: TantivyBackend) -> Document:
doc = Document.objects.create(
title="Invoice 2020 productname",
content=CONTENT,
checksum="pattern-stemming-1",
archive_serial_number=900,
)
backend.add_or_update(doc)
return doc
class TestPrefixStemming:
@pytest.mark.parametrize(
"query",
[
"invoice*",
"electricity*",
"companies*",
"payments*",
"library*",
"title:Invoice*",
],
)
def test_full_word_prefix_matches_its_stem(
self,
backend: TantivyBackend,
indexed_doc: Document,
query: str,
) -> None:
"""
GIVEN:
- A document indexed with content containing "invoice",
"electricity", "companies", "payments", "library" and title
"Invoice 2020 productname"
WHEN:
- A prefix wildcard on the full, unstemmed word is queried
(e.g. "invoice*", "title:Invoice*")
THEN:
- The document matches, since the pattern normalizer offers
the word's stem as an alternative alongside the typed run,
reaching the stemmed index term
"""
assert _matched_ids(backend, query) == {indexed_doc.id}
@pytest.mark.parametrize("query", ["invoic*", "electr*", "payment*"])
def test_already_stemmed_prefix_still_matches(
self,
backend: TantivyBackend,
indexed_doc: Document,
query: str,
) -> None:
"""
GIVEN:
- The same indexed document
WHEN:
- A prefix wildcard is typed already in its stemmed spelling
(e.g. "invoic*")
THEN:
- The document still matches, since the typed-run alternative
is itself a prefix of the stored stemmed term
"""
assert _matched_ids(backend, query) == {indexed_doc.id}
@pytest.mark.parametrize("query", ["univers*", "librar*"])
def test_partial_prefix_reaches_the_stemmed_term(
self,
backend: TantivyBackend,
indexed_doc: Document,
query: str,
) -> None:
"""
GIVEN:
- The same indexed document
WHEN:
- A prefix shorter than a whole word is queried ("univers*",
"librar*")
THEN:
- It still matches, and neither case needs the two-alternative
path to do it: measured under "en", the stemmer leaves
"librar" alone, so it has one form, and that form is a
prefix of the "librari" the index holds for "library";
"univers" stems to the *shorter* "univ", and the run as
typed and its stem are both prefixes of the "univers" the
index holds for "university". The case where the two forms
genuinely diverge, and only one of them matches, is
test_stem_substitution_reaches_both_the_inflection_and_the_compound
"""
assert _matched_ids(backend, query) == {indexed_doc.id}
def test_full_word_reaches_the_stem_but_a_fragment_of_it_does_not(
self,
backend: TantivyBackend,
indexed_doc: Document,
) -> None:
"""
GIVEN:
- The same indexed document, storing "university" as "univers"
WHEN:
- "universities*" and "universit*" are each queried
THEN:
- "universities*" matches, since the stem of "universities" is
that same "univers"; "universit*" matches nothing, since
"universit" is a prefix of neither its own stem nor the
stored term. The alternatives widen recall without turning
a wildcard into a prefix search over the original text, and
usage.md names this exact pair so a reader told that
`universit*` fails is also told which spelling works
"""
assert _matched_ids(backend, "universities*") == {indexed_doc.id}
assert _matched_ids(backend, "universit*") == set()
def test_pattern_past_the_stem_boundary_is_documented_not_fixed(
self,
backend: TantivyBackend,
indexed_doc: Document,
) -> None:
"""
GIVEN:
- The same indexed document, with "productname" indexed as
"productnam"
WHEN:
- "produ*name" (a pattern straddling the stem boundary) is
queried
THEN:
- It matches nothing; produ*name cannot match a stemmed
index, and usage.md must not advertise it. Pinned so the
limitation is deliberate, not accidental
"""
assert _matched_ids(backend, "produ*name") == set()
def test_stem_substitution_reaches_both_the_inflection_and_the_compound(
self,
backend: TantivyBackend,
indexed_doc: Document,
) -> None:
"""
GIVEN:
- The indexed document (containing "copies") plus a second
document titled "Copyright notice" with content "copyright
notice for the work"
WHEN:
- "copy*" and "copyright*" are each queried
THEN:
- "copy*" matches both documents, and "copyright*" matches
only the compound one. English stemming substitutes as well
as truncates: "copy" and "copies" both index as "copi",
while "copyright" keeps its literal "y". Neither form is a
prefix of the other, so no single normalized string reaches
both; the run is therefore emitted as a disjunction of the
folded and stemmed forms, and "copy*" reaches the base
word, its inflections and the compound alike
"""
compound = Document.objects.create(
title="Copyright notice",
content="copyright notice for the work",
checksum="pattern-stemming-2",
archive_serial_number=901,
)
backend.add_or_update(compound)
assert _matched_ids(backend, "copy*") == {indexed_doc.id, compound.id}
assert _matched_ids(backend, "copyright*") == {compound.id}
class TestBracketClassStillFolds:
def test_class_body_matches_case_insensitively(
self,
backend: TantivyBackend,
indexed_doc: Document,
) -> None:
"""
GIVEN:
- The indexed document, titled "Invoice 2020 productname"
WHEN:
- A bracket-class pattern mixing case is queried
("title:[IP]nvoice*")
THEN:
- It matches: the class body is folded per character, which
the alternatives contract preserves only because a lone
character stems to itself
"""
assert _matched_ids(backend, "title:[IP]nvoice*") == {indexed_doc.id}
src/documents/tests/search/test_permission_field_isolation.py
+198
View File
@@ -0,0 +1,198 @@
"""Permission filtering must hold against the real indexed document shape.
Only three of the index's unsigned ``*_id`` columns are load-bearing:
``owner_id``, ``viewer_id`` and ``viewer_group_id``, all read by
build_permission_filter. The rest (correspondent/document_type/storage_path/tag
ids) were written on every document and read by nothing, and were dropped.
These tests index real Documents through the backend's own document builder and
assert result-level visibility per user, so a mistake about which columns are
load-bearing shows up as documents leaking across users rather than as a passing
unit test over a hand-built index.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import pytest
from django.contrib.auth.models import Group
from django.contrib.auth.models import User
from guardian.shortcuts import assign_perm
from documents.models import Correspondent
from documents.models import Document
from documents.models import DocumentType
from documents.models import StoragePath
from documents.models import Tag
if TYPE_CHECKING:
from documents.search._backend import TantivyBackend
pytestmark = [pytest.mark.search, pytest.mark.django_db]
@pytest.fixture
def owner() -> User:
return User.objects.create_user(username="owner")
@pytest.fixture
def stranger() -> User:
return User.objects.create_user(username="stranger")
@pytest.fixture
def viewer() -> User:
return User.objects.create_user(username="viewer")
@pytest.fixture
def group_member() -> User:
user = User.objects.create_user(username="group_member")
user.groups.add(Group.objects.create(name="accounting"))
return user
class TestPermissionFilteringOnIndexedDocuments:
def test_unowned_document_is_visible_to_everyone(
self,
backend: TantivyBackend,
stranger: User,
) -> None:
"""
GIVEN:
- A document with no owner, indexed via the backend's real
document builder
WHEN:
- A stranger (no relation to the document) searches
THEN:
- The document is visible to them
"""
doc = Document.objects.create(
title="Public Invoice",
content="invoice total due",
checksum="perm-unowned",
)
backend.add_or_update(doc)
assert backend.search_ids("invoice", user=stranger) == [doc.pk]
def test_owned_document_is_visible_only_to_its_owner(
self,
backend: TantivyBackend,
owner: User,
stranger: User,
) -> None:
"""
GIVEN:
- A document owned by one user, indexed via the backend's
real document builder
WHEN:
- The owner and an unrelated stranger each search
THEN:
- The owner sees the document; the stranger does not
"""
doc = Document.objects.create(
title="Private Invoice",
content="invoice total due",
checksum="perm-owned",
owner=owner,
)
backend.add_or_update(doc)
assert backend.search_ids("invoice", user=owner) == [doc.pk]
assert backend.search_ids("invoice", user=stranger) == []
def test_explicitly_shared_document_is_visible_to_the_viewer(
self,
backend: TantivyBackend,
owner: User,
viewer: User,
stranger: User,
) -> None:
"""
GIVEN:
- A document owned by one user and explicitly shared with a
second user via guardian's view_document permission,
indexed via the backend's real document builder
WHEN:
- The shared viewer and an unrelated stranger each search
THEN:
- The viewer sees the document; the stranger does not
"""
doc = Document.objects.create(
title="Shared Invoice",
content="invoice total due",
checksum="perm-shared-user",
owner=owner,
)
assign_perm("view_document", viewer, doc)
backend.add_or_update(doc)
assert backend.search_ids("invoice", user=viewer) == [doc.pk]
assert backend.search_ids("invoice", user=stranger) == []
def test_group_shared_document_is_visible_to_group_members(
self,
backend: TantivyBackend,
owner: User,
group_member: User,
stranger: User,
) -> None:
"""
GIVEN:
- A document owned by one user and shared with a group via
guardian's view_document permission, indexed via the
backend's real document builder
WHEN:
- A member of that group and an unrelated stranger each
search
THEN:
- The group member sees the document; the stranger does not
"""
doc = Document.objects.create(
title="Group Invoice",
content="invoice total due",
checksum="perm-shared-group",
owner=owner,
)
assign_perm("view_document", group_member.groups.first(), doc)
backend.add_or_update(doc)
assert backend.search_ids("invoice", user=group_member) == [doc.pk]
assert backend.search_ids("invoice", user=stranger) == []
def test_metadata_does_not_widen_visibility(
self,
backend: TantivyBackend,
owner: User,
stranger: User,
) -> None:
"""
GIVEN:
- A document owned by one user and carrying
correspondent/document_type/storage_path/tag metadata,
indexed via the backend's real document builder
WHEN:
- The owner and an unrelated stranger each search
THEN:
- The owner sees the document; the stranger does not, since
the dropped, non-load-bearing metadata *_id columns must
not widen visibility beyond the owner_id/viewer_id/
viewer_group_id filter
"""
doc = Document.objects.create(
title="Tagged Invoice",
content="invoice total due",
checksum="perm-metadata",
owner=owner,
correspondent=Correspondent.objects.create(name="ACME"),
document_type=DocumentType.objects.create(name="Bill"),
storage_path=StoragePath.objects.create(name="Archive", path="archive/"),
)
doc.tags.add(Tag.objects.create(name="paid"))
backend.add_or_update(doc)
assert backend.search_ids("invoice", user=owner) == [doc.pk]
assert backend.search_ids("invoice", user=stranger) == []
src/documents/tests/search/test_query.py
+315 -693
View File
File diff suppressed because it is too large Load Diff
src/documents/tests/search/test_query_negation.py
+198
View File
@@ -0,0 +1,198 @@
"""Negation must survive the blended query.
parse_user_query ORs an exact clause with optional fuzzy and CJK clauses.
Each of those is built from positive terms only, so unless the query's
exclusions are applied to the blend as a whole, a document the exact
clause excluded is re-admitted by whichever other clause is enabled.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import pytest
from documents.models import Document
if TYPE_CHECKING:
from pytest_django.fixtures import SettingsWrapper
from documents.search._backend import TantivyBackend
pytestmark = [pytest.mark.search, pytest.mark.django_db]
def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
return set(backend.search_ids(query, user=None))
def _index(backend: TantivyBackend, **kwargs: object) -> Document:
doc = Document.objects.create(**kwargs)
backend.add_or_update(doc)
return doc
@pytest.fixture
def fuzzy_enabled(settings: SettingsWrapper) -> None:
"""Enable the fuzzy blend clause. The threshold doubles as a minimum
score filter, so it is set to 0.0: every hit passes and the test sees
the clause's matching behaviour, not the filter's."""
settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.0
class TestNegationConstrainsEveryClause:
@pytest.mark.usefixtures("fuzzy_enabled")
def test_fuzzy_clause_does_not_readmit_an_excluded_document(
self,
backend: TantivyBackend,
) -> None:
"""
GIVEN:
- Two documents both matching a positive term, one of which
also contains a word the query excludes, with the fuzzy
blend clause enabled
WHEN:
- A query combining the positive term with a NOT exclusion is
run
THEN:
- Only the document without the excluded word is returned;
the fuzzy clause (built from positive terms only) does not
readmit the document the exact clause excluded
"""
secret = _index(
backend,
title="Invoice A",
content="invoice total secret",
checksum="neg-fuzzy-1",
)
public = _index(
backend,
title="Invoice B",
content="invoice total public",
checksum="neg-fuzzy-2",
)
assert _matched_ids(backend, "invoice") == {secret.pk, public.pk}
assert _matched_ids(backend, "invoice NOT secret") == {public.pk}
def test_cjk_clause_does_not_readmit_an_excluded_document(
self,
backend: TantivyBackend,
) -> None:
"""
GIVEN:
- Two documents both containing a CJK run, one of which also
contains a word the query excludes
WHEN:
- A query combining the CJK term with a NOT exclusion is run
THEN:
- Only the document without the excluded word is returned;
the CJK clause legitimately carries the CJK run, so
rebuilding it from the AST cannot help here, only applying
the exclusion above the blend keeps the excluded document
out
"""
secret = _index(
backend,
title="Tokyo A",
content="東京都の秘密です secret",
checksum="neg-cjk-1",
)
public = _index(
backend,
title="Tokyo B",
content="東京都の報告書です public",
checksum="neg-cjk-2",
)
assert _matched_ids(backend, "東京") == {secret.pk, public.pk}
assert _matched_ids(backend, "東京 NOT secret") == {public.pk}
@pytest.mark.usefixtures("fuzzy_enabled")
def test_disjunctive_negation_still_admits_the_other_branch(
self,
backend: TantivyBackend,
) -> None:
"""
GIVEN:
- A document matching a positive term and also containing a
word a disjunctive NOT branch excludes, plus an unrelated
document
WHEN:
- A query of the shape "term OR NOT excluded_word" is run
THEN:
- Both documents are returned; "invoice OR NOT secret"
excludes nothing on its own, so a document matching the
left branch stays in even though it contains the excluded
word
"""
secret_invoice = _index(
backend,
title="Invoice A",
content="invoice total secret",
checksum="neg-or-1",
)
unrelated = _index(
backend,
title="Recipe",
content="flour and water",
checksum="neg-or-2",
)
assert _matched_ids(backend, "invoice OR NOT secret") == {
secret_invoice.pk,
unrelated.pk,
}
def test_a_negation_under_or_does_not_constrain_the_cjk_clause(
self,
backend: TantivyBackend,
) -> None:
"""
GIVEN:
- Two CJK documents, one of which also contains a word an OR
branch's own NOT excludes, plus an unrelated latin document
WHEN:
- The exclusion is under a disjunctive OR branch, versus in
conjunctive position
THEN:
- Under OR, the excluded document still matches through the
CJK clause (an exclusion that is one branch's own condition
cannot be restated above the blend without dropping
documents the other branch matches, so it is left where it
is and the CJK clause stays unconstrained by it -- this
shows through here in a way it does not for latin text,
since the exact clause cannot match a CJK run at all, so
the CJK clause is the only thing matching the CJK
documents, and the excluded one comes with it)
- Under conjunctive "AND NOT", the same exclusion is hoisted
and does constrain the CJK clause, pinning the deliberate
limit of the hoist
"""
secret = _index(
backend,
title="Tokyo A",
content="東京都の秘密です secret",
checksum="neg-or-cjk-1",
)
public = _index(
backend,
title="Tokyo B",
content="東京都の報告書です public",
checksum="neg-or-cjk-2",
)
bill = _index(
backend,
title="Bill",
content="bill payment received",
checksum="neg-or-cjk-3",
)
assert _matched_ids(backend, "(東京 AND NOT secret) OR bill") == {
bill.pk,
public.pk,
secret.pk,
}
# The same exclusion in conjunctive position is hoisted, and does
# constrain the CJK clause.
assert _matched_ids(backend, "東京 AND NOT secret") == {public.pk}
src/documents/tests/search/test_translate.py
-810
View File
@@ -1,810 +0,0 @@
from __future__ import annotations
from datetime import UTC
from datetime import datetime
from typing import TYPE_CHECKING
from zoneinfo import ZoneInfo
import pytest
import time_machine
from documents.search._dates import _precision_bounds
if TYPE_CHECKING:
import tantivy
from documents.search._query import _FIELD_BOOSTS
from documents.search._query import DEFAULT_SEARCH_FIELDS
from documents.search._translate import OPEN_HI
from documents.search._translate import OPEN_LO
from documents.search._translate import Comma
from documents.search._translate import FieldRange
from documents.search._translate import FieldValue
from documents.search._translate import FieldValueList
from documents.search._translate import InvalidDateQuery
from documents.search._translate import Passthrough
from documents.search._translate import resolve_commas
from documents.search._translate import scan
from documents.search._translate import translate_query
from documents.search._translate import translate_range
from documents.search._translate import translate_scalar
@pytest.mark.search
class TestPrecisionBounds:
@pytest.mark.parametrize(
("digits", "expected"),
[
("2020", ((2020, 1, 1), (2021, 1, 1))),
("202003", ((2020, 3, 1), (2020, 4, 1))),
("202012", ((2020, 12, 1), (2021, 1, 1))),
("20200115", ((2020, 1, 15), (2020, 1, 16))),
("20201231", ((2020, 12, 31), (2021, 1, 1))),
],
)
def test_valid(self, digits, expected):
lo, hi = _precision_bounds(digits)
assert (lo.year, lo.month, lo.day) == expected[0]
assert (hi.year, hi.month, hi.day) == expected[1]
@pytest.mark.parametrize("digits", ["202023", "20200230", "20201301", "20", "abcd"])
def test_invalid_returns_none(self, digits):
assert _precision_bounds(digits) is None
@pytest.mark.search
class TestScan:
def test_plain_words_are_passthrough(self):
assert scan("bank statement") == [Passthrough("bank statement")]
def test_field_value(self):
assert scan("created:2020") == [FieldValue("created", "2020")]
def test_field_value_in_boolean(self):
toks = scan("created:2020 OR foo")
assert toks == [
FieldValue("created", "2020"),
Passthrough(" OR foo"),
]
def test_field_value_in_parens(self):
toks = scan("(created:2020 OR foo)")
assert toks == [
Passthrough("("),
FieldValue("created", "2020"),
Passthrough(" OR foo)"),
]
def test_quoted_value(self):
assert scan('correspondent:"A B"') == [FieldValue("correspondent", '"A B"')]
def test_field_range(self):
assert scan("created:[2020 TO 2021]") == [
FieldRange("created", "[", "2020", "2021", "]"),
]
@pytest.mark.parametrize(
("query", "expected"),
[
pytest.param(
"created:[2020 to]",
FieldRange("created", "[", "2020", "", "]"),
id="open_upper",
),
pytest.param(
"created:[to 2020]",
FieldRange("created", "[", "", "2020", "]"),
id="open_lower",
),
],
)
def test_open_range(self, query, expected):
assert scan(query) == [expected]
def test_comma_inside_range_not_split(self):
# No depth-0 comma here; the whole thing is one range token.
toks = scan("created:[2020 TO 2021]")
assert len(toks) == 1
# --- Edge-case / regression tests (scan must never raise) ---
def test_url_is_passthrough(self):
# "http" is not a known field; the whole URL must pass through verbatim.
assert scan("http://example.com") == [Passthrough("http://example.com")]
def test_unterminated_quote_is_passthrough(self):
# title is a known field but the quoted value has no closing quote;
# _consume_value returns None so the whole string falls into passthrough.
assert scan('title:"abc') == [Passthrough('title:"abc')]
def test_unterminated_bracket_is_passthrough(self):
# created is a known field but the range bracket is never closed;
# _consume_range returns None so the whole string falls into passthrough.
assert scan("created:[2020") == [Passthrough("created:[2020")]
def test_empty_value_at_end_is_passthrough(self):
# created is a known field but there is no value after the colon
# (_consume_value returns None for start >= n), so passthrough.
assert scan("created:") == [Passthrough("created:")]
def test_value_containing_colon(self):
# The bare-word value reader stops at whitespace/paren, not at colon,
# so "2020:30" is consumed as a single value token.
assert scan("created:2020:30") == [FieldValue("created", "2020:30")]
def test_comma_followed_by_unconsumable_value_stops(self):
# A comma followed by whitespace is neither a value-list continuation nor a
# clause separator: the value stops and the comma stays as passthrough.
assert scan("tag:foo, bar") == [
FieldValue("tag", "foo"),
Passthrough(", bar"),
]
def test_bracket_without_to_is_open_upper_bound(self):
# A bracketed value with no TO falls back to (value, "") -> open upper bound.
assert scan("created:[2020]") == [
FieldRange("created", "[", "2020", "", "]"),
]
def test_known_field_name_midword_is_passthrough(self):
# A known field name embedded mid-word is not a field token (the
# word-boundary guard); the whole run stays passthrough.
assert scan("xtag:foo") == [Passthrough("xtag:foo")]
@pytest.mark.search
class TestCommaResolution:
def test_value_list_multi_value_field(self):
toks = resolve_commas(scan("tag:foo,bar"))
assert toks == [FieldValueList("tag", ("foo", "bar"))]
def test_value_list_three(self):
toks = resolve_commas(scan("tag_id:1,2,3"))
assert toks == [FieldValueList("tag_id", ("1", "2", "3"))]
def test_text_field_comma_is_literal(self):
# correspondent is not multi-value: comma stays inside the value.
toks = resolve_commas(scan("correspondent:foo,bar"))
assert toks == [FieldValue("correspondent", "foo,bar")]
def test_clause_separator_before_known_field(self):
toks = resolve_commas(scan("tag:foo,type:bar"))
assert toks == [FieldValue("tag", "foo"), Comma(), FieldValue("type", "bar")]
def test_clause_separator_after_range(self):
toks = resolve_commas(scan("created:[2020 TO 2021],added:[2022 TO 2023]"))
assert toks == [
FieldRange("created", "[", "2020", "2021", "]"),
Comma(),
FieldRange("added", "[", "2022", "2023", "]"),
]
def test_clause_separator_after_quote(self):
toks = resolve_commas(scan('correspondent:"A B",created:[2020 TO 2021]'))
assert toks == [
FieldValue("correspondent", '"A B"'),
Comma(),
FieldRange("created", "[", "2020", "2021", "]"),
]
def test_url_comma_is_literal_passthrough(self):
toks = resolve_commas(scan("http://example.com/a,b"))
assert toks == [Passthrough("http://example.com/a,b")]
def test_non_multi_value_comma_is_literal(self):
# title is not in MULTI_VALUE_FIELDS: comma stays inside the value.
toks = resolve_commas(scan("title:10,20"))
assert toks == [FieldValue("title", "10,20")]
def test_clause_separator_before_known_date_field(self):
# The comma between a bare value and a known date field acts as a
# clause separator; both sides survive as distinct tokens.
toks = resolve_commas(scan("correspondent:foo,created:[2020 TO 2021]"))
assert toks == [
FieldValue("correspondent", "foo"),
Comma(),
FieldRange("created", "[", "2020", "2021", "]"),
]
@pytest.mark.search
class TestTranslateScalar:
@pytest.mark.parametrize(
("field", "value", "expected"),
[
(
"created",
"2020",
"created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z}",
),
(
"created",
"202003",
"created:[2020-03-01T00:00:00Z TO 2020-04-01T00:00:00Z}",
),
(
"created",
"20200115",
"created:[2020-01-15T00:00:00Z TO 2020-01-16T00:00:00Z}",
),
(
"created",
"2020-01-15",
"created:[2020-01-15T00:00:00Z TO 2020-01-16T00:00:00Z}",
),
(
"created",
"2020-03",
"created:[2020-03-01T00:00:00Z TO 2020-04-01T00:00:00Z}",
),
],
)
def test_partial_and_iso_dates(self, field: str, value: str, expected: str) -> None:
assert translate_scalar(field, value, UTC) == expected
def test_invalid_date_raises(self) -> None:
with pytest.raises(InvalidDateQuery) as exc_info:
translate_scalar("created", "202023", UTC)
assert exc_info.value.field == "created"
assert exc_info.value.value == "202023"
def test_keyword_delegates(self) -> None:
# keyword path produces a half-open range; just assert it is a created range
out = translate_scalar("created", "today", UTC)
assert out.startswith("created:[") and out.endswith("}")
def test_14digit_compact_datetime(self) -> None:
out = translate_scalar("created", "20240115120000", UTC)
assert "20240115120000" not in out
assert out.startswith("created:")
assert out == "created:[2024-01-15T12:00:00Z TO 2024-01-15T12:00:00Z]"
def test_14digit_invalid_month_raises(self) -> None:
with pytest.raises(InvalidDateQuery) as exc_info:
translate_scalar("created", "20231300120000", UTC)
assert exc_info.value.field == "created"
assert exc_info.value.value == "20231300120000"
def test_unrecognized_value_raises(self) -> None:
# A value that is not a keyword, digits, ISO date, or compact timestamp
# raises rather than producing invalid Tantivy syntax or silently matching
# nothing.
with pytest.raises(InvalidDateQuery) as exc_info:
translate_scalar("created", "garbage", UTC)
assert exc_info.value.field == "created"
assert exc_info.value.value == "garbage"
@pytest.mark.search
class TestTranslateRange:
@pytest.mark.parametrize(
("lo", "hi", "expected"),
[
("2005", "2009", "created:[2005-01-01T00:00:00Z TO 2010-01-01T00:00:00Z}"),
(
"202001",
"202006",
"created:[2020-01-01T00:00:00Z TO 2020-07-01T00:00:00Z}",
),
(
"20200101",
"20201231",
"created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z}",
),
(
"2020-01-01",
"2020-12-31",
"created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z}",
),
],
)
def test_absolute_ranges(self, lo, hi, expected):
assert translate_range("created", lo, hi, UTC) == expected
def test_reversed_swaps(self):
assert translate_range("created", "2009", "2005", UTC) == (
"created:[2005-01-01T00:00:00Z TO 2010-01-01T00:00:00Z}"
)
def test_open_upper(self):
out = translate_range("created", "2020", "", UTC)
assert out == f"created:[2020-01-01T00:00:00Z TO {OPEN_HI}]"
def test_open_lower(self):
out = translate_range("created", "", "2020", UTC)
assert out == f"created:[{OPEN_LO} TO 2021-01-01T00:00:00Z}}"
def test_invalid_bound_raises(self):
with pytest.raises(InvalidDateQuery) as exc_info:
translate_range("created", "202023", "2025", UTC)
assert exc_info.value.field == "created"
assert exc_info.value.value == "202023"
def test_invalid_high_bound_raises(self):
# Low bound parses, high bound does not -> raise on the high bound.
with pytest.raises(InvalidDateQuery) as exc_info:
translate_range("created", "2020", "garbage", UTC)
assert exc_info.value.field == "created"
assert exc_info.value.value == "garbage"
@pytest.mark.search
class TestTranslateQuery:
@pytest.mark.parametrize(
("raw", "expected"),
[
(
"created:2020",
"created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z}",
),
("tag:foo,bar", "tag:foo AND tag:bar"),
# 'type' is a user-facing alias rewritten to 'document_type' (the real schema field)
("tag:foo,type:bar", "tag:foo AND document_type:bar"),
(
"created:[2020 TO 2021],added:[2022 TO 2023]",
(
"created:[2020-01-01T00:00:00Z TO 2022-01-01T00:00:00Z}"
" AND "
"added:[2022-01-01T00:00:00Z TO 2024-01-01T00:00:00Z}"
),
),
# correspondent is not multi-value: comma stays literal inside the value
("correspondent:foo,bar", "correspondent:foo,bar"),
],
)
def test_golden(self, raw: str, expected: str) -> None:
assert translate_query(raw, UTC) == expected
@pytest.mark.parametrize(
"raw",
[
"created:2020",
"created:202003",
"created:[20200101 TO 20201231]",
"created:[2020-01-01 TO 2020-12-31]",
"created:[2020 to]",
"created:[to 2020]",
"title:x,created:[2020 TO 2021]",
"created:2020 OR foo",
"(created:2020 OR invoice)",
"tag:foo,type:bar",
"bank statement",
],
)
def test_parse_acceptance(self, index: tantivy.Index, raw: str) -> None:
translated = translate_query(raw, UTC)
# Must not raise:
index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
@pytest.mark.search
class TestFieldAliasing:
"""Whoosh->Tantivy field-name aliasing (type/path -> document_type/storage_path)."""
def test_type_alias(self) -> None:
assert translate_query("type:invoice", UTC) == "document_type:invoice"
def test_path_alias(self) -> None:
assert translate_query("path:/foo/bar", UTC) == "storage_path:/foo/bar"
def test_type_id_alias(self) -> None:
assert translate_query("type_id:5", UTC) == "document_type_id:5"
def test_path_id_alias(self) -> None:
assert translate_query("path_id:7", UTC) == "storage_path_id:7"
def test_clause_separator_plus_alias(self) -> None:
# Comma between known fields acts as AND separator; alias still applied.
assert (
translate_query("tag:foo,type:bar", UTC) == "tag:foo AND document_type:bar"
)
def test_type_range_alias(self) -> None:
# type is not a date field; range passes through verbatim with alias applied.
assert (
translate_query("type:[2020 TO 2021]", UTC)
== "document_type:[2020 TO 2021]"
)
def test_parse_acceptance_type(self, index: tantivy.Index) -> None:
# Translated output must be accepted by the real Tantivy parser.
translated = translate_query("type:invoice", UTC)
index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
def test_parse_acceptance_path(self, index: tantivy.Index) -> None:
translated = translate_query("path:foo", UTC)
index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
# Freeze time so relative-date tests are deterministic.
_FROZEN_NOW = datetime(2026, 3, 28, 12, 0, 0, tzinfo=UTC)
@pytest.mark.search
class TestRelativeRanges:
"""Relative date-range tokens resolved against a frozen clock."""
@time_machine.travel(_FROZEN_NOW, tick=False)
def test_minus_7_days_to_now(self) -> None:
assert translate_query("added:[-7 days to now]", UTC) == (
"added:[2026-03-21T12:00:00Z TO 2026-03-28T12:00:00Z]"
)
@time_machine.travel(_FROZEN_NOW, tick=False)
def test_minus_1_week_to_now(self) -> None:
assert translate_query("added:[-1 week to now]", UTC) == (
"added:[2026-03-21T12:00:00Z TO 2026-03-28T12:00:00Z]"
)
@time_machine.travel(_FROZEN_NOW, tick=False)
def test_minus_1_month_to_now(self) -> None:
assert translate_query("created:[-1 month to now]", UTC) == (
"created:[2026-02-28T12:00:00Z TO 2026-03-28T12:00:00Z]"
)
@time_machine.travel(_FROZEN_NOW, tick=False)
def test_minus_1_year_to_now(self) -> None:
assert translate_query("modified:[-1 year to now]", UTC) == (
"modified:[2025-03-28T12:00:00Z TO 2026-03-28T12:00:00Z]"
)
@time_machine.travel(_FROZEN_NOW, tick=False)
def test_minus_3_hours_to_now(self) -> None:
assert translate_query("added:[-3 hours to now]", UTC) == (
"added:[2026-03-28T09:00:00Z TO 2026-03-28T12:00:00Z]"
)
@time_machine.travel(_FROZEN_NOW, tick=False)
def test_uppercase_units(self) -> None:
assert translate_query("added:[-1 WEEK TO NOW]", UTC) == (
"added:[2026-03-21T12:00:00Z TO 2026-03-28T12:00:00Z]"
)
@time_machine.travel(_FROZEN_NOW, tick=False)
def test_now_minus_7d_compact(self) -> None:
assert translate_query("added:[now-7d TO now]", UTC) == (
"added:[2026-03-21T12:00:00Z TO 2026-03-28T12:00:00Z]"
)
@time_machine.travel(_FROZEN_NOW, tick=False)
def test_reversed_range_swapped(self) -> None:
# now+1h TO now-1h is reversed; translate_range swaps -> lo=now-1h, hi=now+1h
assert translate_query("added:[now+1h TO now-1h]", UTC) == (
"added:[2026-03-28T11:00:00Z TO 2026-03-28T13:00:00Z]"
)
@pytest.mark.parametrize(
"raw",
[
"added:[-7 days to now]",
"added:[-1 week to now]",
"created:[-1 month to now]",
"modified:[-1 year to now]",
"added:[-3 hours to now]",
"added:[now-7d TO now]",
"added:[now+1h TO now-1h]",
],
)
@time_machine.travel(_FROZEN_NOW, tick=False)
def test_parse_acceptance(self, index: tantivy.Index, raw: str) -> None:
translated = translate_query(raw, UTC)
index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
@pytest.mark.search
class TestWhooshUnitAbbreviations:
"""
Whoosh's PlusMinus date grammar accepted abbreviated unit spellings
(e.g. "yrs", "mos", "wks", "hrs", "mins", "secs"); saved views/searches
created under the old Whoosh backend can contain those tokens (see
https://github.com/paperless-ngx/paperless-ngx/issues/13482), so the
Tantivy translator must still accept them.
"""
@time_machine.travel(_FROZEN_NOW, tick=False)
def test_minus_999_yrs(self) -> None:
assert translate_query("created:[-999yrs to now]", UTC) == (
"created:[1027-03-28T12:00:00Z TO 2026-03-28T12:00:00Z]"
)
@pytest.mark.parametrize(
("token", "expected_lo"),
[
("-1y", "2025-03-28T12:00:00Z"),
("-1yr", "2025-03-28T12:00:00Z"),
("-3mos", "2025-12-28T12:00:00Z"),
("-3mo", "2025-12-28T12:00:00Z"),
("-2wks", "2026-03-14T12:00:00Z"),
("-2wk", "2026-03-14T12:00:00Z"),
("-5dys", "2026-03-23T12:00:00Z"),
("-5dy", "2026-03-23T12:00:00Z"),
("-1hrs", "2026-03-28T11:00:00Z"),
("-1hr", "2026-03-28T11:00:00Z"),
("-10mins", "2026-03-28T11:50:00Z"),
("-10min", "2026-03-28T11:50:00Z"),
("-30secs", "2026-03-28T11:59:30Z"),
("-30sec", "2026-03-28T11:59:30Z"),
],
)
@time_machine.travel(_FROZEN_NOW, tick=False)
def test_abbreviated_units(self, token: str, expected_lo: str) -> None:
assert translate_query(f"added:[{token} to now]", UTC) == (
f"added:[{expected_lo} TO 2026-03-28T12:00:00Z]"
)
@pytest.mark.parametrize(
"raw",
[
"created:[-999yrs to now]",
"added:[-1y to now]",
"created:[-3mos to now]",
"added:[-2wks to now]",
"added:[-5dys to now]",
"added:[-1hrs to now]",
"added:[-10mins to now]",
"added:[-30secs to now]",
],
)
@time_machine.travel(_FROZEN_NOW, tick=False)
def test_parse_acceptance(self, index: tantivy.Index, raw: str) -> None:
translated = translate_query(raw, UTC)
index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
@pytest.mark.search
class TestOperatorNormalization:
"""Post-render operator normalization in translate_query."""
def test_spaced_dash_removed(self) -> None:
assert (
translate_query("H52.1 - Kurzsichtigkeit", UTC) == "H52.1 Kurzsichtigkeit"
)
def test_spaced_dash_simple(self) -> None:
assert translate_query("bar - baz", UTC) == "bar baz"
def test_trailing_operator_stripped(self) -> None:
assert translate_query("foo -", UTC) == "foo"
def test_date_range_preserved(self) -> None:
out = translate_query("created:[2020 TO 2021]", UTC)
# Must not corrupt the ISO range
assert out == "created:[2020-01-01T00:00:00Z TO 2022-01-01T00:00:00Z}"
def test_date_scalar_with_or(self) -> None:
out = translate_query("created:2020 OR foo", UTC)
# The created scalar becomes a range; " OR foo" passes through verbatim.
assert out.startswith("created:[")
assert "OR foo" in out
def test_parse_acceptance_spaced_dash(self, index: tantivy.Index) -> None:
translated = translate_query("H52.1 - Kurzsichtigkeit", UTC)
index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
def test_parse_acceptance_trailing_op(self, index: tantivy.Index) -> None:
translated = translate_query("foo -", UTC)
index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
@pytest.mark.search
class TestMultiWordDateKeywords:
"""scan() must consume multi-word date keywords as a single value."""
def test_scan_previous_week_as_single_token(self) -> None:
# "created:previous week" must produce one FieldValue with value "previous week",
# not FieldValue("created","previous") + Passthrough(" week").
toks = scan("created:previous week")
assert toks == [FieldValue("created", "previous week")]
def test_scan_this_month_as_single_token(self) -> None:
toks = scan("added:this month")
assert toks == [FieldValue("added", "this month")]
def test_scan_previous_month_as_single_token(self) -> None:
toks = scan("created:previous month")
assert toks == [FieldValue("created", "previous month")]
def test_scan_this_year_as_single_token(self) -> None:
toks = scan("added:this year")
assert toks == [FieldValue("added", "this year")]
def test_scan_previous_year_as_single_token(self) -> None:
toks = scan("created:previous year")
assert toks == [FieldValue("created", "previous year")]
def test_scan_previous_quarter_as_single_token(self) -> None:
toks = scan("created:previous quarter")
assert toks == [FieldValue("created", "previous quarter")]
def test_quoted_multi_word_keyword_still_works(self) -> None:
# The quoted form must continue to work as before.
toks = scan('created:"previous week"')
assert toks == [FieldValue("created", '"previous week"')]
def test_non_date_field_not_affected(self) -> None:
# "previous" stops at the space for non-date fields; " week" passes through.
toks = scan("correspondent:previous week")
assert toks == [
FieldValue("correspondent", "previous"),
Passthrough(" week"),
]
@pytest.mark.search
class TestKeywordDateResolution:
"""Relative date keywords resolve to exact ISO ranges against a frozen clock.
Frozen at 2026-03-28 12:00 UTC (a Saturday in Q1) so the week, month,
quarter and year rollovers are all exercised by a single anchor.
"""
# created is a DateField: bounds are UTC midnight, no timezone offset.
@pytest.mark.parametrize(
("keyword", "expected"),
[
pytest.param(
"today",
"created:[2026-03-28T00:00:00Z TO 2026-03-29T00:00:00Z}",
id="today",
),
pytest.param(
"yesterday",
"created:[2026-03-27T00:00:00Z TO 2026-03-28T00:00:00Z}",
id="yesterday",
),
pytest.param(
"previous week",
"created:[2026-03-16T00:00:00Z TO 2026-03-23T00:00:00Z}",
id="previous-week",
),
pytest.param(
"this month",
"created:[2026-03-01T00:00:00Z TO 2026-04-01T00:00:00Z}",
id="this-month",
),
pytest.param(
"previous month",
"created:[2026-02-01T00:00:00Z TO 2026-03-01T00:00:00Z}",
id="previous-month",
),
pytest.param(
"this year",
"created:[2026-01-01T00:00:00Z TO 2027-01-01T00:00:00Z}",
id="this-year",
),
pytest.param(
"previous year",
"created:[2025-01-01T00:00:00Z TO 2026-01-01T00:00:00Z}",
id="previous-year",
),
pytest.param(
"previous quarter",
"created:[2025-10-01T00:00:00Z TO 2026-01-01T00:00:00Z}",
id="previous-quarter",
),
],
)
@time_machine.travel(_FROZEN_NOW, tick=False)
def test_date_only_field_keyword_ranges(
self,
keyword: str,
expected: str,
) -> None:
assert translate_query(f"created:{keyword}", UTC) == expected
# added is a DateTimeField: local-tz midnight converted to UTC. Tokyo
# (+09:00, no DST) shifts each midnight boundary back to 15:00Z the day
# before, so this also exercises the local-midnight offset path.
@pytest.mark.parametrize(
("keyword", "expected"),
[
pytest.param(
"today",
"added:[2026-03-27T15:00:00Z TO 2026-03-28T15:00:00Z}",
id="today",
),
pytest.param(
"yesterday",
"added:[2026-03-26T15:00:00Z TO 2026-03-27T15:00:00Z}",
id="yesterday",
),
pytest.param(
"previous week",
"added:[2026-03-15T15:00:00Z TO 2026-03-22T15:00:00Z}",
id="previous-week",
),
pytest.param(
"this month",
"added:[2026-02-28T15:00:00Z TO 2026-03-31T15:00:00Z}",
id="this-month",
),
pytest.param(
"previous month",
"added:[2026-01-31T15:00:00Z TO 2026-02-28T15:00:00Z}",
id="previous-month",
),
pytest.param(
"this year",
"added:[2025-12-31T15:00:00Z TO 2026-12-31T15:00:00Z}",
id="this-year",
),
pytest.param(
"previous year",
"added:[2024-12-31T15:00:00Z TO 2025-12-31T15:00:00Z}",
id="previous-year",
),
pytest.param(
"previous quarter",
"added:[2025-09-30T15:00:00Z TO 2025-12-31T15:00:00Z}",
id="previous-quarter",
),
],
)
@time_machine.travel(_FROZEN_NOW, tick=False)
def test_datetime_field_keyword_ranges_local_tz(
self,
keyword: str,
expected: str,
) -> None:
assert translate_query(f"added:{keyword}", ZoneInfo("Asia/Tokyo")) == expected
@pytest.mark.search
class TestISODatetimeBounds:
"""Full ISO datetime tokens in range bounds must be parsed directly."""
def test_translate_range_iso_bounds_passthrough(self) -> None:
# Already-ISO datetime bounds must pass through as-is (exact instant).
result = translate_range(
"created",
"2020-01-01T00:00:00Z",
"2021-01-01T00:00:00Z",
UTC,
)
assert result == "created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z]"
def test_translate_query_iso_range_preserved(self) -> None:
q = "created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z]"
assert translate_query(q, UTC) == q
def test_translate_query_comma_separated_iso_ranges(self) -> None:
q = (
"created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z],"
"added:[2026-05-01T00:00:00Z TO 2026-06-01T00:00:00Z]"
)
result = translate_query(q, UTC)
assert result == (
"created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z]"
" AND "
"added:[2026-05-01T00:00:00Z TO 2026-06-01T00:00:00Z]"
)
def test_translate_query_text_before_comma_separated_date_clause(self) -> None:
result = translate_query("schäfersee,created:previous year", UTC)
assert result == (
"schäfersee AND created:[2025-01-01T00:00:00Z TO 2026-01-01T00:00:00Z}"
)
def test_invalid_iso_datetime_raises(self) -> None:
# A token with "T" that is not valid ISO datetime -> raise.
with pytest.raises(InvalidDateQuery) as exc_info:
translate_range(
"created",
"2020-01-01T99:00:00Z",
"2021-01-01T00:00:00Z",
UTC,
)
assert exc_info.value.field == "created"
assert exc_info.value.value == "2020-01-01T99:00:00Z"
def test_parse_acceptance_iso_bounds(self, index: tantivy.Index) -> None:
q = "created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z]"
translated = translate_query(q, UTC)
index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
def test_parse_acceptance_comma_iso_ranges(self, index: tantivy.Index) -> None:
q = (
"created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z],"
"added:[2026-05-01T00:00:00Z TO 2026-06-01T00:00:00Z]"
)
translated = translate_query(q, UTC)
index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
src/documents/tests/test_api_bulk_download.py
+26
View File
@@ -339,3 +339,29 @@ class TestBulkDownload(DirectoriesMixin, SampleDirMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
self.assertEqual(response.content, b"Insufficient permissions")
def test_bad_search_query_returns_400(self) -> None:
"""
GIVEN:
- Bulk download request selects documents via a saved-search
query filter
WHEN:
- The query contains a malformed field value (an invalid date)
THEN:
- The response is a 400 naming the bad value, exactly like the
search list endpoint, never a 500
"""
response = self.client.post(
self.ENDPOINT,
json.dumps(
{
"all": True,
"filters": {"query": "added:notadate"},
"content": "originals",
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn(b"notadate", response.content)
src/documents/tests/test_api_bulk_edit.py
+27
View File
@@ -2059,3 +2059,30 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(LogEntry.objects.filter(object_pk=self.doc1.id).count(), 2)
def test_api_bulk_edit_with_bad_search_query_returns_400(self) -> None:
"""
GIVEN:
- Bulk edit request selects documents via a saved-search query
filter
WHEN:
- The query contains a malformed field value (an invalid date)
THEN:
- The response is a 400 naming the bad value, exactly like the
search list endpoint, never a 500
"""
response = self.client.post(
"/api/documents/bulk_edit/",
json.dumps(
{
"all": True,
"filters": {"query": "added:notadate"},
"method": "set_storage_path",
"parameters": {"storage_path": self.sp1.id},
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn(b"notadate", response.content)
src/documents/tests/test_api_search.py
+121
View File
@@ -786,6 +786,10 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
tick=False,
):
response = self.client.get("/api/documents/?query=added:previous month")
assert response.status_code == 200, (
f"expected a successful search response, got {response.status_code}: "
f"{response.data!r}"
)
results = response.data["results"]
self.assertEqual(len(results), 1)
@@ -818,6 +822,26 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn("invalid-date", str(response.data["query"]))
def test_search_multiple_bad_fields_returns_all_messages(self) -> None:
"""
GIVEN:
- One document added
WHEN:
- Query with multiple bad fields (e.g. invalid date and invalid number)
THEN:
- 400 Bad Request with error messages for every bad field,
so the user can fix them all in one round-trip
"""
response = self.client.get(
"/api/documents/",
{"query": "created:notadate AND asn:notanumber"},
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
messages = response.data["query"]
self.assertEqual(len(messages), 2)
self.assertTrue(any("created" in m for m in messages))
self.assertTrue(any("asn" in m for m in messages))
@override_settings(
TIME_ZONE="UTC",
)
@@ -861,6 +885,29 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
results = response.data["results"]
self.assertEqual({r["id"] for r in results}, {1, 2})
@mock.patch("documents.search._backend.parse_user_query")
def test_search_parser_bug_surfaces_as_500_not_400(self, m) -> None:
"""
GIVEN:
- The query parser itself fails (a whoosh-compat bug, per
QueryParserError's own contract: not user-fixable input)
WHEN:
- Any search request runs
THEN:
- The error surfaces as a 500 monitoring can see, never a 400
blaming the user for a library defect
"""
from whoosh_compat.errors import QueryParserError
m.side_effect = QueryParserError("synthetic parser bug")
self.client.raise_request_exception = False
response = self.client.get("/api/documents/?query=anything")
self.assertEqual(
response.status_code,
status.HTTP_500_INTERNAL_SERVER_ERROR,
)
@mock.patch("documents.search._backend.TantivyBackend.autocomplete")
def test_search_autocomplete_limits(self, m) -> None:
"""
@@ -2058,3 +2105,77 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
response = self.client.get("/api/search/?query=no")
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
def _assert_query_finds(self, doc: Document, query: str) -> None:
get_backend().add_or_update(doc)
response = self.client.get("/api/documents/", {"query": query})
self.assertEqual(response.status_code, status.HTTP_200_OK)
ids = [r["id"] for r in response.data["results"]]
self.assertIn(doc.id, ids)
def test_search_by_asn(self) -> None:
"""
GIVEN:
- A document with an archive serial number, indexed
WHEN:
- A query filters by "asn:<value>"
THEN:
- The document is found
"""
doc = Document.objects.create(
title="Has ASN",
content="content",
checksum="asn-checksum",
archive_serial_number=555,
)
self._assert_query_finds(doc, "asn:555")
def test_search_by_page_count(self) -> None:
"""
GIVEN:
- A document with a page count, indexed
WHEN:
- A query filters by "page_count:<value>"
THEN:
- The document is found
"""
doc = Document.objects.create(
title="Multi-page",
content="content",
checksum="page-count-checksum",
page_count=42,
)
self._assert_query_finds(doc, "page_count:42")
def test_search_by_original_filename(self) -> None:
"""
GIVEN:
- A document with an original filename, indexed
WHEN:
- A query filters by "original_filename:<value>"
THEN:
- The document is found
"""
doc = Document.objects.create(
title="Named file",
content="content",
checksum="filename-checksum",
original_filename="quarterly-report.pdf",
)
self._assert_query_finds(doc, "original_filename:quarterly-report.pdf")
def test_search_by_checksum(self) -> None:
"""
GIVEN:
- A document with a checksum, indexed
WHEN:
- A query filters by "checksum:<value>"
THEN:
- The document is found
"""
doc = Document.objects.create(
title="Checksum doc",
content="content",
checksum="deadbeef1234",
)
self._assert_query_finds(doc, "checksum:deadbeef1234")
src/documents/tests/test_api_search_errors.py
+344
View File
@@ -0,0 +1,344 @@
"""The search list endpoint's exception handling: what becomes a 400 and
what a library defect surfaces as instead.
Companion to documents/tests/search/test_error_routing.py, which pins the
Cause -> SearchQueryError/QueryError routing inside documents/search/_query.py.
These tests pin the layer above it: DocumentViewSet.list's own except clauses,
which decide what an already-routed error becomes on the wire.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import pytest
from rest_framework import status
from whoosh_compat.errors import Cause
from whoosh_compat.errors import Diagnostic
from whoosh_compat.errors import DiagnosticKind
from whoosh_compat.errors import QueryError
from documents.search import SearchQueryError
from documents.tests.factories import DocumentFactory
if TYPE_CHECKING:
from rest_framework.test import APIClient
from documents.models import Document
pytestmark = [pytest.mark.django_db, pytest.mark.usefixtures("_search_index")]
@pytest.fixture
def indexed_document() -> Document:
from documents.search import get_backend
doc = DocumentFactory.create(title="quarterly invoice", content="acme corp")
get_backend().add_or_update(doc)
return doc
class TestSearchQueryErrorStillBecomesA400:
def test_search_query_error_becomes_a_400_naming_the_field(
self,
admin_client: APIClient,
monkeypatch: pytest.MonkeyPatch,
indexed_document: Document,
) -> None:
"""
GIVEN:
- parse_user_query() raising a SearchQueryError naming a field
WHEN:
- The document list endpoint is queried
THEN:
- The response is a 400 whose body names the field
"""
import documents.search._backend as backend_mod
def raise_search_query_error(*args: object, **kwargs: object) -> object:
raise SearchQueryError("bad value for field 'added'")
monkeypatch.setattr(
backend_mod,
"parse_user_query",
raise_search_query_error,
)
response = admin_client.get("/api/documents/?query=anything")
assert response.status_code == status.HTTP_400_BAD_REQUEST
assert "added" in str(response.data["query"])
class TestLibraryDefectsPropagate:
"""The exact regression this task exists to fix: an unexpected or
INTERNAL-cause library error must not be relabeled a 400."""
def test_unexpected_exception_is_not_converted_to_a_400(
self,
admin_client: APIClient,
monkeypatch: pytest.MonkeyPatch,
indexed_document: Document,
) -> None:
"""
GIVEN:
- parse_user_query() raising an unrelated exception
(ZeroDivisionError), not a SearchQueryError
WHEN:
- The document list endpoint is queried
THEN:
- The exception propagates unconverted, rather than being
relabeled a 400
"""
import documents.search._backend as backend_mod
def raise_zero_division(*args: object, **kwargs: object) -> object:
raise ZeroDivisionError("synthetic bug, unrelated to search grammar")
monkeypatch.setattr(
backend_mod,
"parse_user_query",
raise_zero_division,
)
with pytest.raises(ZeroDivisionError):
admin_client.get("/api/documents/?query=anything")
def test_internal_cause_query_error_is_not_converted_to_a_400(
self,
admin_client: APIClient,
monkeypatch: pytest.MonkeyPatch,
indexed_document: Document,
) -> None:
"""
GIVEN:
- A real query string running through the real parse and
routing pipeline (pre-parse rewrites, wc.parse(), and
_map_emit_error's own Cause routing all run for real), except
the final emit call (tantivy_emit) is forced to report a
library-internal defect (Cause.INTERNAL) - the one
library-internal failure mode reachable from a real query
WHEN:
- The document list endpoint is queried
THEN:
- The QueryError propagates unconverted, rather than being
relabeled a 400
"""
import documents.search._query as query_mod
def raise_internal(*args: object, **kwargs: object) -> object:
raise QueryError(
Diagnostic(
kind=DiagnosticKind.BACKEND_REJECTED,
cause=Cause.INTERNAL,
message="synthetic whoosh-compat emitter defect",
),
)
monkeypatch.setattr(query_mod, "tantivy_emit", raise_internal)
with pytest.raises(QueryError):
admin_client.get("/api/documents/?query=invoice")
class TestSelectionPathsAgreeWithSearch:
"""DocumentSelectionMixin backs bulk edit, bulk download, and a
more_like_id selection filter. It catches only SearchQueryError -- the
same contract the search list endpoint enforces above -- so all three
must map SearchQueryError to a 400 and let anything else surface."""
def test_bulk_edit_maps_search_query_error_to_a_400(
self,
admin_client: APIClient,
monkeypatch: pytest.MonkeyPatch,
indexed_document: Document,
) -> None:
"""
GIVEN:
- parse_user_query() raising a SearchQueryError naming a field
WHEN:
- The bulk_edit endpoint is called with a query filter
THEN:
- The response is a 400 whose body names the field
"""
import documents.search._backend as backend_mod
def raise_search_query_error(*args: object, **kwargs: object) -> object:
raise SearchQueryError("bad value for field 'added'")
monkeypatch.setattr(
backend_mod,
"parse_user_query",
raise_search_query_error,
)
response = admin_client.post(
"/api/documents/bulk_edit/",
{
"documents": [],
"all": True,
"filters": {"query": "anything"},
"method": "set_document_type",
"parameters": {"document_type": None},
},
format="json",
)
assert response.status_code == status.HTTP_400_BAD_REQUEST
assert "added" in str(response.data["query"])
def test_bulk_edit_lets_an_unexpected_exception_surface(
self,
admin_client: APIClient,
monkeypatch: pytest.MonkeyPatch,
indexed_document: Document,
) -> None:
"""
GIVEN:
- parse_user_query() raising an unrelated exception
(ZeroDivisionError), not a SearchQueryError
WHEN:
- The bulk_edit endpoint is called with a query filter
THEN:
- The exception propagates unconverted, rather than being
relabeled a 400
"""
import documents.search._backend as backend_mod
def raise_zero_division(*args: object, **kwargs: object) -> object:
raise ZeroDivisionError("synthetic bug, unrelated to search grammar")
monkeypatch.setattr(
backend_mod,
"parse_user_query",
raise_zero_division,
)
with pytest.raises(ZeroDivisionError):
admin_client.post(
"/api/documents/bulk_edit/",
{
"documents": [],
"all": True,
"filters": {"query": "anything"},
"method": "set_document_type",
"parameters": {"document_type": None},
},
format="json",
)
def test_bulk_download_maps_search_query_error_to_a_400(
self,
admin_client: APIClient,
monkeypatch: pytest.MonkeyPatch,
indexed_document: Document,
) -> None:
"""
GIVEN:
- parse_user_query() raising a SearchQueryError naming a field
WHEN:
- The bulk_download endpoint is called with a query filter
THEN:
- The response is a 400 whose body names the field
"""
import documents.search._backend as backend_mod
def raise_search_query_error(*args: object, **kwargs: object) -> object:
raise SearchQueryError("bad value for field 'added'")
monkeypatch.setattr(
backend_mod,
"parse_user_query",
raise_search_query_error,
)
response = admin_client.post(
"/api/documents/bulk_download/",
{
"documents": [],
"all": True,
"filters": {"query": "anything"},
},
format="json",
)
assert response.status_code == status.HTTP_400_BAD_REQUEST
assert "added" in str(response.data["query"])
def test_more_like_id_selection_filter_maps_search_query_error_to_a_400(
self,
admin_client: APIClient,
monkeypatch: pytest.MonkeyPatch,
indexed_document: Document,
) -> None:
"""
GIVEN:
- TantivyBackend.more_like_this_ids() raising a
SearchQueryError
WHEN:
- The bulk_download endpoint is called with a more_like_id
filter
THEN:
- The response is a 400
"""
import documents.search._backend as backend_mod
def raise_search_query_error(*args: object, **kwargs: object) -> object:
raise SearchQueryError("similar-document lookup is unavailable")
monkeypatch.setattr(
backend_mod.TantivyBackend,
"more_like_this_ids",
raise_search_query_error,
)
response = admin_client.post(
"/api/documents/bulk_download/",
{
"documents": [],
"all": True,
"filters": {"more_like_id": indexed_document.pk},
},
format="json",
)
assert response.status_code == status.HTTP_400_BAD_REQUEST
def test_more_like_id_selection_filter_lets_an_unexpected_exception_surface(
self,
admin_client: APIClient,
monkeypatch: pytest.MonkeyPatch,
indexed_document: Document,
) -> None:
"""
GIVEN:
- TantivyBackend.more_like_this_ids() raising an unrelated
exception (ZeroDivisionError), not a SearchQueryError
WHEN:
- The bulk_download endpoint is called with a more_like_id
filter
THEN:
- The exception propagates unconverted, rather than being
relabeled a 400
"""
import documents.search._backend as backend_mod
def raise_zero_division(*args: object, **kwargs: object) -> object:
raise ZeroDivisionError("synthetic bug, unrelated to similarity lookup")
monkeypatch.setattr(
backend_mod.TantivyBackend,
"more_like_this_ids",
raise_zero_division,
)
with pytest.raises(ZeroDivisionError):
admin_client.post(
"/api/documents/bulk_download/",
{
"documents": [],
"all": True,
"filters": {"more_like_id": indexed_document.pk},
},
format="json",
)
src/documents/tests/test_api_search_unterminated_date_range.py
+88
View File
@@ -0,0 +1,88 @@
"""An unterminated ``[`` date range bracket at the API level.
``created:[2020`` (with or without a dangling ``to <value>``) now raises
BAD_DATE and the search endpoint returns HTTP 400, where it used to parse
past the missing ``]`` and silently pass the malformed range through.
A 400 is correct: malformed input should fail loudly rather than silently
matching an unintended query. Pinned at the API level -- the layer a user
or client actually sees -- rather than only against the parser directly.
The properly closed decoy proves the bracket is what matters, not
whoosh-compat's date grammar generally: ``created:[2020 to 2021]`` parses
and searches cleanly.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import pytest
from rest_framework import status
from documents.tests.factories import DocumentFactory
if TYPE_CHECKING:
from rest_framework.test import APIClient
from documents.models import Document
pytestmark = [pytest.mark.django_db, pytest.mark.usefixtures("_search_index")]
@pytest.fixture
def indexed_document() -> Document:
from documents.search import get_backend
doc = DocumentFactory.create(title="quarterly invoice", content="acme corp")
get_backend().add_or_update(doc)
return doc
class TestUnterminatedBracketReturnsA400:
@pytest.mark.parametrize(
"query",
[
pytest.param("created:[2020", id="missing_upper_bound_and_bracket"),
pytest.param("created:[2020 to 2021", id="missing_closing_bracket"),
],
)
def test_unterminated_bracket_is_a_400(
self,
admin_client: APIClient,
indexed_document: Document,
query: str,
) -> None:
"""
GIVEN:
- The search endpoint
WHEN:
- A date-range query with a missing closing `]` (with or
without a dangling upper bound) is submitted
THEN:
- The response is a 400 naming the field, rather than parsing
past the missing bracket and silently passing the malformed
range through
"""
response = admin_client.get(f"/api/documents/?query={query}")
assert response.status_code == status.HTTP_400_BAD_REQUEST
assert "created" in str(response.data["query"])
def test_properly_closed_bracket_still_searches_cleanly(
self,
admin_client: APIClient,
indexed_document: Document,
) -> None:
"""
GIVEN:
- The search endpoint
WHEN:
- A properly closed date-range query is submitted
THEN:
- The response is a 200 (the decoy proving the missing
bracket, not whoosh-compat's date grammar generally, is
what the 400 above is about)
"""
response = admin_client.get(
"/api/documents/?query=created:[2020 to 2021]",
)
assert response.status_code == status.HTTP_200_OK
src/documents/views.py
+60 -25
View File
@@ -16,6 +16,7 @@ from time import mktime
from time import sleep
from typing import TYPE_CHECKING
from typing import Any
from typing import Final
from typing import Literal
from typing import NamedTuple
from unicodedata import normalize
@@ -286,17 +287,40 @@ logger = logging.getLogger("paperless.api")
_TANTIVY_INTERSECT_THRESHOLD = 5_000
_TANTIVY_SEARCH_PARAM_NAMES = ("text", "title_search", "query", "more_like_id")
# whoosh-compat's fieldname tagger (used only for SearchMode.QUERY, via the
# whoosh grammar in parse_user_query) is O(n^2) in plain word characters:
# measured at ~0.96s/10k chars, ~3.67s/20k, ~14.4s/40k against the real field
# registry. Django's DATA_UPLOAD_MAX_MEMORY_SIZE default (2.5 MB) does not
# bound this on the POST-body selection-filter path, so an unbounded query
# is a single-request CPU exhaustion vector. 4096 chars caps the worst case
# at roughly 0.16s (quadratic extrapolation from the measurements above),
# far beyond any plausible hand-typed advanced query, while still being fast
# enough to absorb inside a request handler. Applied to all three modes at
# this shared choke point: TEXT and TITLE route through simple_search_tokens
# instead and measure linear even at 20k chars, so the cap is hygiene for
# them, not a fix, but a single limit here is simpler than one exemption.
# Not exposed as a PAPERLESS_* setting: this is a hard security boundary,
# not a tunable, and a raisable ceiling would let a misconfiguration
# reintroduce the exact hazard this exists to close.
_MAX_QUERY_LENGTH: Final[int] = 4096
def _get_tantivy_query_and_mode(params):
from documents.search import QueryTooLongError
from documents.search import SearchMode
if "text" in params:
return str(params["text"]), SearchMode.TEXT
if "title_search" in params:
return str(params["title_search"]), SearchMode.TITLE
if "query" in params:
return str(params["query"]), SearchMode.QUERY
return None # pragma: no cover
raw, mode = str(params["text"]), SearchMode.TEXT
elif "title_search" in params:
raw, mode = str(params["title_search"]), SearchMode.TITLE
elif "query" in params:
raw, mode = str(params["query"]), SearchMode.QUERY
else:
return None # pragma: no cover
if len(raw) > _MAX_QUERY_LENGTH:
raise QueryTooLongError(len(raw), _MAX_QUERY_LENGTH)
return raw, mode
def _get_more_like_id(query_params: dict[str, Any], user: User | None) -> int:
@@ -2453,6 +2477,7 @@ class UnifiedSearchViewSet(DocumentViewSet):
from documents.search import TantivyBackend
from documents.search import TantivyRelevanceList
from documents.search import get_backend
from documents.search import search_query_error_messages
def parse_search_params() -> SearchParams:
"""Extract query string, search mode, and ordering from request."""
@@ -2643,15 +2668,10 @@ class UnifiedSearchViewSet(DocumentViewSet):
except ValidationError:
raise
except SearchQueryError as e:
# User-fixable query error (e.g. an unparsable date): surface the
# specific message so the user can correct it, rather than a generic
# 400 or silently empty results.
raise ValidationError({"query": [str(e)]}) from e
except Exception as e:
logger.warning(f"An error occurred listing search results: {e!s}")
return HttpResponseBadRequest(
"Error listing search results, check logs for more detail.",
)
# User-fixable query error(s) (e.g. unparsable dates/numbers):
# surface every offending field's message, not just the first,
# so the user can fix them all in one round-trip.
raise ValidationError({"query": search_query_error_messages(e)}) from e
@action(detail=False, methods=["GET"], name="Get Next ASN")
def next_asn(self, request, *args, **kwargs):
@@ -2787,23 +2807,34 @@ class DocumentSelectionMixin:
},
)
from documents.search import SearchQueryError
from documents.search import get_backend
from documents.search import search_query_error_messages
filter_name = search_filters[0]
backend = get_backend()
search_user = None if user.is_superuser else user
if filter_name == "more_like_id":
more_like_doc_id = _get_more_like_id(filters, user)
try:
if filter_name == "more_like_id":
more_like_doc_id = _get_more_like_id(filters, user)
search_ids = backend.more_like_this_ids(more_like_doc_id, user=search_user)
else:
query_str, search_mode = _get_tantivy_query_and_mode(filters)
search_ids = backend.search_ids(
query_str,
user=search_user,
search_mode=search_mode,
)
search_ids = backend.more_like_this_ids(
more_like_doc_id,
user=search_user,
)
else:
query_str, search_mode = _get_tantivy_query_and_mode(filters)
search_ids = backend.search_ids(
query_str,
user=search_user,
search_mode=search_mode,
)
except SearchQueryError as e:
# Same user-fixable-query mapping as the search list endpoint:
# a bad date/number in a bulk selection filter is a 400 naming
# the value, never a 500.
raise ValidationError({"query": search_query_error_messages(e)}) from e
return search_ids
@@ -3644,6 +3675,10 @@ class GlobalSearchView(PassUserMixin):
return HttpResponseBadRequest("Query required")
if len(query) < 3:
return HttpResponseBadRequest("Query must be at least 3 characters")
if len(query) > _MAX_QUERY_LENGTH:
return HttpResponseBadRequest(
f"Query must be at most {_MAX_QUERY_LENGTH} characters",
)
db_only = request.query_params.get("db_only", False)