paperless-ngx/src/documents/tests/search/test_query.py

from __future__ import annotations

from datetime import UTC
from datetime import datetime
from typing import TYPE_CHECKING

import pytest
import tantivy
import time_machine

from documents.search._backend import build_permission_filter
from documents.search._errors import InvalidDateQuery
from documents.search._errors import InvalidNumberQuery
from documents.search._errors import MultipleSearchQueryErrors
from documents.search._errors import SearchQueryError
from documents.search._query import parse_simple_text_highlight_query
from documents.search._query import parse_user_query
from documents.search._schema import build_schema
from documents.search._tokenizer import register_tokenizers

if TYPE_CHECKING:
    from django.contrib.auth.base_user import AbstractBaseUser

pytestmark = pytest.mark.search


@pytest.fixture(scope="module")
def query_index() -> tantivy.Index:
    """An in-memory, unstemmed index shared read-only across this module's
    parse-only tests (none of them index documents)."""
    schema = build_schema()
    idx = tantivy.Index(schema, path=None)
    register_tokenizers(idx, "")
    return idx


@pytest.fixture(scope="module")
def populated_index() -> tantivy.Index:
    """An index holding one document, so a query matching nothing is
    distinguishable from one matching everything."""
    idx = tantivy.Index(build_schema(), path=None)
    register_tokenizers(idx, "")
    writer = idx.writer()
    doc = tantivy.Document()
    doc.add_unsigned("id", 1)
    doc.add_text("content", "needle in indexed content")
    writer.add_document(doc)
    writer.commit()
    idx.reload()
    return idx


def _highlight_hit_count(index: tantivy.Index, raw_query: str) -> int:
    query = parse_simple_text_highlight_query(index, raw_query)
    return index.searcher().search(query, limit=1).count


class TestParseUserQuery:
    """parse_user_query runs the full preprocessing pipeline."""

    def test_returns_tantivy_query(self, query_index: tantivy.Index) -> None:
        """
        GIVEN:
            - An in-memory index built from build_schema()
        WHEN:
            - parse_user_query() parses plain text ("invoice")
        THEN:
            - It returns a tantivy.Query, the object type callers depend on
        """
        assert isinstance(parse_user_query(query_index, "invoice", UTC), tantivy.Query)

    @pytest.mark.parametrize(
        "raw_query",
        [
            pytest.param("invoice", id="plain_text"),
            pytest.param("created:today", id="date_keyword"),
            pytest.param("created:[2005 to 2009]", id="whoosh_date_range"),
            pytest.param('added:"previous month"', id="quoted_date_phrase"),
            pytest.param("title:202[0-1]*", id="bracket_class_wildcard"),
        ],
    )
    def test_fuzzy_mode_does_not_raise(
        self,
        query_index: tantivy.Index,
        settings,
        raw_query: str,
    ) -> None:
        """
        GIVEN:
            - The fuzzy blend clause enabled (ADVANCED_FUZZY_SEARCH_THRESHOLD
              set), and a query that is valid whoosh grammar tantivy's own
              query parser (used only by the fuzzy blend clause) cannot parse
        WHEN:
            - parse_user_query() parses it
        THEN:
            - It returns a tantivy.Query rather than raising: the fuzzy
              clause (_try_parse_fuzzy_query) must degrade gracefully
              instead of failing the whole query
        """
        settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.5
        assert isinstance(parse_user_query(query_index, raw_query, UTC), tantivy.Query)

    def test_date_keyword_resolves_without_raising(
        self,
        query_index: tantivy.Index,
    ) -> None:
        """
        GIVEN:
            - A frozen "now" and a query using a date keyword ("today")
        WHEN:
            - parse_user_query() parses it (whoosh-compat's DateParserPlugin
              resolves "today" against the AST directly - no string rewrite
              to an ISO range happens anywhere in this pipeline)
        THEN:
            - The emitted tantivy query builds cleanly, returning a
              tantivy.Query
        """
        with time_machine.travel(datetime(2026, 3, 28, 12, 0, tzinfo=UTC), tick=False):
            q = parse_user_query(query_index, "created:today", UTC)
        assert isinstance(q, tantivy.Query)

    @pytest.mark.parametrize(
        "raw_query",
        [
            pytest.param("h52.1 - kurzsichtigkeit", id="icd_code_dash_description"),
            pytest.param("H52.1 - asd", id="icd_code_uppercase"),
            pytest.param("h52.1 -", id="trailing_minus"),
            pytest.param(". -", id="dot_trailing_minus"),
            pytest.param("h52. -", id="partial_code_trailing_minus"),
            pytest.param(".12 -", id="dot_number_trailing_minus"),
            pytest.param("h52.1 - ku", id="partial_word_after_dash"),
        ],
    )
    def test_spaced_dash_queries_do_not_raise(
        self,
        query_index: tantivy.Index,
        raw_query: str,
    ) -> None:
        """
        GIVEN:
            - A query in the ICD-code-with-spaced-dash shape that previously
              broke parsing (e.g. "h52.1 - kurzsichtigkeit")
        WHEN:
            - parse_user_query() parses it
        THEN:
            - It returns a tantivy.Query rather than raising
        """
        assert isinstance(parse_user_query(query_index, raw_query, UTC), tantivy.Query)

    def test_invalid_date_propagates_not_swallowed(
        self,
        query_index: tantivy.Index,
    ) -> None:
        """
        GIVEN:
            - A query with an invalid date value ("created:202023")
        WHEN:
            - parse_user_query() parses it
        THEN:
            - It raises InvalidDateQuery naming the field and value, rather
              than falling back to silently parsing the raw (invalid) date;
              a bad date diagnostic from whoosh-compat always maps to
              InvalidDateQuery and must propagate so the view can return a
              400
        """
        with pytest.raises(InvalidDateQuery) as exc_info:
            parse_user_query(query_index, "created:202023", UTC)
        assert exc_info.value.field == "created"
        assert exc_info.value.value == "202023"

    def test_invalid_number_raises_invalid_number_query(
        self,
        query_index: tantivy.Index,
    ) -> None:
        """
        GIVEN:
            - A query with an invalid numeric value ("asn:notanumber")
        WHEN:
            - parse_user_query() parses it
        THEN:
            - It raises InvalidNumberQuery naming the field and value
        """
        with pytest.raises(InvalidNumberQuery) as exc_info:
            parse_user_query(query_index, "asn:notanumber", UTC)
        assert exc_info.value.field == "asn"
        assert exc_info.value.value == "notanumber"

    def test_multiple_bad_fields_raise_multiple_search_query_errors(
        self,
        query_index: tantivy.Index,
    ) -> None:
        """
        GIVEN:
            - A query with two independently invalid fields
              ("created:notadate AND asn:notanumber")
        WHEN:
            - parse_user_query() parses it
        THEN:
            - It raises MultipleSearchQueryErrors aggregating both
              underlying errors (one InvalidDateQuery, one
              InvalidNumberQuery), rather than surfacing only the first
        """
        with pytest.raises(MultipleSearchQueryErrors) as exc_info:
            parse_user_query(
                query_index,
                "created:notadate AND asn:notanumber",
                UTC,
            )
        assert len(exc_info.value.errors) == 2
        kinds = {type(e) for e in exc_info.value.errors}
        assert kinds == {InvalidDateQuery, InvalidNumberQuery}


class TestParseSimpleTextHighlightQuery:
    """parse_simple_text_highlight_query must not raise on natural-language queries."""

    @pytest.mark.parametrize(
        "raw_query",
        [
            pytest.param("h52.1 - kurzsichtigkeit", id="icd_code_dash_description"),
            pytest.param("H52.1 - asd", id="icd_code_uppercase"),
            pytest.param("h52.1 -", id="trailing_minus"),
            pytest.param(". -", id="dot_trailing_minus"),
            pytest.param(".12 -", id="dot_number_trailing_minus"),
            pytest.param("f84.0 - v.a. autismusspektrumstorung", id="complex_icd_dash"),
        ],
    )
    def test_spaced_dash_queries_do_not_raise(
        self,
        query_index: tantivy.Index,
        raw_query: str,
    ) -> None:
        """
        GIVEN:
            - A query in the ICD-code-with-spaced-dash shape that previously
              broke parsing (e.g. "h52.1 - kurzsichtigkeit")
        WHEN:
            - parse_simple_text_highlight_query() parses it
        THEN:
            - It returns a tantivy.Query rather than raising
        """
        assert isinstance(
            parse_simple_text_highlight_query(query_index, raw_query),
            tantivy.Query,
        )

    def test_a_real_token_matches_the_corpus(
        self,
        populated_index: tantivy.Index,
    ) -> None:
        """
        GIVEN:
            - An index holding one document containing "needle"
        WHEN:
            - A highlight query for "needle" is run
        THEN:
            - It matches the document; without this control, an empty
              corpus would make the two assertions below pass for a query
              that matches every document
        """
        assert _highlight_hit_count(populated_index, "needle") == 1

    def test_empty_query_matches_no_document(
        self,
        populated_index: tantivy.Index,
    ) -> None:
        """
        GIVEN:
            - An index holding one document
        WHEN:
            - A highlight query for the empty string is run
        THEN:
            - It matches nothing, rather than degenerating to match-all
        """
        assert _highlight_hit_count(populated_index, "") == 0

    def test_all_operators_query_matches_no_document(
        self,
        populated_index: tantivy.Index,
    ) -> None:
        """
        GIVEN:
            - An index holding one document
        WHEN:
            - A highlight query consisting only of operator characters
              ("- +", no real terms) is run
        THEN:
            - It matches nothing, rather than degenerating to match-all
        """
        assert _highlight_hit_count(populated_index, "- +") == 0


class TestPermissionFilter:
    """
    build_permission_filter tests use an in-memory index - no DB access needed.

    Users are constructed as unsaved model instances (django_user_model(pk=N))
    so no database round-trip occurs; only .pk is read by build_permission_filter.
    """

    @pytest.fixture
    def perm_index(self) -> tantivy.Index:
        schema = build_schema()
        idx = tantivy.Index(schema, path=None)
        register_tokenizers(idx, "")
        return idx

    def _add_doc(
        self,
        idx: tantivy.Index,
        doc_id: int,
        owner_id: int | None = None,
        viewer_ids: tuple[int, ...] = (),
    ) -> None:
        writer = idx.writer()
        doc = tantivy.Document()
        doc.add_unsigned("id", doc_id)
        # Only add owner_id field if the document has an owner
        if owner_id is not None:
            doc.add_unsigned("owner_id", owner_id)
        for vid in viewer_ids:
            doc.add_unsigned("viewer_id", vid)
        writer.add_document(doc)
        writer.commit()
        idx.reload()

    def test_perm_no_owner_visible_to_any_user(
        self,
        perm_index: tantivy.Index,
        django_user_model: type[AbstractBaseUser],
    ) -> None:
        """
        GIVEN:
            - A document with no owner
        WHEN:
            - build_permission_filter() builds a filter for an arbitrary
              user
        THEN:
            - The document is visible (unowned documents are visible to
              every user)
        """
        self._add_doc(perm_index, doc_id=1, owner_id=None)
        user = django_user_model(pk=99)
        perm = build_permission_filter(perm_index.schema, user)
        assert perm_index.searcher().search(perm, limit=10).count == 1

    def test_perm_owned_by_user_is_visible(
        self,
        perm_index: tantivy.Index,
        django_user_model: type[AbstractBaseUser],
    ) -> None:
        """
        GIVEN:
            - A document owned by user 42
        WHEN:
            - build_permission_filter() builds a filter for user 42 (the
              owner)
        THEN:
            - The document is visible
        """
        self._add_doc(perm_index, doc_id=2, owner_id=42)
        user = django_user_model(pk=42)
        perm = build_permission_filter(perm_index.schema, user)
        assert perm_index.searcher().search(perm, limit=10).count == 1

    def test_perm_owned_by_other_not_visible(
        self,
        perm_index: tantivy.Index,
        django_user_model: type[AbstractBaseUser],
    ) -> None:
        """
        GIVEN:
            - A document owned by user 42
        WHEN:
            - build_permission_filter() builds a filter for a different
              user (99)
        THEN:
            - The document is not visible
        """
        self._add_doc(perm_index, doc_id=3, owner_id=42)
        user = django_user_model(pk=99)
        perm = build_permission_filter(perm_index.schema, user)
        assert perm_index.searcher().search(perm, limit=10).count == 0

    def test_perm_shared_viewer_is_visible(
        self,
        perm_index: tantivy.Index,
        django_user_model: type[AbstractBaseUser],
    ) -> None:
        """
        GIVEN:
            - A document owned by user 42 and explicitly shared with
              viewer 99
        WHEN:
            - build_permission_filter() builds a filter for the viewer (99)
        THEN:
            - The document is visible
        """
        self._add_doc(perm_index, doc_id=4, owner_id=42, viewer_ids=(99,))
        user = django_user_model(pk=99)
        perm = build_permission_filter(perm_index.schema, user)
        assert perm_index.searcher().search(perm, limit=10).count == 1

    def test_perm_only_owned_docs_hidden_from_others(
        self,
        perm_index: tantivy.Index,
        django_user_model: type[AbstractBaseUser],
    ) -> None:
        """
        GIVEN:
            - One document owned by user 10 and one unowned document
        WHEN:
            - build_permission_filter() builds a filter for a third user
              (20, who owns neither)
        THEN:
            - Only the unowned document is visible
        """
        self._add_doc(perm_index, doc_id=5, owner_id=10)  # owned by 10
        self._add_doc(perm_index, doc_id=6, owner_id=None)  # unowned
        user = django_user_model(pk=20)
        perm = build_permission_filter(perm_index.schema, user)
        assert perm_index.searcher().search(perm, limit=10).count == 1  # only unowned


class TestSearchQueryErrors:
    def test_invalid_date_query_is_a_search_query_error(self) -> None:
        """
        GIVEN:
            - An InvalidDateQuery constructed with a field and bad value
        WHEN:
            - It is inspected
        THEN:
            - It is a SearchQueryError carrying the field/value, and both
              appear in its string form
        """
        err = InvalidDateQuery("created", "notadate")
        assert isinstance(err, SearchQueryError)
        assert err.field == "created"
        assert err.value == "notadate"
        assert "created" in str(err)
        assert "notadate" in str(err)

    def test_invalid_number_query_is_a_search_query_error(self) -> None:
        """
        GIVEN:
            - An InvalidNumberQuery constructed with a field and bad value
        WHEN:
            - It is inspected
        THEN:
            - It is a SearchQueryError carrying the field/value, and both
              appear in its string form
        """
        err = InvalidNumberQuery("asn", "notanumber")
        assert isinstance(err, SearchQueryError)
        assert err.field == "asn"
        assert err.value == "notanumber"
        assert "asn" in str(err)
        assert "notanumber" in str(err)

    def test_multiple_search_query_errors_aggregates(self) -> None:
        """
        GIVEN:
            - Two sub-errors (an InvalidDateQuery and an InvalidNumberQuery)
        WHEN:
            - They are wrapped in a MultipleSearchQueryErrors
        THEN:
            - It is a SearchQueryError, preserves both sub-errors, and both
              fields appear in its string form
        """
        sub_errors = [
            InvalidDateQuery("created", "notadate"),
            InvalidNumberQuery("asn", "notanumber"),
        ]
        err = MultipleSearchQueryErrors(sub_errors)
        assert isinstance(err, SearchQueryError)
        assert err.errors == tuple(sub_errors)
        assert "created" in str(err)
        assert "asn" in str(err)


class TestEmitErrorContract:
    """A QueryError from emit() surfaces as a SearchQueryError (HTTP 400).

    The Cause-based routing table itself is covered in test_error_routing.py.
    """

    def test_exists_requires_fast_gets_the_user_facing_rewrite(
        self,
        query_index: tantivy.Index,
    ) -> None:
        """
        GIVEN:
            - A field:* existence query against a non-fast JSON subpath
              ("notes.user:*")
        WHEN:
            - parse_user_query() maps the resulting QueryError
        THEN:
            - The user-facing message is paperless's own rewrite, not
              whoosh-compat's: whoosh-compat's own message advises a
              host-side fast=True config change the user can't act on, so
              this checks OUR wording (whoosh-compat's own wording is that
              library's own test suite's job)
        """
        with pytest.raises(SearchQueryError) as exc_info:
            parse_user_query(query_index, "notes.user:*", UTC)
        assert str(exc_info.value) == (
            "Existence searches (field:*) are not supported for field 'notes.user'."
        )