paperless-ngx/src/documents/tests/search/test_schema_version.py

"""SCHEMA_VERSION must change whenever build_schema()'s field list or order does.

tantivy compares schemas by *ordered* field list. ``Index.open()`` loads the
schema from the index's own ``meta.json``, so reads against an index built by an
older release keep working after a field reorder. Writes do not:
``WriteBatch.__enter__`` calls ``tantivy.Index(build_schema(), path=...)``, an
open-or-create that raises ``ValueError`` on any schema difference. Nothing
catches that ValueError, so consumption, index_document and bulk edit all
hard-fail while ``/api/status/`` still reports the index healthy.

The only thing that saves such an install is ``needs_rebuild()`` noticing the
version stamped in ``.index_settings.json`` is stale.
"""

from __future__ import annotations

import json
from typing import TYPE_CHECKING

import pytest
import tantivy
from django.conf import settings as django_settings

from documents.search._schema import build_schema
from documents.search._schema import needs_rebuild
from documents.search._schema import open_or_rebuild_index

if TYPE_CHECKING:
    from pathlib import Path

pytestmark = [pytest.mark.search]

RELEASED_V1_SCHEMA_VERSION = 1


def _build_released_v1_schema() -> tantivy.Schema:
    """Frozen copy of build_schema() as shipped in v3.0.x (schema version 1).

    Deliberately duplicated rather than imported: it must keep describing the
    on-disk layout of already-deployed indexes even as build_schema() evolves.
    """
    sb = tantivy.SchemaBuilder()

    sb.add_unsigned_field("id", stored=True, indexed=True, fast=True)
    sb.add_text_field("checksum", stored=True, tokenizer_name="raw")

    for field in (
        "title",
        "correspondent",
        "document_type",
        "storage_path",
        "original_filename",
        "content",
    ):
        sb.add_text_field(field, stored=True, tokenizer_name="paperless_text")

    for field in ("title_sort", "correspondent_sort", "type_sort"):
        sb.add_text_field(
            field,
            stored=False,
            tokenizer_name="simple_analyzer",
            fast=True,
        )

    for field in (
        "bigram_content",
        "bigram_title",
        "bigram_correspondent",
        "bigram_document_type",
        "bigram_tag",
    ):
        sb.add_text_field(field, stored=False, tokenizer_name="bigram_analyzer")

    for field in ("simple_title", "simple_content"):
        sb.add_text_field(field, stored=False, tokenizer_name="simple_search_analyzer")

    sb.add_text_field("autocomplete_word", stored=False, tokenizer_name="raw")
    sb.add_text_field("tag", stored=True, tokenizer_name="paperless_text")

    sb.add_json_field("notes", stored=True, tokenizer_name="paperless_text")
    sb.add_text_field("notes_text", stored=True, tokenizer_name="paperless_text")
    sb.add_json_field("custom_fields", stored=True, tokenizer_name="paperless_text")

    for field in (
        "correspondent_id",
        "document_type_id",
        "storage_path_id",
        "tag_id",
        "owner_id",
        "viewer_id",
        "viewer_group_id",
    ):
        sb.add_unsigned_field(field, stored=False, indexed=True, fast=True)

    for field in ("created", "modified", "added"):
        sb.add_date_field(field, stored=True, indexed=True, fast=True)

    for field in ("asn", "page_count", "num_notes"):
        sb.add_unsigned_field(field, stored=True, indexed=True, fast=True)

    return sb.build()


@pytest.fixture
def released_v1_index(tmp_path: Path) -> Path:
    """An index directory as a v3.0.x install would leave it on disk."""
    index_dir = tmp_path / "index"
    index_dir.mkdir()
    tantivy.Index(_build_released_v1_schema(), path=str(index_dir))
    (index_dir / ".index_settings.json").write_text(
        json.dumps(
            {
                "schema_version": RELEASED_V1_SCHEMA_VERSION,
                "language": django_settings.SEARCH_LANGUAGE,
            },
        ),
    )
    return index_dir


class TestUpgradeFromReleasedV1Index:
    def test_released_v1_index_is_flagged_for_rebuild(
        self,
        released_v1_index: Path,
    ) -> None:
        """
        GIVEN:
            - An index directory laid out exactly as a v3.0.x (schema
              version 1) install would leave it
        WHEN:
            - needs_rebuild() is called
        THEN:
            - It returns True; if this fails,
              `document_index reindex --if-needed` prints "Search index is
              up to date" and skips, leaving the mismatched index in place
        """
        assert needs_rebuild(released_v1_index) is True

    def test_opening_a_v1_index_leaves_it_writable(
        self,
        released_v1_index: Path,
    ) -> None:
        """
        GIVEN:
            - A v1 index directory
        WHEN:
            - open_or_rebuild_index() is called against it
        THEN:
            - The directory can be reopened with the current schema
              without raising; end to end, open_or_rebuild_index must
              hand back an index the write path can reopen. Before the
              version bump, needs_rebuild() returned False here, and the
              stale directory survived untouched, so every subsequent
              write against it raised tantivy's own schema-mismatch
              ValueError
        """
        open_or_rebuild_index(released_v1_index)

        tantivy.Index(build_schema(), path=str(released_v1_index))

    def test_rebuilt_index_is_not_rebuilt_again(
        self,
        released_v1_index: Path,
    ) -> None:
        """
        GIVEN:
            - A v1 index directory that has just been rebuilt by
              open_or_rebuild_index()
        WHEN:
            - needs_rebuild() is called again
        THEN:
            - It returns False; the rebuild must stamp the version it
              actually wrote, otherwise every startup wipes and reindexes
              the whole corpus
        """
        open_or_rebuild_index(released_v1_index)

        assert needs_rebuild(released_v1_index) is False