diff --git a/src/documents/tests/search/test_schema_fingerprint.py b/src/documents/tests/search/test_schema_fingerprint.py new file mode 100644 index 000000000..253861247 --- /dev/null +++ b/src/documents/tests/search/test_schema_fingerprint.py @@ -0,0 +1,492 @@ +"""The schema fingerprint stamped into .index_settings.json. + +tantivy compares schemas by *ordered* field list, and `tantivy.Index(schema, +path=...)` (what every write path does) raises on any difference. SCHEMA_VERSION +is the manual guard against that, but build_schema() is edited for *parser* +reasons - adding an alias, flipping fast=True, adding a subpath - by people not +thinking about the on-disk index, and forgetting the bump is exactly how this +branch's bug happened. + +The fingerprint is the automatic guard: it hashes the field descriptor list that +build_schema() itself iterates, so any change to a field's name, kind, options +or *position* forces a rebuild on its own. +""" + +from __future__ import annotations + +import hashlib +import json +from typing import TYPE_CHECKING + +import pytest +import tantivy + +from documents.search import _schema +from documents.search._schema import SCHEMA_VERSION +from documents.search._schema import FieldDescriptor +from documents.search._schema import _write_sentinels +from documents.search._schema import build_schema +from documents.search._schema import field_descriptors +from documents.search._schema import needs_rebuild +from documents.search._schema import schema_fingerprint + +if TYPE_CHECKING: + from pathlib import Path + + from pytest_django.fixtures import SettingsWrapper + +pytestmark = pytest.mark.search + +# The on-disk field layout of a v2 index, pinned as data. Any edit here is an +# index-format change: it must come with a rebuild, which the fingerprint now +# forces automatically. Reproduced from build_schema()'s output as it stood +# before the descriptor refactor, so it also pins that the refactor changed +# nothing. +PINNED_DESCRIPTORS: tuple[FieldDescriptor, ...] = ( + FieldDescriptor("id", "u64", stored=True, indexed=True, fast=True, tokenizer=None), + FieldDescriptor( + "title", + "text", + stored=True, + indexed=True, + fast=False, + tokenizer="paperless_text", + ), + FieldDescriptor( + "content", + "text", + stored=True, + indexed=True, + fast=False, + tokenizer="paperless_text", + ), + FieldDescriptor( + "correspondent", + "text", + stored=True, + indexed=True, + fast=False, + tokenizer="paperless_text", + ), + FieldDescriptor( + "document_type", + "text", + stored=True, + indexed=True, + fast=False, + tokenizer="paperless_text", + ), + FieldDescriptor( + "storage_path", + "text", + stored=True, + indexed=True, + fast=False, + tokenizer="paperless_text", + ), + FieldDescriptor( + "original_filename", + "text", + stored=True, + indexed=True, + fast=False, + tokenizer="paperless_text", + ), + FieldDescriptor( + "tag", + "text", + stored=True, + indexed=True, + fast=False, + tokenizer="paperless_text", + ), + FieldDescriptor( + "checksum", + "text", + stored=True, + indexed=True, + fast=False, + tokenizer="raw", + ), + FieldDescriptor("asn", "u64", stored=True, indexed=True, fast=True, tokenizer=None), + FieldDescriptor( + "page_count", + "u64", + stored=True, + indexed=True, + fast=True, + tokenizer=None, + ), + FieldDescriptor( + "num_notes", + "u64", + stored=True, + indexed=True, + fast=True, + tokenizer=None, + ), + FieldDescriptor( + "created", + "date", + stored=True, + indexed=True, + fast=True, + tokenizer=None, + ), + FieldDescriptor( + "modified", + "date", + stored=True, + indexed=True, + fast=True, + tokenizer=None, + ), + FieldDescriptor( + "added", + "date", + stored=True, + indexed=True, + fast=True, + tokenizer=None, + ), + FieldDescriptor( + "notes", + "json", + stored=True, + indexed=True, + fast=False, + tokenizer="paperless_text", + ), + FieldDescriptor( + "notes_text", + "text", + stored=True, + indexed=True, + fast=False, + tokenizer="paperless_text", + ), + FieldDescriptor( + "custom_fields", + "json", + stored=True, + indexed=True, + fast=False, + tokenizer="paperless_text", + ), + FieldDescriptor( + "title_sort", + "text", + stored=False, + indexed=True, + fast=True, + tokenizer="simple_analyzer", + ), + FieldDescriptor( + "correspondent_sort", + "text", + stored=False, + indexed=True, + fast=True, + tokenizer="simple_analyzer", + ), + FieldDescriptor( + "type_sort", + "text", + stored=False, + indexed=True, + fast=True, + tokenizer="simple_analyzer", + ), + FieldDescriptor( + "bigram_content", + "text", + stored=False, + indexed=True, + fast=False, + tokenizer="bigram_analyzer", + ), + FieldDescriptor( + "bigram_title", + "text", + stored=False, + indexed=True, + fast=False, + tokenizer="bigram_analyzer", + ), + FieldDescriptor( + "bigram_correspondent", + "text", + stored=False, + indexed=True, + fast=False, + tokenizer="bigram_analyzer", + ), + FieldDescriptor( + "bigram_document_type", + "text", + stored=False, + indexed=True, + fast=False, + tokenizer="bigram_analyzer", + ), + FieldDescriptor( + "bigram_tag", + "text", + stored=False, + indexed=True, + fast=False, + tokenizer="bigram_analyzer", + ), + FieldDescriptor( + "simple_title", + "text", + stored=False, + indexed=True, + fast=False, + tokenizer="simple_search_analyzer", + ), + FieldDescriptor( + "simple_content", + "text", + stored=False, + indexed=True, + fast=False, + tokenizer="simple_search_analyzer", + ), + FieldDescriptor( + "autocomplete_word", + "text", + stored=False, + indexed=True, + fast=False, + tokenizer="raw", + ), + FieldDescriptor( + "owner_id", + "u64", + stored=False, + indexed=True, + fast=True, + tokenizer=None, + ), + FieldDescriptor( + "viewer_id", + "u64", + stored=False, + indexed=True, + fast=True, + tokenizer=None, + ), + FieldDescriptor( + "viewer_group_id", + "u64", + stored=False, + indexed=True, + fast=True, + tokenizer=None, + ), +) + + +def _schema_fields(schema: tantivy.Schema) -> list[dict]: + """The tantivy-level field list, in declaration order. + + tantivy-py 0.26 exposes no public introspection API on Schema, so + __reduce__() (its pickling hook) is the only way to recover the field list. + It is used here, in a test, precisely because it is the representation the + persisted fingerprint must NOT depend on. + """ + return schema.__reduce__()[1][0]["inner"] + + +def _sentinels(index_dir: Path, **overrides: object) -> None: + data = { + "schema_version": SCHEMA_VERSION, + "language": None, + "schema_fingerprint": schema_fingerprint(), + } + data.update(overrides) + (index_dir / ".index_settings.json").write_text(json.dumps(data)) + + +class TestDescriptorsDescribeTheBuiltSchema: + def test_descriptors_match_the_pinned_field_layout(self) -> None: + assert tuple(field_descriptors()) == PINNED_DESCRIPTORS + + def test_built_schema_matches_the_descriptors(self) -> None: + """The descriptors are not a parallel description - they are the input. + + Reading the built schema back proves the loop honours every option, so + a descriptor edit cannot claim a shape the SchemaBuilder did not build. + """ + kinds = {"text": "text", "json": "json_object", "u64": "u64", "date": "date"} + built = [ + ( + field["name"], + field["type"], + field["options"]["stored"], + bool(field["options"].get("fast")), + (field["options"].get("indexing") or {}).get("tokenizer"), + ) + for field in _schema_fields(build_schema()) + ] + expected = [ + ( + descriptor.name, + kinds[descriptor.kind], + descriptor.stored, + descriptor.fast, + descriptor.tokenizer, + ) + for descriptor in field_descriptors() + ] + assert built == expected + + +class TestFingerprintSensitivity: + def test_a_field_option_change_moves_the_fingerprint( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + before = schema_fingerprint() + changed = field_descriptors() + changed[1] = changed[1]._replace(fast=True) + monkeypatch.setattr(_schema, "field_descriptors", lambda: changed) + + assert schema_fingerprint() != before + + def test_reordering_alone_moves_the_fingerprint( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """The original bug: same fields, different declaration order. + + A set- or dict-based fingerprint would be blind to this, and tantivy + would reject every write against the existing index. + """ + before = schema_fingerprint() + swapped = field_descriptors() + swapped[1], swapped[2] = swapped[2], swapped[1] + monkeypatch.setattr(_schema, "field_descriptors", lambda: swapped) + + assert schema_fingerprint() != before + + def test_repeated_calls_agree(self) -> None: + assert schema_fingerprint() == schema_fingerprint() + + +class TestFingerprintIsIndependentOfTantivy: + def test_a_tantivy_option_key_addition_would_not_move_it(self) -> None: + """A tantivy-py upgrade must not force a global reindex. + + Hashing schema.__reduce__() would do exactly that: the simulated new + option key below changes that payload for every user with no schema + change at all. + """ + fields = _schema_fields(build_schema()) + upgraded = [ + {**field, "options": {**field["options"], "coerce": True}} + for field in fields + ] + assert _hash(upgraded) != _hash(fields) + assert schema_fingerprint() == _fingerprint_of(field_descriptors()) + + def test_fingerprint_never_touches_the_schema_builder( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + before = schema_fingerprint() + + class _RemovedSchemaBuilder: + def __init__(self) -> None: + raise AssertionError("tantivy.SchemaBuilder was consulted") + + monkeypatch.setattr(tantivy, "SchemaBuilder", _RemovedSchemaBuilder) + with pytest.raises(AssertionError): + build_schema() + + assert schema_fingerprint() == before + + +def _hash(payload: object) -> str: + return hashlib.blake2b(json.dumps(payload).encode()).hexdigest() + + +def _fingerprint_of(descriptors: list[FieldDescriptor]) -> str: + return _hash([list(descriptor) for descriptor in descriptors]) + + +class TestNeedsRebuildOnFingerprint: + def test_matching_fingerprint_does_not_rebuild( + self, + index_dir: Path, + settings: SettingsWrapper, + ) -> None: + settings.SEARCH_LANGUAGE = None + _sentinels(index_dir) + + assert needs_rebuild(index_dir) is False + + def test_stale_fingerprint_rebuilds_despite_a_matching_version( + self, + index_dir: Path, + settings: SettingsWrapper, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """The failure this task exists to prevent: schema edited, version not + bumped. Without the fingerprint check, `reindex --if-needed` reports the + index up to date and every write then raises.""" + settings.SEARCH_LANGUAGE = None + _sentinels(index_dir) + extended = [ + *field_descriptors(), + FieldDescriptor( + "new_field", + "u64", + stored=False, + indexed=True, + fast=True, + tokenizer=None, + ), + ] + monkeypatch.setattr(_schema, "field_descriptors", lambda: extended) + + assert needs_rebuild(index_dir) is True + + def test_reordered_schema_rebuilds( + self, + index_dir: Path, + settings: SettingsWrapper, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + settings.SEARCH_LANGUAGE = None + _sentinels(index_dir) + reordered = field_descriptors() + reordered[1], reordered[2] = reordered[2], reordered[1] + monkeypatch.setattr(_schema, "field_descriptors", lambda: reordered) + + assert needs_rebuild(index_dir) is True + + def test_missing_fingerprint_rebuilds( + self, + index_dir: Path, + settings: SettingsWrapper, + ) -> None: + """No seeding: an index whose schema shape nobody recorded is rebuilt + rather than trusted.""" + settings.SEARCH_LANGUAGE = None + (index_dir / ".index_settings.json").write_text( + json.dumps({"schema_version": SCHEMA_VERSION, "language": None}), + ) + + assert needs_rebuild(index_dir) is True + + def test_written_sentinels_satisfy_the_check( + self, + index_dir: Path, + settings: SettingsWrapper, + ) -> None: + settings.SEARCH_LANGUAGE = "en" + _write_sentinels(index_dir) + + assert needs_rebuild(index_dir) is False diff --git a/src/documents/tests/search/test_schema_version.py b/src/documents/tests/search/test_schema_version.py new file mode 100644 index 000000000..902256c48 --- /dev/null +++ b/src/documents/tests/search/test_schema_version.py @@ -0,0 +1,164 @@ +"""SCHEMA_VERSION must change whenever build_schema()'s field list or order does. + +tantivy compares schemas by *ordered* field list. ``Index.open()`` loads the +schema from the index's own ``meta.json``, so reads against an index built by an +older release keep working after a field reorder. Writes do not: +``WriteBatch.__enter__`` calls ``tantivy.Index(build_schema(), path=...)``, an +open-or-create that raises ``ValueError`` on any schema difference. Nothing +catches that ValueError, so consumption, index_document and bulk edit all +hard-fail while ``/api/status/`` still reports the index healthy. + +The only thing that saves such an install is ``needs_rebuild()`` noticing the +version stamped in ``.index_settings.json`` is stale. +""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING + +import pytest +import tantivy +from django.conf import settings as django_settings + +from documents.search._schema import build_schema +from documents.search._schema import needs_rebuild +from documents.search._schema import open_or_rebuild_index + +if TYPE_CHECKING: + from pathlib import Path + +pytestmark = [pytest.mark.search] + +RELEASED_V1_SCHEMA_VERSION = 1 + + +def _build_released_v1_schema() -> tantivy.Schema: + """Frozen copy of build_schema() as shipped in v3.0.x (schema version 1). + + Deliberately duplicated rather than imported: it must keep describing the + on-disk layout of already-deployed indexes even as build_schema() evolves. + """ + sb = tantivy.SchemaBuilder() + + sb.add_unsigned_field("id", stored=True, indexed=True, fast=True) + sb.add_text_field("checksum", stored=True, tokenizer_name="raw") + + for field in ( + "title", + "correspondent", + "document_type", + "storage_path", + "original_filename", + "content", + ): + sb.add_text_field(field, stored=True, tokenizer_name="paperless_text") + + for field in ("title_sort", "correspondent_sort", "type_sort"): + sb.add_text_field( + field, + stored=False, + tokenizer_name="simple_analyzer", + fast=True, + ) + + for field in ( + "bigram_content", + "bigram_title", + "bigram_correspondent", + "bigram_document_type", + "bigram_tag", + ): + sb.add_text_field(field, stored=False, tokenizer_name="bigram_analyzer") + + for field in ("simple_title", "simple_content"): + sb.add_text_field(field, stored=False, tokenizer_name="simple_search_analyzer") + + sb.add_text_field("autocomplete_word", stored=False, tokenizer_name="raw") + sb.add_text_field("tag", stored=True, tokenizer_name="paperless_text") + + sb.add_json_field("notes", stored=True, tokenizer_name="paperless_text") + sb.add_text_field("notes_text", stored=True, tokenizer_name="paperless_text") + sb.add_json_field("custom_fields", stored=True, tokenizer_name="paperless_text") + + for field in ( + "correspondent_id", + "document_type_id", + "storage_path_id", + "tag_id", + "owner_id", + "viewer_id", + "viewer_group_id", + ): + sb.add_unsigned_field(field, stored=False, indexed=True, fast=True) + + for field in ("created", "modified", "added"): + sb.add_date_field(field, stored=True, indexed=True, fast=True) + + for field in ("asn", "page_count", "num_notes"): + sb.add_unsigned_field(field, stored=True, indexed=True, fast=True) + + return sb.build() + + +@pytest.fixture +def released_v1_index(tmp_path: Path) -> Path: + """An index directory as a v3.0.x install would leave it on disk.""" + index_dir = tmp_path / "index" + index_dir.mkdir() + tantivy.Index(_build_released_v1_schema(), path=str(index_dir)) + (index_dir / ".index_settings.json").write_text( + json.dumps( + { + "schema_version": RELEASED_V1_SCHEMA_VERSION, + "language": django_settings.SEARCH_LANGUAGE, + }, + ), + ) + return index_dir + + +class TestUpgradeFromReleasedV1Index: + def test_released_v1_index_is_flagged_for_rebuild( + self, + released_v1_index: Path, + ) -> None: + """The current schema differs from v1's, so the sentinel must be stale. + + If this fails, `document_index reindex --if-needed` prints "Search index + is up to date" and skips, leaving the mismatched index in place. + """ + assert needs_rebuild(released_v1_index) is True + + def test_v1_index_rejects_writes_against_the_current_schema( + self, + released_v1_index: Path, + ) -> None: + """The failure mode the version bump exists to prevent. + + This is exactly what WriteBatch.__enter__ does on every index write. + """ + with pytest.raises(ValueError, match="schema does not match"): + tantivy.Index(build_schema(), path=str(released_v1_index)) + + def test_opening_a_v1_index_leaves_it_writable( + self, + released_v1_index: Path, + ) -> None: + """End to end: open_or_rebuild_index must hand back an index that the + write path can reopen. Before the version bump, needs_rebuild() returned + False here, the stale directory survived untouched, and every subsequent + write raised the ValueError above.""" + open_or_rebuild_index(released_v1_index) + + tantivy.Index(build_schema(), path=str(released_v1_index)) + + def test_rebuilt_index_is_not_rebuilt_again( + self, + released_v1_index: Path, + ) -> None: + """The rebuild must stamp the version it actually wrote, otherwise every + startup wipes and reindexes the whole corpus.""" + open_or_rebuild_index(released_v1_index) + + assert needs_rebuild(released_v1_index) is False