diff --git a/src/documents/search/_schema.py b/src/documents/search/_schema.py index 6a6c7bc60..ee4ad6b5e 100644 --- a/src/documents/search/_schema.py +++ b/src/documents/search/_schema.py @@ -1,10 +1,12 @@ from __future__ import annotations +import hashlib import json import logging import shutil from typing import TYPE_CHECKING from typing import Final +from typing import NamedTuple from typing import cast import tantivy @@ -26,6 +28,196 @@ logger = logging.getLogger("paperless.search") SCHEMA_VERSION: Final[int] = 2 +class FieldDescriptor(NamedTuple): + """One tantivy field, in declaration order. + + The descriptor vocabulary is paperless', not tantivy-py's: it is both the + input to the SchemaBuilder and the input to schema_fingerprint(), so the + persisted fingerprint cannot move under a tantivy-py upgrade. + """ + + name: str + kind: str + stored: bool + indexed: bool + fast: bool + tokenizer: str | None + + +def _public_field_descriptors() -> list[FieldDescriptor]: + """Descriptors for the query-visible fields declared in PUBLIC_FIELDS.""" + descriptors: list[FieldDescriptor] = [] + for field in PUBLIC_FIELDS: + if field.kind is FieldKind.TEXT: + descriptors.append( + FieldDescriptor( + field.name, + "text", + stored=True, + indexed=True, + fast=False, + tokenizer="paperless_text", + ), + ) + elif field.kind is FieldKind.KEYWORD: + descriptors.append( + FieldDescriptor( + field.name, + "text", + stored=True, + indexed=True, + fast=False, + tokenizer="raw", + ), + ) + elif field.kind is FieldKind.U64: + descriptors.append( + FieldDescriptor( + field.name, + "u64", + stored=True, + indexed=True, + fast=field.fast, + tokenizer=None, + ), + ) + elif field.kind in (FieldKind.DATE, FieldKind.DATETIME): + descriptors.append( + FieldDescriptor( + field.name, + "date", + stored=True, + indexed=True, + fast=field.fast, + tokenizer=None, + ), + ) + elif field.kind is FieldKind.JSON: + descriptors.append( + FieldDescriptor( + field.name, + "json", + stored=True, + indexed=True, + fast=False, + tokenizer="paperless_text", + ), + ) + if field.name == "notes": + # Plain-text companion for snippet generation — tantivy's + # SnippetGenerator does not support JSON fields. Schema-only, + # no query-syntax meaning, not in PUBLIC_FIELDS. + descriptors.append( + FieldDescriptor( + "notes_text", + "text", + stored=True, + indexed=True, + fast=False, + tokenizer="paperless_text", + ), + ) + return descriptors + + +def field_descriptors() -> list[FieldDescriptor]: + """Every field of the document index, in the order tantivy declares them. + + tantivy compares schemas by *ordered* field list, so the order here is + part of the on-disk contract: schema_fingerprint() hashes it and + needs_rebuild() acts on the result. + """ + return [ + FieldDescriptor( + "id", + "u64", + stored=True, + indexed=True, + fast=True, + tokenizer=None, + ), + *_public_field_descriptors(), + # Shadow sort fields - fast, not stored + *( + FieldDescriptor( + name, + "text", + stored=False, + indexed=True, + fast=True, + tokenizer="simple_analyzer", + ) + for name in ("title_sort", "correspondent_sort", "type_sort") + ), + # CJK support - not stored, indexed only + *( + FieldDescriptor( + name, + "text", + stored=False, + indexed=True, + fast=False, + tokenizer="bigram_analyzer", + ) + for name in ( + "bigram_content", + "bigram_title", + "bigram_correspondent", + "bigram_document_type", + "bigram_tag", + ) + ), + # Simple substring search support for title/content - not stored, + # indexed only + *( + FieldDescriptor( + name, + "text", + stored=False, + indexed=True, + fast=False, + tokenizer="simple_search_analyzer", + ) + for name in ("simple_title", "simple_content") + ), + # Autocomplete prefix scan via terms_with_prefix, which walks the + # field's term dictionary - so the field must be indexed (term dict), + # not stored. The stored value is never read back, so storing it only + # wastes space. + FieldDescriptor( + "autocomplete_word", + "text", + stored=False, + indexed=True, + fast=False, + tokenizer="raw", + ), + # Permission filter columns, read by build_permission_filter. + *( + FieldDescriptor( + name, + "u64", + stored=False, + indexed=True, + fast=True, + tokenizer=None, + ) + for name in ("owner_id", "viewer_id", "viewer_group_id") + ), + ] + + +def schema_fingerprint() -> str: + """Hash of the field descriptors, stamped into .index_settings.json. + + Changes whenever a field is added, removed, retyped, re-optioned or + reordered, so an index built from a different schema shape is detected + even when SCHEMA_VERSION was not bumped. + """ + payload = json.dumps([list(descriptor) for descriptor in field_descriptors()]) + return hashlib.blake2b(payload.encode()).hexdigest() + + def build_schema() -> tantivy.Schema: """ Build the Tantivy schema for the paperless document index. @@ -39,83 +231,37 @@ def build_schema() -> tantivy.Schema: """ sb = tantivy.SchemaBuilder() - sb.add_unsigned_field("id", stored=True, indexed=True, fast=True) - - for field in PUBLIC_FIELDS: - if field.kind is FieldKind.TEXT: - sb.add_text_field(field.name, stored=True, tokenizer_name="paperless_text") - elif field.kind is FieldKind.KEYWORD: - sb.add_text_field(field.name, stored=True, tokenizer_name="raw") - elif field.kind is FieldKind.U64: + for descriptor in field_descriptors(): + if descriptor.kind == "text": + sb.add_text_field( + descriptor.name, + stored=descriptor.stored, + fast=descriptor.fast, + tokenizer_name=cast("str", descriptor.tokenizer), + ) + elif descriptor.kind == "json": + sb.add_json_field( + descriptor.name, + stored=descriptor.stored, + fast=descriptor.fast, + tokenizer_name=cast("str", descriptor.tokenizer), + ) + elif descriptor.kind == "u64": sb.add_unsigned_field( - field.name, - stored=True, - indexed=True, - fast=field.fast, + descriptor.name, + stored=descriptor.stored, + indexed=descriptor.indexed, + fast=descriptor.fast, ) - elif field.kind in (FieldKind.DATE, FieldKind.DATETIME): + elif descriptor.kind == "date": sb.add_date_field( - field.name, - stored=True, - indexed=True, - fast=field.fast, + descriptor.name, + stored=descriptor.stored, + indexed=descriptor.indexed, + fast=descriptor.fast, ) - elif field.kind is FieldKind.JSON: - sb.add_json_field(field.name, stored=True, tokenizer_name="paperless_text") - if field.name == "notes": - # Plain-text companion for snippet generation — tantivy's - # SnippetGenerator does not support JSON fields. Schema-only, - # no query-syntax meaning, not in PUBLIC_FIELDS. - sb.add_text_field( - "notes_text", - stored=True, - tokenizer_name="paperless_text", - ) - - # Shadow sort fields - fast, not stored/indexed - for field in ("title_sort", "correspondent_sort", "type_sort"): - sb.add_text_field( - field, - stored=False, - tokenizer_name="simple_analyzer", - fast=True, - ) - - # CJK support - not stored, indexed only - sb.add_text_field("bigram_content", stored=False, tokenizer_name="bigram_analyzer") - sb.add_text_field("bigram_title", stored=False, tokenizer_name="bigram_analyzer") - sb.add_text_field( - "bigram_correspondent", - stored=False, - tokenizer_name="bigram_analyzer", - ) - sb.add_text_field( - "bigram_document_type", - stored=False, - tokenizer_name="bigram_analyzer", - ) - sb.add_text_field("bigram_tag", stored=False, tokenizer_name="bigram_analyzer") - - # Simple substring search support for title/content - not stored, indexed only - sb.add_text_field( - "simple_title", - stored=False, - tokenizer_name="simple_search_analyzer", - ) - sb.add_text_field( - "simple_content", - stored=False, - tokenizer_name="simple_search_analyzer", - ) - - # Autocomplete prefix scan via terms_with_prefix, which walks the field's - # term dictionary - so the field must be indexed (term dict), not stored. - # The stored value is never read back, so storing it only wastes space. - sb.add_text_field("autocomplete_word", stored=False, tokenizer_name="raw") - - # Permission filter columns, read by build_permission_filter. - for field in ("owner_id", "viewer_id", "viewer_group_id"): - sb.add_unsigned_field(field, stored=False, indexed=True, fast=True) + else: + raise ValueError(f"Unknown schema field kind: {descriptor.kind}") return sb.build() @@ -124,9 +270,9 @@ def needs_rebuild(index_dir: Path) -> bool: """ Check if the search index needs rebuilding. - Reads .index_settings.json to compare the stored schema version and - search language against the current configuration. Returns True if the - file is missing, unparsable, or either value mismatches. + Reads .index_settings.json to compare the stored schema version, search + language and schema fingerprint against the current configuration. Returns + True if the file is missing, unparsable, or any value mismatches. Args: index_dir: Path to the search index directory @@ -145,6 +291,9 @@ def needs_rebuild(index_dir: Path) -> bool: if "language" not in data or data["language"] != settings.SEARCH_LANGUAGE: logger.info("Search index language changed - rebuilding.") return True + if data.get("schema_fingerprint") != schema_fingerprint(): + logger.info("Search index schema fingerprint mismatch - rebuilding.") + return True except ValueError: return True return False @@ -175,6 +324,7 @@ def _write_sentinels(index_dir: Path) -> None: { "schema_version": SCHEMA_VERSION, "language": settings.SEARCH_LANGUAGE, + "schema_fingerprint": schema_fingerprint(), }, ), ) diff --git a/src/documents/tests/search/test_schema.py b/src/documents/tests/search/test_schema.py index 0358b0e13..739f98171 100644 --- a/src/documents/tests/search/test_schema.py +++ b/src/documents/tests/search/test_schema.py @@ -12,6 +12,7 @@ from documents.search._fields import PUBLIC_FIELDS from documents.search._schema import SCHEMA_VERSION from documents.search._schema import build_schema from documents.search._schema import needs_rebuild +from documents.search._schema import schema_fingerprint from documents.search._tokenizer import register_tokenizers if TYPE_CHECKING: @@ -35,7 +36,13 @@ class TestNeedsRebuild: ) -> None: settings.SEARCH_LANGUAGE = "en" (index_dir / ".index_settings.json").write_text( - json.dumps({"schema_version": SCHEMA_VERSION, "language": "en"}), + json.dumps( + { + "schema_version": SCHEMA_VERSION, + "language": "en", + "schema_fingerprint": schema_fingerprint(), + }, + ), ) assert needs_rebuild(index_dir) is False diff --git a/src/documents/tests/search/test_schema_fingerprint.py b/src/documents/tests/search/test_schema_fingerprint.py new file mode 100644 index 000000000..253861247 --- /dev/null +++ b/src/documents/tests/search/test_schema_fingerprint.py @@ -0,0 +1,492 @@ +"""The schema fingerprint stamped into .index_settings.json. + +tantivy compares schemas by *ordered* field list, and `tantivy.Index(schema, +path=...)` (what every write path does) raises on any difference. SCHEMA_VERSION +is the manual guard against that, but build_schema() is edited for *parser* +reasons - adding an alias, flipping fast=True, adding a subpath - by people not +thinking about the on-disk index, and forgetting the bump is exactly how this +branch's bug happened. + +The fingerprint is the automatic guard: it hashes the field descriptor list that +build_schema() itself iterates, so any change to a field's name, kind, options +or *position* forces a rebuild on its own. +""" + +from __future__ import annotations + +import hashlib +import json +from typing import TYPE_CHECKING + +import pytest +import tantivy + +from documents.search import _schema +from documents.search._schema import SCHEMA_VERSION +from documents.search._schema import FieldDescriptor +from documents.search._schema import _write_sentinels +from documents.search._schema import build_schema +from documents.search._schema import field_descriptors +from documents.search._schema import needs_rebuild +from documents.search._schema import schema_fingerprint + +if TYPE_CHECKING: + from pathlib import Path + + from pytest_django.fixtures import SettingsWrapper + +pytestmark = pytest.mark.search + +# The on-disk field layout of a v2 index, pinned as data. Any edit here is an +# index-format change: it must come with a rebuild, which the fingerprint now +# forces automatically. Reproduced from build_schema()'s output as it stood +# before the descriptor refactor, so it also pins that the refactor changed +# nothing. +PINNED_DESCRIPTORS: tuple[FieldDescriptor, ...] = ( + FieldDescriptor("id", "u64", stored=True, indexed=True, fast=True, tokenizer=None), + FieldDescriptor( + "title", + "text", + stored=True, + indexed=True, + fast=False, + tokenizer="paperless_text", + ), + FieldDescriptor( + "content", + "text", + stored=True, + indexed=True, + fast=False, + tokenizer="paperless_text", + ), + FieldDescriptor( + "correspondent", + "text", + stored=True, + indexed=True, + fast=False, + tokenizer="paperless_text", + ), + FieldDescriptor( + "document_type", + "text", + stored=True, + indexed=True, + fast=False, + tokenizer="paperless_text", + ), + FieldDescriptor( + "storage_path", + "text", + stored=True, + indexed=True, + fast=False, + tokenizer="paperless_text", + ), + FieldDescriptor( + "original_filename", + "text", + stored=True, + indexed=True, + fast=False, + tokenizer="paperless_text", + ), + FieldDescriptor( + "tag", + "text", + stored=True, + indexed=True, + fast=False, + tokenizer="paperless_text", + ), + FieldDescriptor( + "checksum", + "text", + stored=True, + indexed=True, + fast=False, + tokenizer="raw", + ), + FieldDescriptor("asn", "u64", stored=True, indexed=True, fast=True, tokenizer=None), + FieldDescriptor( + "page_count", + "u64", + stored=True, + indexed=True, + fast=True, + tokenizer=None, + ), + FieldDescriptor( + "num_notes", + "u64", + stored=True, + indexed=True, + fast=True, + tokenizer=None, + ), + FieldDescriptor( + "created", + "date", + stored=True, + indexed=True, + fast=True, + tokenizer=None, + ), + FieldDescriptor( + "modified", + "date", + stored=True, + indexed=True, + fast=True, + tokenizer=None, + ), + FieldDescriptor( + "added", + "date", + stored=True, + indexed=True, + fast=True, + tokenizer=None, + ), + FieldDescriptor( + "notes", + "json", + stored=True, + indexed=True, + fast=False, + tokenizer="paperless_text", + ), + FieldDescriptor( + "notes_text", + "text", + stored=True, + indexed=True, + fast=False, + tokenizer="paperless_text", + ), + FieldDescriptor( + "custom_fields", + "json", + stored=True, + indexed=True, + fast=False, + tokenizer="paperless_text", + ), + FieldDescriptor( + "title_sort", + "text", + stored=False, + indexed=True, + fast=True, + tokenizer="simple_analyzer", + ), + FieldDescriptor( + "correspondent_sort", + "text", + stored=False, + indexed=True, + fast=True, + tokenizer="simple_analyzer", + ), + FieldDescriptor( + "type_sort", + "text", + stored=False, + indexed=True, + fast=True, + tokenizer="simple_analyzer", + ), + FieldDescriptor( + "bigram_content", + "text", + stored=False, + indexed=True, + fast=False, + tokenizer="bigram_analyzer", + ), + FieldDescriptor( + "bigram_title", + "text", + stored=False, + indexed=True, + fast=False, + tokenizer="bigram_analyzer", + ), + FieldDescriptor( + "bigram_correspondent", + "text", + stored=False, + indexed=True, + fast=False, + tokenizer="bigram_analyzer", + ), + FieldDescriptor( + "bigram_document_type", + "text", + stored=False, + indexed=True, + fast=False, + tokenizer="bigram_analyzer", + ), + FieldDescriptor( + "bigram_tag", + "text", + stored=False, + indexed=True, + fast=False, + tokenizer="bigram_analyzer", + ), + FieldDescriptor( + "simple_title", + "text", + stored=False, + indexed=True, + fast=False, + tokenizer="simple_search_analyzer", + ), + FieldDescriptor( + "simple_content", + "text", + stored=False, + indexed=True, + fast=False, + tokenizer="simple_search_analyzer", + ), + FieldDescriptor( + "autocomplete_word", + "text", + stored=False, + indexed=True, + fast=False, + tokenizer="raw", + ), + FieldDescriptor( + "owner_id", + "u64", + stored=False, + indexed=True, + fast=True, + tokenizer=None, + ), + FieldDescriptor( + "viewer_id", + "u64", + stored=False, + indexed=True, + fast=True, + tokenizer=None, + ), + FieldDescriptor( + "viewer_group_id", + "u64", + stored=False, + indexed=True, + fast=True, + tokenizer=None, + ), +) + + +def _schema_fields(schema: tantivy.Schema) -> list[dict]: + """The tantivy-level field list, in declaration order. + + tantivy-py 0.26 exposes no public introspection API on Schema, so + __reduce__() (its pickling hook) is the only way to recover the field list. + It is used here, in a test, precisely because it is the representation the + persisted fingerprint must NOT depend on. + """ + return schema.__reduce__()[1][0]["inner"] + + +def _sentinels(index_dir: Path, **overrides: object) -> None: + data = { + "schema_version": SCHEMA_VERSION, + "language": None, + "schema_fingerprint": schema_fingerprint(), + } + data.update(overrides) + (index_dir / ".index_settings.json").write_text(json.dumps(data)) + + +class TestDescriptorsDescribeTheBuiltSchema: + def test_descriptors_match_the_pinned_field_layout(self) -> None: + assert tuple(field_descriptors()) == PINNED_DESCRIPTORS + + def test_built_schema_matches_the_descriptors(self) -> None: + """The descriptors are not a parallel description - they are the input. + + Reading the built schema back proves the loop honours every option, so + a descriptor edit cannot claim a shape the SchemaBuilder did not build. + """ + kinds = {"text": "text", "json": "json_object", "u64": "u64", "date": "date"} + built = [ + ( + field["name"], + field["type"], + field["options"]["stored"], + bool(field["options"].get("fast")), + (field["options"].get("indexing") or {}).get("tokenizer"), + ) + for field in _schema_fields(build_schema()) + ] + expected = [ + ( + descriptor.name, + kinds[descriptor.kind], + descriptor.stored, + descriptor.fast, + descriptor.tokenizer, + ) + for descriptor in field_descriptors() + ] + assert built == expected + + +class TestFingerprintSensitivity: + def test_a_field_option_change_moves_the_fingerprint( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + before = schema_fingerprint() + changed = field_descriptors() + changed[1] = changed[1]._replace(fast=True) + monkeypatch.setattr(_schema, "field_descriptors", lambda: changed) + + assert schema_fingerprint() != before + + def test_reordering_alone_moves_the_fingerprint( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """The original bug: same fields, different declaration order. + + A set- or dict-based fingerprint would be blind to this, and tantivy + would reject every write against the existing index. + """ + before = schema_fingerprint() + swapped = field_descriptors() + swapped[1], swapped[2] = swapped[2], swapped[1] + monkeypatch.setattr(_schema, "field_descriptors", lambda: swapped) + + assert schema_fingerprint() != before + + def test_repeated_calls_agree(self) -> None: + assert schema_fingerprint() == schema_fingerprint() + + +class TestFingerprintIsIndependentOfTantivy: + def test_a_tantivy_option_key_addition_would_not_move_it(self) -> None: + """A tantivy-py upgrade must not force a global reindex. + + Hashing schema.__reduce__() would do exactly that: the simulated new + option key below changes that payload for every user with no schema + change at all. + """ + fields = _schema_fields(build_schema()) + upgraded = [ + {**field, "options": {**field["options"], "coerce": True}} + for field in fields + ] + assert _hash(upgraded) != _hash(fields) + assert schema_fingerprint() == _fingerprint_of(field_descriptors()) + + def test_fingerprint_never_touches_the_schema_builder( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + before = schema_fingerprint() + + class _RemovedSchemaBuilder: + def __init__(self) -> None: + raise AssertionError("tantivy.SchemaBuilder was consulted") + + monkeypatch.setattr(tantivy, "SchemaBuilder", _RemovedSchemaBuilder) + with pytest.raises(AssertionError): + build_schema() + + assert schema_fingerprint() == before + + +def _hash(payload: object) -> str: + return hashlib.blake2b(json.dumps(payload).encode()).hexdigest() + + +def _fingerprint_of(descriptors: list[FieldDescriptor]) -> str: + return _hash([list(descriptor) for descriptor in descriptors]) + + +class TestNeedsRebuildOnFingerprint: + def test_matching_fingerprint_does_not_rebuild( + self, + index_dir: Path, + settings: SettingsWrapper, + ) -> None: + settings.SEARCH_LANGUAGE = None + _sentinels(index_dir) + + assert needs_rebuild(index_dir) is False + + def test_stale_fingerprint_rebuilds_despite_a_matching_version( + self, + index_dir: Path, + settings: SettingsWrapper, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """The failure this task exists to prevent: schema edited, version not + bumped. Without the fingerprint check, `reindex --if-needed` reports the + index up to date and every write then raises.""" + settings.SEARCH_LANGUAGE = None + _sentinels(index_dir) + extended = [ + *field_descriptors(), + FieldDescriptor( + "new_field", + "u64", + stored=False, + indexed=True, + fast=True, + tokenizer=None, + ), + ] + monkeypatch.setattr(_schema, "field_descriptors", lambda: extended) + + assert needs_rebuild(index_dir) is True + + def test_reordered_schema_rebuilds( + self, + index_dir: Path, + settings: SettingsWrapper, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + settings.SEARCH_LANGUAGE = None + _sentinels(index_dir) + reordered = field_descriptors() + reordered[1], reordered[2] = reordered[2], reordered[1] + monkeypatch.setattr(_schema, "field_descriptors", lambda: reordered) + + assert needs_rebuild(index_dir) is True + + def test_missing_fingerprint_rebuilds( + self, + index_dir: Path, + settings: SettingsWrapper, + ) -> None: + """No seeding: an index whose schema shape nobody recorded is rebuilt + rather than trusted.""" + settings.SEARCH_LANGUAGE = None + (index_dir / ".index_settings.json").write_text( + json.dumps({"schema_version": SCHEMA_VERSION, "language": None}), + ) + + assert needs_rebuild(index_dir) is True + + def test_written_sentinels_satisfy_the_check( + self, + index_dir: Path, + settings: SettingsWrapper, + ) -> None: + settings.SEARCH_LANGUAGE = "en" + _write_sentinels(index_dir) + + assert needs_rebuild(index_dir) is False