test(search): move schema fingerprint/version tests from PR2

Both files exercise documents.search._schema exclusively (build_schema,
schema_fingerprint, needs_rebuild, open_or_rebuild_index) with no
dependency on whoosh-compat query routing, so they belong with the
schema/field-registry work rather than PR2's query rewrite.
This commit is contained in:
stumpylog
2026-08-28 09:46:57 -07:00
parent 2f79549f92
commit 03e05e0804
2 changed files with 656 additions and 0 deletions
@@ -0,0 +1,492 @@
"""The schema fingerprint stamped into .index_settings.json.
tantivy compares schemas by *ordered* field list, and `tantivy.Index(schema,
path=...)` (what every write path does) raises on any difference. SCHEMA_VERSION
is the manual guard against that, but build_schema() is edited for *parser*
reasons - adding an alias, flipping fast=True, adding a subpath - by people not
thinking about the on-disk index, and forgetting the bump is exactly how this
branch's bug happened.
The fingerprint is the automatic guard: it hashes the field descriptor list that
build_schema() itself iterates, so any change to a field's name, kind, options
or *position* forces a rebuild on its own.
"""
from __future__ import annotations
import hashlib
import json
from typing import TYPE_CHECKING
import pytest
import tantivy
from documents.search import _schema
from documents.search._schema import SCHEMA_VERSION
from documents.search._schema import FieldDescriptor
from documents.search._schema import _write_sentinels
from documents.search._schema import build_schema
from documents.search._schema import field_descriptors
from documents.search._schema import needs_rebuild
from documents.search._schema import schema_fingerprint
if TYPE_CHECKING:
from pathlib import Path
from pytest_django.fixtures import SettingsWrapper
pytestmark = pytest.mark.search
# The on-disk field layout of a v2 index, pinned as data. Any edit here is an
# index-format change: it must come with a rebuild, which the fingerprint now
# forces automatically. Reproduced from build_schema()'s output as it stood
# before the descriptor refactor, so it also pins that the refactor changed
# nothing.
PINNED_DESCRIPTORS: tuple[FieldDescriptor, ...] = (
FieldDescriptor("id", "u64", stored=True, indexed=True, fast=True, tokenizer=None),
FieldDescriptor(
"title",
"text",
stored=True,
indexed=True,
fast=False,
tokenizer="paperless_text",
),
FieldDescriptor(
"content",
"text",
stored=True,
indexed=True,
fast=False,
tokenizer="paperless_text",
),
FieldDescriptor(
"correspondent",
"text",
stored=True,
indexed=True,
fast=False,
tokenizer="paperless_text",
),
FieldDescriptor(
"document_type",
"text",
stored=True,
indexed=True,
fast=False,
tokenizer="paperless_text",
),
FieldDescriptor(
"storage_path",
"text",
stored=True,
indexed=True,
fast=False,
tokenizer="paperless_text",
),
FieldDescriptor(
"original_filename",
"text",
stored=True,
indexed=True,
fast=False,
tokenizer="paperless_text",
),
FieldDescriptor(
"tag",
"text",
stored=True,
indexed=True,
fast=False,
tokenizer="paperless_text",
),
FieldDescriptor(
"checksum",
"text",
stored=True,
indexed=True,
fast=False,
tokenizer="raw",
),
FieldDescriptor("asn", "u64", stored=True, indexed=True, fast=True, tokenizer=None),
FieldDescriptor(
"page_count",
"u64",
stored=True,
indexed=True,
fast=True,
tokenizer=None,
),
FieldDescriptor(
"num_notes",
"u64",
stored=True,
indexed=True,
fast=True,
tokenizer=None,
),
FieldDescriptor(
"created",
"date",
stored=True,
indexed=True,
fast=True,
tokenizer=None,
),
FieldDescriptor(
"modified",
"date",
stored=True,
indexed=True,
fast=True,
tokenizer=None,
),
FieldDescriptor(
"added",
"date",
stored=True,
indexed=True,
fast=True,
tokenizer=None,
),
FieldDescriptor(
"notes",
"json",
stored=True,
indexed=True,
fast=False,
tokenizer="paperless_text",
),
FieldDescriptor(
"notes_text",
"text",
stored=True,
indexed=True,
fast=False,
tokenizer="paperless_text",
),
FieldDescriptor(
"custom_fields",
"json",
stored=True,
indexed=True,
fast=False,
tokenizer="paperless_text",
),
FieldDescriptor(
"title_sort",
"text",
stored=False,
indexed=True,
fast=True,
tokenizer="simple_analyzer",
),
FieldDescriptor(
"correspondent_sort",
"text",
stored=False,
indexed=True,
fast=True,
tokenizer="simple_analyzer",
),
FieldDescriptor(
"type_sort",
"text",
stored=False,
indexed=True,
fast=True,
tokenizer="simple_analyzer",
),
FieldDescriptor(
"bigram_content",
"text",
stored=False,
indexed=True,
fast=False,
tokenizer="bigram_analyzer",
),
FieldDescriptor(
"bigram_title",
"text",
stored=False,
indexed=True,
fast=False,
tokenizer="bigram_analyzer",
),
FieldDescriptor(
"bigram_correspondent",
"text",
stored=False,
indexed=True,
fast=False,
tokenizer="bigram_analyzer",
),
FieldDescriptor(
"bigram_document_type",
"text",
stored=False,
indexed=True,
fast=False,
tokenizer="bigram_analyzer",
),
FieldDescriptor(
"bigram_tag",
"text",
stored=False,
indexed=True,
fast=False,
tokenizer="bigram_analyzer",
),
FieldDescriptor(
"simple_title",
"text",
stored=False,
indexed=True,
fast=False,
tokenizer="simple_search_analyzer",
),
FieldDescriptor(
"simple_content",
"text",
stored=False,
indexed=True,
fast=False,
tokenizer="simple_search_analyzer",
),
FieldDescriptor(
"autocomplete_word",
"text",
stored=False,
indexed=True,
fast=False,
tokenizer="raw",
),
FieldDescriptor(
"owner_id",
"u64",
stored=False,
indexed=True,
fast=True,
tokenizer=None,
),
FieldDescriptor(
"viewer_id",
"u64",
stored=False,
indexed=True,
fast=True,
tokenizer=None,
),
FieldDescriptor(
"viewer_group_id",
"u64",
stored=False,
indexed=True,
fast=True,
tokenizer=None,
),
)
def _schema_fields(schema: tantivy.Schema) -> list[dict]:
"""The tantivy-level field list, in declaration order.
tantivy-py 0.26 exposes no public introspection API on Schema, so
__reduce__() (its pickling hook) is the only way to recover the field list.
It is used here, in a test, precisely because it is the representation the
persisted fingerprint must NOT depend on.
"""
return schema.__reduce__()[1][0]["inner"]
def _sentinels(index_dir: Path, **overrides: object) -> None:
data = {
"schema_version": SCHEMA_VERSION,
"language": None,
"schema_fingerprint": schema_fingerprint(),
}
data.update(overrides)
(index_dir / ".index_settings.json").write_text(json.dumps(data))
class TestDescriptorsDescribeTheBuiltSchema:
def test_descriptors_match_the_pinned_field_layout(self) -> None:
assert tuple(field_descriptors()) == PINNED_DESCRIPTORS
def test_built_schema_matches_the_descriptors(self) -> None:
"""The descriptors are not a parallel description - they are the input.
Reading the built schema back proves the loop honours every option, so
a descriptor edit cannot claim a shape the SchemaBuilder did not build.
"""
kinds = {"text": "text", "json": "json_object", "u64": "u64", "date": "date"}
built = [
(
field["name"],
field["type"],
field["options"]["stored"],
bool(field["options"].get("fast")),
(field["options"].get("indexing") or {}).get("tokenizer"),
)
for field in _schema_fields(build_schema())
]
expected = [
(
descriptor.name,
kinds[descriptor.kind],
descriptor.stored,
descriptor.fast,
descriptor.tokenizer,
)
for descriptor in field_descriptors()
]
assert built == expected
class TestFingerprintSensitivity:
def test_a_field_option_change_moves_the_fingerprint(
self,
monkeypatch: pytest.MonkeyPatch,
) -> None:
before = schema_fingerprint()
changed = field_descriptors()
changed[1] = changed[1]._replace(fast=True)
monkeypatch.setattr(_schema, "field_descriptors", lambda: changed)
assert schema_fingerprint() != before
def test_reordering_alone_moves_the_fingerprint(
self,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""The original bug: same fields, different declaration order.
A set- or dict-based fingerprint would be blind to this, and tantivy
would reject every write against the existing index.
"""
before = schema_fingerprint()
swapped = field_descriptors()
swapped[1], swapped[2] = swapped[2], swapped[1]
monkeypatch.setattr(_schema, "field_descriptors", lambda: swapped)
assert schema_fingerprint() != before
def test_repeated_calls_agree(self) -> None:
assert schema_fingerprint() == schema_fingerprint()
class TestFingerprintIsIndependentOfTantivy:
def test_a_tantivy_option_key_addition_would_not_move_it(self) -> None:
"""A tantivy-py upgrade must not force a global reindex.
Hashing schema.__reduce__() would do exactly that: the simulated new
option key below changes that payload for every user with no schema
change at all.
"""
fields = _schema_fields(build_schema())
upgraded = [
{**field, "options": {**field["options"], "coerce": True}}
for field in fields
]
assert _hash(upgraded) != _hash(fields)
assert schema_fingerprint() == _fingerprint_of(field_descriptors())
def test_fingerprint_never_touches_the_schema_builder(
self,
monkeypatch: pytest.MonkeyPatch,
) -> None:
before = schema_fingerprint()
class _RemovedSchemaBuilder:
def __init__(self) -> None:
raise AssertionError("tantivy.SchemaBuilder was consulted")
monkeypatch.setattr(tantivy, "SchemaBuilder", _RemovedSchemaBuilder)
with pytest.raises(AssertionError):
build_schema()
assert schema_fingerprint() == before
def _hash(payload: object) -> str:
return hashlib.blake2b(json.dumps(payload).encode()).hexdigest()
def _fingerprint_of(descriptors: list[FieldDescriptor]) -> str:
return _hash([list(descriptor) for descriptor in descriptors])
class TestNeedsRebuildOnFingerprint:
def test_matching_fingerprint_does_not_rebuild(
self,
index_dir: Path,
settings: SettingsWrapper,
) -> None:
settings.SEARCH_LANGUAGE = None
_sentinels(index_dir)
assert needs_rebuild(index_dir) is False
def test_stale_fingerprint_rebuilds_despite_a_matching_version(
self,
index_dir: Path,
settings: SettingsWrapper,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""The failure this task exists to prevent: schema edited, version not
bumped. Without the fingerprint check, `reindex --if-needed` reports the
index up to date and every write then raises."""
settings.SEARCH_LANGUAGE = None
_sentinels(index_dir)
extended = [
*field_descriptors(),
FieldDescriptor(
"new_field",
"u64",
stored=False,
indexed=True,
fast=True,
tokenizer=None,
),
]
monkeypatch.setattr(_schema, "field_descriptors", lambda: extended)
assert needs_rebuild(index_dir) is True
def test_reordered_schema_rebuilds(
self,
index_dir: Path,
settings: SettingsWrapper,
monkeypatch: pytest.MonkeyPatch,
) -> None:
settings.SEARCH_LANGUAGE = None
_sentinels(index_dir)
reordered = field_descriptors()
reordered[1], reordered[2] = reordered[2], reordered[1]
monkeypatch.setattr(_schema, "field_descriptors", lambda: reordered)
assert needs_rebuild(index_dir) is True
def test_missing_fingerprint_rebuilds(
self,
index_dir: Path,
settings: SettingsWrapper,
) -> None:
"""No seeding: an index whose schema shape nobody recorded is rebuilt
rather than trusted."""
settings.SEARCH_LANGUAGE = None
(index_dir / ".index_settings.json").write_text(
json.dumps({"schema_version": SCHEMA_VERSION, "language": None}),
)
assert needs_rebuild(index_dir) is True
def test_written_sentinels_satisfy_the_check(
self,
index_dir: Path,
settings: SettingsWrapper,
) -> None:
settings.SEARCH_LANGUAGE = "en"
_write_sentinels(index_dir)
assert needs_rebuild(index_dir) is False
@@ -0,0 +1,164 @@
"""SCHEMA_VERSION must change whenever build_schema()'s field list or order does.
tantivy compares schemas by *ordered* field list. ``Index.open()`` loads the
schema from the index's own ``meta.json``, so reads against an index built by an
older release keep working after a field reorder. Writes do not:
``WriteBatch.__enter__`` calls ``tantivy.Index(build_schema(), path=...)``, an
open-or-create that raises ``ValueError`` on any schema difference. Nothing
catches that ValueError, so consumption, index_document and bulk edit all
hard-fail while ``/api/status/`` still reports the index healthy.
The only thing that saves such an install is ``needs_rebuild()`` noticing the
version stamped in ``.index_settings.json`` is stale.
"""
from __future__ import annotations
import json
from typing import TYPE_CHECKING
import pytest
import tantivy
from django.conf import settings as django_settings
from documents.search._schema import build_schema
from documents.search._schema import needs_rebuild
from documents.search._schema import open_or_rebuild_index
if TYPE_CHECKING:
from pathlib import Path
pytestmark = [pytest.mark.search]
RELEASED_V1_SCHEMA_VERSION = 1
def _build_released_v1_schema() -> tantivy.Schema:
"""Frozen copy of build_schema() as shipped in v3.0.x (schema version 1).
Deliberately duplicated rather than imported: it must keep describing the
on-disk layout of already-deployed indexes even as build_schema() evolves.
"""
sb = tantivy.SchemaBuilder()
sb.add_unsigned_field("id", stored=True, indexed=True, fast=True)
sb.add_text_field("checksum", stored=True, tokenizer_name="raw")
for field in (
"title",
"correspondent",
"document_type",
"storage_path",
"original_filename",
"content",
):
sb.add_text_field(field, stored=True, tokenizer_name="paperless_text")
for field in ("title_sort", "correspondent_sort", "type_sort"):
sb.add_text_field(
field,
stored=False,
tokenizer_name="simple_analyzer",
fast=True,
)
for field in (
"bigram_content",
"bigram_title",
"bigram_correspondent",
"bigram_document_type",
"bigram_tag",
):
sb.add_text_field(field, stored=False, tokenizer_name="bigram_analyzer")
for field in ("simple_title", "simple_content"):
sb.add_text_field(field, stored=False, tokenizer_name="simple_search_analyzer")
sb.add_text_field("autocomplete_word", stored=False, tokenizer_name="raw")
sb.add_text_field("tag", stored=True, tokenizer_name="paperless_text")
sb.add_json_field("notes", stored=True, tokenizer_name="paperless_text")
sb.add_text_field("notes_text", stored=True, tokenizer_name="paperless_text")
sb.add_json_field("custom_fields", stored=True, tokenizer_name="paperless_text")
for field in (
"correspondent_id",
"document_type_id",
"storage_path_id",
"tag_id",
"owner_id",
"viewer_id",
"viewer_group_id",
):
sb.add_unsigned_field(field, stored=False, indexed=True, fast=True)
for field in ("created", "modified", "added"):
sb.add_date_field(field, stored=True, indexed=True, fast=True)
for field in ("asn", "page_count", "num_notes"):
sb.add_unsigned_field(field, stored=True, indexed=True, fast=True)
return sb.build()
@pytest.fixture
def released_v1_index(tmp_path: Path) -> Path:
"""An index directory as a v3.0.x install would leave it on disk."""
index_dir = tmp_path / "index"
index_dir.mkdir()
tantivy.Index(_build_released_v1_schema(), path=str(index_dir))
(index_dir / ".index_settings.json").write_text(
json.dumps(
{
"schema_version": RELEASED_V1_SCHEMA_VERSION,
"language": django_settings.SEARCH_LANGUAGE,
},
),
)
return index_dir
class TestUpgradeFromReleasedV1Index:
def test_released_v1_index_is_flagged_for_rebuild(
self,
released_v1_index: Path,
) -> None:
"""The current schema differs from v1's, so the sentinel must be stale.
If this fails, `document_index reindex --if-needed` prints "Search index
is up to date" and skips, leaving the mismatched index in place.
"""
assert needs_rebuild(released_v1_index) is True
def test_v1_index_rejects_writes_against_the_current_schema(
self,
released_v1_index: Path,
) -> None:
"""The failure mode the version bump exists to prevent.
This is exactly what WriteBatch.__enter__ does on every index write.
"""
with pytest.raises(ValueError, match="schema does not match"):
tantivy.Index(build_schema(), path=str(released_v1_index))
def test_opening_a_v1_index_leaves_it_writable(
self,
released_v1_index: Path,
) -> None:
"""End to end: open_or_rebuild_index must hand back an index that the
write path can reopen. Before the version bump, needs_rebuild() returned
False here, the stale directory survived untouched, and every subsequent
write raised the ValueError above."""
open_or_rebuild_index(released_v1_index)
tantivy.Index(build_schema(), path=str(released_v1_index))
def test_rebuilt_index_is_not_rebuilt_again(
self,
released_v1_index: Path,
) -> None:
"""The rebuild must stamp the version it actually wrote, otherwise every
startup wipes and reindexes the whole corpus."""
open_or_rebuild_index(released_v1_index)
assert needs_rebuild(released_v1_index) is False