mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-08-25 20:23:18 +00:00
feat(search): detect schema shape changes with a schema fingerprint
build_schema() was half table-driven and half hardcoded, so editing it for parser reasons could change the on-disk field list without anyone bumping SCHEMA_VERSION. tantivy compares schemas by ordered field list, so such an edit leaves reads working while every write raises. Complete the table: build_schema() now iterates an explicit list of field descriptors covering id, the PUBLIC_FIELDS expansion, the sort shadow, bigram, simple_* and autocomplete fields and the permission columns. The same list is hashed into a schema_fingerprint() that is stamped into .index_settings.json and compared by needs_rebuild() as a fourth check alongside the existing schema version and language checks. The fingerprint is computed from paperless' own descriptors rather than tantivy's schema representation, so a tantivy-py option-key rename or addition cannot silently force a global reindex. The emitted schema is byte-identical to the previous one. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
stumpylog
co-authored by
Claude Opus 5
Claude Opus 5
parent
9554390a08
commit
44886aabd1
+226
-76
@@ -1,10 +1,12 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import shutil
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import Final
|
||||
from typing import NamedTuple
|
||||
from typing import cast
|
||||
|
||||
import tantivy
|
||||
@@ -26,6 +28,196 @@ logger = logging.getLogger("paperless.search")
|
||||
SCHEMA_VERSION: Final[int] = 2
|
||||
|
||||
|
||||
class FieldDescriptor(NamedTuple):
|
||||
"""One tantivy field, in declaration order.
|
||||
|
||||
The descriptor vocabulary is paperless', not tantivy-py's: it is both the
|
||||
input to the SchemaBuilder and the input to schema_fingerprint(), so the
|
||||
persisted fingerprint cannot move under a tantivy-py upgrade.
|
||||
"""
|
||||
|
||||
name: str
|
||||
kind: str
|
||||
stored: bool
|
||||
indexed: bool
|
||||
fast: bool
|
||||
tokenizer: str | None
|
||||
|
||||
|
||||
def _public_field_descriptors() -> list[FieldDescriptor]:
|
||||
"""Descriptors for the query-visible fields declared in PUBLIC_FIELDS."""
|
||||
descriptors: list[FieldDescriptor] = []
|
||||
for field in PUBLIC_FIELDS:
|
||||
if field.kind is FieldKind.TEXT:
|
||||
descriptors.append(
|
||||
FieldDescriptor(
|
||||
field.name,
|
||||
"text",
|
||||
stored=True,
|
||||
indexed=True,
|
||||
fast=False,
|
||||
tokenizer="paperless_text",
|
||||
),
|
||||
)
|
||||
elif field.kind is FieldKind.KEYWORD:
|
||||
descriptors.append(
|
||||
FieldDescriptor(
|
||||
field.name,
|
||||
"text",
|
||||
stored=True,
|
||||
indexed=True,
|
||||
fast=False,
|
||||
tokenizer="raw",
|
||||
),
|
||||
)
|
||||
elif field.kind is FieldKind.U64:
|
||||
descriptors.append(
|
||||
FieldDescriptor(
|
||||
field.name,
|
||||
"u64",
|
||||
stored=True,
|
||||
indexed=True,
|
||||
fast=field.fast,
|
||||
tokenizer=None,
|
||||
),
|
||||
)
|
||||
elif field.kind in (FieldKind.DATE, FieldKind.DATETIME):
|
||||
descriptors.append(
|
||||
FieldDescriptor(
|
||||
field.name,
|
||||
"date",
|
||||
stored=True,
|
||||
indexed=True,
|
||||
fast=field.fast,
|
||||
tokenizer=None,
|
||||
),
|
||||
)
|
||||
elif field.kind is FieldKind.JSON:
|
||||
descriptors.append(
|
||||
FieldDescriptor(
|
||||
field.name,
|
||||
"json",
|
||||
stored=True,
|
||||
indexed=True,
|
||||
fast=False,
|
||||
tokenizer="paperless_text",
|
||||
),
|
||||
)
|
||||
if field.name == "notes":
|
||||
# Plain-text companion for snippet generation — tantivy's
|
||||
# SnippetGenerator does not support JSON fields. Schema-only,
|
||||
# no query-syntax meaning, not in PUBLIC_FIELDS.
|
||||
descriptors.append(
|
||||
FieldDescriptor(
|
||||
"notes_text",
|
||||
"text",
|
||||
stored=True,
|
||||
indexed=True,
|
||||
fast=False,
|
||||
tokenizer="paperless_text",
|
||||
),
|
||||
)
|
||||
return descriptors
|
||||
|
||||
|
||||
def field_descriptors() -> list[FieldDescriptor]:
|
||||
"""Every field of the document index, in the order tantivy declares them.
|
||||
|
||||
tantivy compares schemas by *ordered* field list, so the order here is
|
||||
part of the on-disk contract: schema_fingerprint() hashes it and
|
||||
needs_rebuild() acts on the result.
|
||||
"""
|
||||
return [
|
||||
FieldDescriptor(
|
||||
"id",
|
||||
"u64",
|
||||
stored=True,
|
||||
indexed=True,
|
||||
fast=True,
|
||||
tokenizer=None,
|
||||
),
|
||||
*_public_field_descriptors(),
|
||||
# Shadow sort fields - fast, not stored
|
||||
*(
|
||||
FieldDescriptor(
|
||||
name,
|
||||
"text",
|
||||
stored=False,
|
||||
indexed=True,
|
||||
fast=True,
|
||||
tokenizer="simple_analyzer",
|
||||
)
|
||||
for name in ("title_sort", "correspondent_sort", "type_sort")
|
||||
),
|
||||
# CJK support - not stored, indexed only
|
||||
*(
|
||||
FieldDescriptor(
|
||||
name,
|
||||
"text",
|
||||
stored=False,
|
||||
indexed=True,
|
||||
fast=False,
|
||||
tokenizer="bigram_analyzer",
|
||||
)
|
||||
for name in (
|
||||
"bigram_content",
|
||||
"bigram_title",
|
||||
"bigram_correspondent",
|
||||
"bigram_document_type",
|
||||
"bigram_tag",
|
||||
)
|
||||
),
|
||||
# Simple substring search support for title/content - not stored,
|
||||
# indexed only
|
||||
*(
|
||||
FieldDescriptor(
|
||||
name,
|
||||
"text",
|
||||
stored=False,
|
||||
indexed=True,
|
||||
fast=False,
|
||||
tokenizer="simple_search_analyzer",
|
||||
)
|
||||
for name in ("simple_title", "simple_content")
|
||||
),
|
||||
# Autocomplete prefix scan via terms_with_prefix, which walks the
|
||||
# field's term dictionary - so the field must be indexed (term dict),
|
||||
# not stored. The stored value is never read back, so storing it only
|
||||
# wastes space.
|
||||
FieldDescriptor(
|
||||
"autocomplete_word",
|
||||
"text",
|
||||
stored=False,
|
||||
indexed=True,
|
||||
fast=False,
|
||||
tokenizer="raw",
|
||||
),
|
||||
# Permission filter columns, read by build_permission_filter.
|
||||
*(
|
||||
FieldDescriptor(
|
||||
name,
|
||||
"u64",
|
||||
stored=False,
|
||||
indexed=True,
|
||||
fast=True,
|
||||
tokenizer=None,
|
||||
)
|
||||
for name in ("owner_id", "viewer_id", "viewer_group_id")
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def schema_fingerprint() -> str:
|
||||
"""Hash of the field descriptors, stamped into .index_settings.json.
|
||||
|
||||
Changes whenever a field is added, removed, retyped, re-optioned or
|
||||
reordered, so an index built from a different schema shape is detected
|
||||
even when SCHEMA_VERSION was not bumped.
|
||||
"""
|
||||
payload = json.dumps([list(descriptor) for descriptor in field_descriptors()])
|
||||
return hashlib.blake2b(payload.encode()).hexdigest()
|
||||
|
||||
|
||||
def build_schema() -> tantivy.Schema:
|
||||
"""
|
||||
Build the Tantivy schema for the paperless document index.
|
||||
@@ -39,83 +231,37 @@ def build_schema() -> tantivy.Schema:
|
||||
"""
|
||||
sb = tantivy.SchemaBuilder()
|
||||
|
||||
sb.add_unsigned_field("id", stored=True, indexed=True, fast=True)
|
||||
|
||||
for field in PUBLIC_FIELDS:
|
||||
if field.kind is FieldKind.TEXT:
|
||||
sb.add_text_field(field.name, stored=True, tokenizer_name="paperless_text")
|
||||
elif field.kind is FieldKind.KEYWORD:
|
||||
sb.add_text_field(field.name, stored=True, tokenizer_name="raw")
|
||||
elif field.kind is FieldKind.U64:
|
||||
for descriptor in field_descriptors():
|
||||
if descriptor.kind == "text":
|
||||
sb.add_text_field(
|
||||
descriptor.name,
|
||||
stored=descriptor.stored,
|
||||
fast=descriptor.fast,
|
||||
tokenizer_name=cast("str", descriptor.tokenizer),
|
||||
)
|
||||
elif descriptor.kind == "json":
|
||||
sb.add_json_field(
|
||||
descriptor.name,
|
||||
stored=descriptor.stored,
|
||||
fast=descriptor.fast,
|
||||
tokenizer_name=cast("str", descriptor.tokenizer),
|
||||
)
|
||||
elif descriptor.kind == "u64":
|
||||
sb.add_unsigned_field(
|
||||
field.name,
|
||||
stored=True,
|
||||
indexed=True,
|
||||
fast=field.fast,
|
||||
descriptor.name,
|
||||
stored=descriptor.stored,
|
||||
indexed=descriptor.indexed,
|
||||
fast=descriptor.fast,
|
||||
)
|
||||
elif field.kind in (FieldKind.DATE, FieldKind.DATETIME):
|
||||
elif descriptor.kind == "date":
|
||||
sb.add_date_field(
|
||||
field.name,
|
||||
stored=True,
|
||||
indexed=True,
|
||||
fast=field.fast,
|
||||
descriptor.name,
|
||||
stored=descriptor.stored,
|
||||
indexed=descriptor.indexed,
|
||||
fast=descriptor.fast,
|
||||
)
|
||||
elif field.kind is FieldKind.JSON:
|
||||
sb.add_json_field(field.name, stored=True, tokenizer_name="paperless_text")
|
||||
if field.name == "notes":
|
||||
# Plain-text companion for snippet generation — tantivy's
|
||||
# SnippetGenerator does not support JSON fields. Schema-only,
|
||||
# no query-syntax meaning, not in PUBLIC_FIELDS.
|
||||
sb.add_text_field(
|
||||
"notes_text",
|
||||
stored=True,
|
||||
tokenizer_name="paperless_text",
|
||||
)
|
||||
|
||||
# Shadow sort fields - fast, not stored/indexed
|
||||
for field in ("title_sort", "correspondent_sort", "type_sort"):
|
||||
sb.add_text_field(
|
||||
field,
|
||||
stored=False,
|
||||
tokenizer_name="simple_analyzer",
|
||||
fast=True,
|
||||
)
|
||||
|
||||
# CJK support - not stored, indexed only
|
||||
sb.add_text_field("bigram_content", stored=False, tokenizer_name="bigram_analyzer")
|
||||
sb.add_text_field("bigram_title", stored=False, tokenizer_name="bigram_analyzer")
|
||||
sb.add_text_field(
|
||||
"bigram_correspondent",
|
||||
stored=False,
|
||||
tokenizer_name="bigram_analyzer",
|
||||
)
|
||||
sb.add_text_field(
|
||||
"bigram_document_type",
|
||||
stored=False,
|
||||
tokenizer_name="bigram_analyzer",
|
||||
)
|
||||
sb.add_text_field("bigram_tag", stored=False, tokenizer_name="bigram_analyzer")
|
||||
|
||||
# Simple substring search support for title/content - not stored, indexed only
|
||||
sb.add_text_field(
|
||||
"simple_title",
|
||||
stored=False,
|
||||
tokenizer_name="simple_search_analyzer",
|
||||
)
|
||||
sb.add_text_field(
|
||||
"simple_content",
|
||||
stored=False,
|
||||
tokenizer_name="simple_search_analyzer",
|
||||
)
|
||||
|
||||
# Autocomplete prefix scan via terms_with_prefix, which walks the field's
|
||||
# term dictionary - so the field must be indexed (term dict), not stored.
|
||||
# The stored value is never read back, so storing it only wastes space.
|
||||
sb.add_text_field("autocomplete_word", stored=False, tokenizer_name="raw")
|
||||
|
||||
# Permission filter columns, read by build_permission_filter.
|
||||
for field in ("owner_id", "viewer_id", "viewer_group_id"):
|
||||
sb.add_unsigned_field(field, stored=False, indexed=True, fast=True)
|
||||
else:
|
||||
raise ValueError(f"Unknown schema field kind: {descriptor.kind}")
|
||||
|
||||
return sb.build()
|
||||
|
||||
@@ -124,9 +270,9 @@ def needs_rebuild(index_dir: Path) -> bool:
|
||||
"""
|
||||
Check if the search index needs rebuilding.
|
||||
|
||||
Reads .index_settings.json to compare the stored schema version and
|
||||
search language against the current configuration. Returns True if the
|
||||
file is missing, unparsable, or either value mismatches.
|
||||
Reads .index_settings.json to compare the stored schema version, search
|
||||
language and schema fingerprint against the current configuration. Returns
|
||||
True if the file is missing, unparsable, or any value mismatches.
|
||||
|
||||
Args:
|
||||
index_dir: Path to the search index directory
|
||||
@@ -145,6 +291,9 @@ def needs_rebuild(index_dir: Path) -> bool:
|
||||
if "language" not in data or data["language"] != settings.SEARCH_LANGUAGE:
|
||||
logger.info("Search index language changed - rebuilding.")
|
||||
return True
|
||||
if data.get("schema_fingerprint") != schema_fingerprint():
|
||||
logger.info("Search index schema fingerprint mismatch - rebuilding.")
|
||||
return True
|
||||
except ValueError:
|
||||
return True
|
||||
return False
|
||||
@@ -175,6 +324,7 @@ def _write_sentinels(index_dir: Path) -> None:
|
||||
{
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"language": settings.SEARCH_LANGUAGE,
|
||||
"schema_fingerprint": schema_fingerprint(),
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
@@ -12,6 +12,7 @@ from documents.search._fields import PUBLIC_FIELDS
|
||||
from documents.search._schema import SCHEMA_VERSION
|
||||
from documents.search._schema import build_schema
|
||||
from documents.search._schema import needs_rebuild
|
||||
from documents.search._schema import schema_fingerprint
|
||||
from documents.search._tokenizer import register_tokenizers
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -35,7 +36,13 @@ class TestNeedsRebuild:
|
||||
) -> None:
|
||||
settings.SEARCH_LANGUAGE = "en"
|
||||
(index_dir / ".index_settings.json").write_text(
|
||||
json.dumps({"schema_version": SCHEMA_VERSION, "language": "en"}),
|
||||
json.dumps(
|
||||
{
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"language": "en",
|
||||
"schema_fingerprint": schema_fingerprint(),
|
||||
},
|
||||
),
|
||||
)
|
||||
assert needs_rebuild(index_dir) is False
|
||||
|
||||
|
||||
@@ -0,0 +1,492 @@
|
||||
"""The schema fingerprint stamped into .index_settings.json.
|
||||
|
||||
tantivy compares schemas by *ordered* field list, and `tantivy.Index(schema,
|
||||
path=...)` (what every write path does) raises on any difference. SCHEMA_VERSION
|
||||
is the manual guard against that, but build_schema() is edited for *parser*
|
||||
reasons - adding an alias, flipping fast=True, adding a subpath - by people not
|
||||
thinking about the on-disk index, and forgetting the bump is exactly how this
|
||||
branch's bug happened.
|
||||
|
||||
The fingerprint is the automatic guard: it hashes the field descriptor list that
|
||||
build_schema() itself iterates, so any change to a field's name, kind, options
|
||||
or *position* forces a rebuild on its own.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
import tantivy
|
||||
|
||||
from documents.search import _schema
|
||||
from documents.search._schema import SCHEMA_VERSION
|
||||
from documents.search._schema import FieldDescriptor
|
||||
from documents.search._schema import _write_sentinels
|
||||
from documents.search._schema import build_schema
|
||||
from documents.search._schema import field_descriptors
|
||||
from documents.search._schema import needs_rebuild
|
||||
from documents.search._schema import schema_fingerprint
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
from pytest_django.fixtures import SettingsWrapper
|
||||
|
||||
pytestmark = pytest.mark.search
|
||||
|
||||
# The on-disk field layout of a v2 index, pinned as data. Any edit here is an
|
||||
# index-format change: it must come with a rebuild, which the fingerprint now
|
||||
# forces automatically. Reproduced from build_schema()'s output as it stood
|
||||
# before the descriptor refactor, so it also pins that the refactor changed
|
||||
# nothing.
|
||||
PINNED_DESCRIPTORS: tuple[FieldDescriptor, ...] = (
|
||||
FieldDescriptor("id", "u64", stored=True, indexed=True, fast=True, tokenizer=None),
|
||||
FieldDescriptor(
|
||||
"title",
|
||||
"text",
|
||||
stored=True,
|
||||
indexed=True,
|
||||
fast=False,
|
||||
tokenizer="paperless_text",
|
||||
),
|
||||
FieldDescriptor(
|
||||
"content",
|
||||
"text",
|
||||
stored=True,
|
||||
indexed=True,
|
||||
fast=False,
|
||||
tokenizer="paperless_text",
|
||||
),
|
||||
FieldDescriptor(
|
||||
"correspondent",
|
||||
"text",
|
||||
stored=True,
|
||||
indexed=True,
|
||||
fast=False,
|
||||
tokenizer="paperless_text",
|
||||
),
|
||||
FieldDescriptor(
|
||||
"document_type",
|
||||
"text",
|
||||
stored=True,
|
||||
indexed=True,
|
||||
fast=False,
|
||||
tokenizer="paperless_text",
|
||||
),
|
||||
FieldDescriptor(
|
||||
"storage_path",
|
||||
"text",
|
||||
stored=True,
|
||||
indexed=True,
|
||||
fast=False,
|
||||
tokenizer="paperless_text",
|
||||
),
|
||||
FieldDescriptor(
|
||||
"original_filename",
|
||||
"text",
|
||||
stored=True,
|
||||
indexed=True,
|
||||
fast=False,
|
||||
tokenizer="paperless_text",
|
||||
),
|
||||
FieldDescriptor(
|
||||
"tag",
|
||||
"text",
|
||||
stored=True,
|
||||
indexed=True,
|
||||
fast=False,
|
||||
tokenizer="paperless_text",
|
||||
),
|
||||
FieldDescriptor(
|
||||
"checksum",
|
||||
"text",
|
||||
stored=True,
|
||||
indexed=True,
|
||||
fast=False,
|
||||
tokenizer="raw",
|
||||
),
|
||||
FieldDescriptor("asn", "u64", stored=True, indexed=True, fast=True, tokenizer=None),
|
||||
FieldDescriptor(
|
||||
"page_count",
|
||||
"u64",
|
||||
stored=True,
|
||||
indexed=True,
|
||||
fast=True,
|
||||
tokenizer=None,
|
||||
),
|
||||
FieldDescriptor(
|
||||
"num_notes",
|
||||
"u64",
|
||||
stored=True,
|
||||
indexed=True,
|
||||
fast=True,
|
||||
tokenizer=None,
|
||||
),
|
||||
FieldDescriptor(
|
||||
"created",
|
||||
"date",
|
||||
stored=True,
|
||||
indexed=True,
|
||||
fast=True,
|
||||
tokenizer=None,
|
||||
),
|
||||
FieldDescriptor(
|
||||
"modified",
|
||||
"date",
|
||||
stored=True,
|
||||
indexed=True,
|
||||
fast=True,
|
||||
tokenizer=None,
|
||||
),
|
||||
FieldDescriptor(
|
||||
"added",
|
||||
"date",
|
||||
stored=True,
|
||||
indexed=True,
|
||||
fast=True,
|
||||
tokenizer=None,
|
||||
),
|
||||
FieldDescriptor(
|
||||
"notes",
|
||||
"json",
|
||||
stored=True,
|
||||
indexed=True,
|
||||
fast=False,
|
||||
tokenizer="paperless_text",
|
||||
),
|
||||
FieldDescriptor(
|
||||
"notes_text",
|
||||
"text",
|
||||
stored=True,
|
||||
indexed=True,
|
||||
fast=False,
|
||||
tokenizer="paperless_text",
|
||||
),
|
||||
FieldDescriptor(
|
||||
"custom_fields",
|
||||
"json",
|
||||
stored=True,
|
||||
indexed=True,
|
||||
fast=False,
|
||||
tokenizer="paperless_text",
|
||||
),
|
||||
FieldDescriptor(
|
||||
"title_sort",
|
||||
"text",
|
||||
stored=False,
|
||||
indexed=True,
|
||||
fast=True,
|
||||
tokenizer="simple_analyzer",
|
||||
),
|
||||
FieldDescriptor(
|
||||
"correspondent_sort",
|
||||
"text",
|
||||
stored=False,
|
||||
indexed=True,
|
||||
fast=True,
|
||||
tokenizer="simple_analyzer",
|
||||
),
|
||||
FieldDescriptor(
|
||||
"type_sort",
|
||||
"text",
|
||||
stored=False,
|
||||
indexed=True,
|
||||
fast=True,
|
||||
tokenizer="simple_analyzer",
|
||||
),
|
||||
FieldDescriptor(
|
||||
"bigram_content",
|
||||
"text",
|
||||
stored=False,
|
||||
indexed=True,
|
||||
fast=False,
|
||||
tokenizer="bigram_analyzer",
|
||||
),
|
||||
FieldDescriptor(
|
||||
"bigram_title",
|
||||
"text",
|
||||
stored=False,
|
||||
indexed=True,
|
||||
fast=False,
|
||||
tokenizer="bigram_analyzer",
|
||||
),
|
||||
FieldDescriptor(
|
||||
"bigram_correspondent",
|
||||
"text",
|
||||
stored=False,
|
||||
indexed=True,
|
||||
fast=False,
|
||||
tokenizer="bigram_analyzer",
|
||||
),
|
||||
FieldDescriptor(
|
||||
"bigram_document_type",
|
||||
"text",
|
||||
stored=False,
|
||||
indexed=True,
|
||||
fast=False,
|
||||
tokenizer="bigram_analyzer",
|
||||
),
|
||||
FieldDescriptor(
|
||||
"bigram_tag",
|
||||
"text",
|
||||
stored=False,
|
||||
indexed=True,
|
||||
fast=False,
|
||||
tokenizer="bigram_analyzer",
|
||||
),
|
||||
FieldDescriptor(
|
||||
"simple_title",
|
||||
"text",
|
||||
stored=False,
|
||||
indexed=True,
|
||||
fast=False,
|
||||
tokenizer="simple_search_analyzer",
|
||||
),
|
||||
FieldDescriptor(
|
||||
"simple_content",
|
||||
"text",
|
||||
stored=False,
|
||||
indexed=True,
|
||||
fast=False,
|
||||
tokenizer="simple_search_analyzer",
|
||||
),
|
||||
FieldDescriptor(
|
||||
"autocomplete_word",
|
||||
"text",
|
||||
stored=False,
|
||||
indexed=True,
|
||||
fast=False,
|
||||
tokenizer="raw",
|
||||
),
|
||||
FieldDescriptor(
|
||||
"owner_id",
|
||||
"u64",
|
||||
stored=False,
|
||||
indexed=True,
|
||||
fast=True,
|
||||
tokenizer=None,
|
||||
),
|
||||
FieldDescriptor(
|
||||
"viewer_id",
|
||||
"u64",
|
||||
stored=False,
|
||||
indexed=True,
|
||||
fast=True,
|
||||
tokenizer=None,
|
||||
),
|
||||
FieldDescriptor(
|
||||
"viewer_group_id",
|
||||
"u64",
|
||||
stored=False,
|
||||
indexed=True,
|
||||
fast=True,
|
||||
tokenizer=None,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _schema_fields(schema: tantivy.Schema) -> list[dict]:
|
||||
"""The tantivy-level field list, in declaration order.
|
||||
|
||||
tantivy-py 0.26 exposes no public introspection API on Schema, so
|
||||
__reduce__() (its pickling hook) is the only way to recover the field list.
|
||||
It is used here, in a test, precisely because it is the representation the
|
||||
persisted fingerprint must NOT depend on.
|
||||
"""
|
||||
return schema.__reduce__()[1][0]["inner"]
|
||||
|
||||
|
||||
def _sentinels(index_dir: Path, **overrides: object) -> None:
|
||||
data = {
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"language": None,
|
||||
"schema_fingerprint": schema_fingerprint(),
|
||||
}
|
||||
data.update(overrides)
|
||||
(index_dir / ".index_settings.json").write_text(json.dumps(data))
|
||||
|
||||
|
||||
class TestDescriptorsDescribeTheBuiltSchema:
|
||||
def test_descriptors_match_the_pinned_field_layout(self) -> None:
|
||||
assert tuple(field_descriptors()) == PINNED_DESCRIPTORS
|
||||
|
||||
def test_built_schema_matches_the_descriptors(self) -> None:
|
||||
"""The descriptors are not a parallel description - they are the input.
|
||||
|
||||
Reading the built schema back proves the loop honours every option, so
|
||||
a descriptor edit cannot claim a shape the SchemaBuilder did not build.
|
||||
"""
|
||||
kinds = {"text": "text", "json": "json_object", "u64": "u64", "date": "date"}
|
||||
built = [
|
||||
(
|
||||
field["name"],
|
||||
field["type"],
|
||||
field["options"]["stored"],
|
||||
bool(field["options"].get("fast")),
|
||||
(field["options"].get("indexing") or {}).get("tokenizer"),
|
||||
)
|
||||
for field in _schema_fields(build_schema())
|
||||
]
|
||||
expected = [
|
||||
(
|
||||
descriptor.name,
|
||||
kinds[descriptor.kind],
|
||||
descriptor.stored,
|
||||
descriptor.fast,
|
||||
descriptor.tokenizer,
|
||||
)
|
||||
for descriptor in field_descriptors()
|
||||
]
|
||||
assert built == expected
|
||||
|
||||
|
||||
class TestFingerprintSensitivity:
|
||||
def test_a_field_option_change_moves_the_fingerprint(
|
||||
self,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
before = schema_fingerprint()
|
||||
changed = field_descriptors()
|
||||
changed[1] = changed[1]._replace(fast=True)
|
||||
monkeypatch.setattr(_schema, "field_descriptors", lambda: changed)
|
||||
|
||||
assert schema_fingerprint() != before
|
||||
|
||||
def test_reordering_alone_moves_the_fingerprint(
|
||||
self,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""The original bug: same fields, different declaration order.
|
||||
|
||||
A set- or dict-based fingerprint would be blind to this, and tantivy
|
||||
would reject every write against the existing index.
|
||||
"""
|
||||
before = schema_fingerprint()
|
||||
swapped = field_descriptors()
|
||||
swapped[1], swapped[2] = swapped[2], swapped[1]
|
||||
monkeypatch.setattr(_schema, "field_descriptors", lambda: swapped)
|
||||
|
||||
assert schema_fingerprint() != before
|
||||
|
||||
def test_repeated_calls_agree(self) -> None:
|
||||
assert schema_fingerprint() == schema_fingerprint()
|
||||
|
||||
|
||||
class TestFingerprintIsIndependentOfTantivy:
|
||||
def test_a_tantivy_option_key_addition_would_not_move_it(self) -> None:
|
||||
"""A tantivy-py upgrade must not force a global reindex.
|
||||
|
||||
Hashing schema.__reduce__() would do exactly that: the simulated new
|
||||
option key below changes that payload for every user with no schema
|
||||
change at all.
|
||||
"""
|
||||
fields = _schema_fields(build_schema())
|
||||
upgraded = [
|
||||
{**field, "options": {**field["options"], "coerce": True}}
|
||||
for field in fields
|
||||
]
|
||||
assert _hash(upgraded) != _hash(fields)
|
||||
assert schema_fingerprint() == _fingerprint_of(field_descriptors())
|
||||
|
||||
def test_fingerprint_never_touches_the_schema_builder(
|
||||
self,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
before = schema_fingerprint()
|
||||
|
||||
class _RemovedSchemaBuilder:
|
||||
def __init__(self) -> None:
|
||||
raise AssertionError("tantivy.SchemaBuilder was consulted")
|
||||
|
||||
monkeypatch.setattr(tantivy, "SchemaBuilder", _RemovedSchemaBuilder)
|
||||
with pytest.raises(AssertionError):
|
||||
build_schema()
|
||||
|
||||
assert schema_fingerprint() == before
|
||||
|
||||
|
||||
def _hash(payload: object) -> str:
|
||||
return hashlib.blake2b(json.dumps(payload).encode()).hexdigest()
|
||||
|
||||
|
||||
def _fingerprint_of(descriptors: list[FieldDescriptor]) -> str:
|
||||
return _hash([list(descriptor) for descriptor in descriptors])
|
||||
|
||||
|
||||
class TestNeedsRebuildOnFingerprint:
|
||||
def test_matching_fingerprint_does_not_rebuild(
|
||||
self,
|
||||
index_dir: Path,
|
||||
settings: SettingsWrapper,
|
||||
) -> None:
|
||||
settings.SEARCH_LANGUAGE = None
|
||||
_sentinels(index_dir)
|
||||
|
||||
assert needs_rebuild(index_dir) is False
|
||||
|
||||
def test_stale_fingerprint_rebuilds_despite_a_matching_version(
|
||||
self,
|
||||
index_dir: Path,
|
||||
settings: SettingsWrapper,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""The failure this task exists to prevent: schema edited, version not
|
||||
bumped. Without the fingerprint check, `reindex --if-needed` reports the
|
||||
index up to date and every write then raises."""
|
||||
settings.SEARCH_LANGUAGE = None
|
||||
_sentinels(index_dir)
|
||||
extended = [
|
||||
*field_descriptors(),
|
||||
FieldDescriptor(
|
||||
"new_field",
|
||||
"u64",
|
||||
stored=False,
|
||||
indexed=True,
|
||||
fast=True,
|
||||
tokenizer=None,
|
||||
),
|
||||
]
|
||||
monkeypatch.setattr(_schema, "field_descriptors", lambda: extended)
|
||||
|
||||
assert needs_rebuild(index_dir) is True
|
||||
|
||||
def test_reordered_schema_rebuilds(
|
||||
self,
|
||||
index_dir: Path,
|
||||
settings: SettingsWrapper,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
settings.SEARCH_LANGUAGE = None
|
||||
_sentinels(index_dir)
|
||||
reordered = field_descriptors()
|
||||
reordered[1], reordered[2] = reordered[2], reordered[1]
|
||||
monkeypatch.setattr(_schema, "field_descriptors", lambda: reordered)
|
||||
|
||||
assert needs_rebuild(index_dir) is True
|
||||
|
||||
def test_missing_fingerprint_rebuilds(
|
||||
self,
|
||||
index_dir: Path,
|
||||
settings: SettingsWrapper,
|
||||
) -> None:
|
||||
"""No seeding: an index whose schema shape nobody recorded is rebuilt
|
||||
rather than trusted."""
|
||||
settings.SEARCH_LANGUAGE = None
|
||||
(index_dir / ".index_settings.json").write_text(
|
||||
json.dumps({"schema_version": SCHEMA_VERSION, "language": None}),
|
||||
)
|
||||
|
||||
assert needs_rebuild(index_dir) is True
|
||||
|
||||
def test_written_sentinels_satisfy_the_check(
|
||||
self,
|
||||
index_dir: Path,
|
||||
settings: SettingsWrapper,
|
||||
) -> None:
|
||||
settings.SEARCH_LANGUAGE = "en"
|
||||
_write_sentinels(index_dir)
|
||||
|
||||
assert needs_rebuild(index_dir) is False
|
||||
Reference in New Issue
Block a user