diff --git a/src/documents/search/_schema.py b/src/documents/search/_schema.py index bb0361b2c..e850a2e98 100644 --- a/src/documents/search/_schema.py +++ b/src/documents/search/_schema.py @@ -9,6 +9,9 @@ from typing import cast import tantivy from django.conf import settings +from whoosh_compat import FieldKind + +from documents.search._fields import PUBLIC_FIELDS if TYPE_CHECKING: from pathlib import Path @@ -33,17 +36,37 @@ def build_schema() -> tantivy.Schema: sb = tantivy.SchemaBuilder() sb.add_unsigned_field("id", stored=True, indexed=True, fast=True) - sb.add_text_field("checksum", stored=True, tokenizer_name="raw") - for field in ( - "title", - "correspondent", - "document_type", - "storage_path", - "original_filename", - "content", - ): - sb.add_text_field(field, stored=True, tokenizer_name="paperless_text") + for field in PUBLIC_FIELDS: + if field.kind is FieldKind.TEXT: + sb.add_text_field(field.name, stored=True, tokenizer_name="paperless_text") + elif field.kind is FieldKind.KEYWORD: + sb.add_text_field(field.name, stored=True, tokenizer_name="raw") + elif field.kind is FieldKind.U64: + sb.add_unsigned_field( + field.name, + stored=True, + indexed=True, + fast=field.fast, + ) + elif field.kind in (FieldKind.DATE, FieldKind.DATETIME): + sb.add_date_field( + field.name, + stored=True, + indexed=True, + fast=field.fast, + ) + elif field.kind is FieldKind.JSON: + sb.add_json_field(field.name, stored=True, tokenizer_name="paperless_text") + if field.name == "notes": + # Plain-text companion for snippet generation — tantivy's + # SnippetGenerator does not support JSON fields. Schema-only, + # no query-syntax meaning, not in PUBLIC_FIELDS. + sb.add_text_field( + "notes_text", + stored=True, + tokenizer_name="paperless_text", + ) # Shadow sort fields - fast, not stored/indexed for field in ("title_sort", "correspondent_sort", "type_sort"): @@ -86,15 +109,6 @@ def build_schema() -> tantivy.Schema: # The stored value is never read back, so storing it only wastes space. sb.add_text_field("autocomplete_word", stored=False, tokenizer_name="raw") - sb.add_text_field("tag", stored=True, tokenizer_name="paperless_text") - - # JSON fields — structured queries: notes.user:alice, custom_fields.name:invoice - sb.add_json_field("notes", stored=True, tokenizer_name="paperless_text") - # Plain-text companion for notes — tantivy's SnippetGenerator does not support - # JSON fields, so highlights require a text field with the same content. - sb.add_text_field("notes_text", stored=True, tokenizer_name="paperless_text") - sb.add_json_field("custom_fields", stored=True, tokenizer_name="paperless_text") - for field in ( "correspondent_id", "document_type_id", @@ -106,12 +120,6 @@ def build_schema() -> tantivy.Schema: ): sb.add_unsigned_field(field, stored=False, indexed=True, fast=True) - for field in ("created", "modified", "added"): - sb.add_date_field(field, stored=True, indexed=True, fast=True) - - for field in ("asn", "page_count", "num_notes"): - sb.add_unsigned_field(field, stored=True, indexed=True, fast=True) - return sb.build() diff --git a/src/documents/tests/search/test_schema.py b/src/documents/tests/search/test_schema.py index 7219df580..83ad17ab3 100644 --- a/src/documents/tests/search/test_schema.py +++ b/src/documents/tests/search/test_schema.py @@ -1,12 +1,18 @@ from __future__ import annotations import json +from datetime import UTC +from datetime import datetime from typing import TYPE_CHECKING import pytest +import tantivy +from documents.search._fields import PUBLIC_FIELDS from documents.search._schema import SCHEMA_VERSION +from documents.search._schema import build_schema from documents.search._schema import needs_rebuild +from documents.search._tokenizer import register_tokenizers if TYPE_CHECKING: from pathlib import Path @@ -76,3 +82,47 @@ class TestNeedsRebuild: json.dumps({"schema_version": SCHEMA_VERSION, "language": "en"}), ) assert needs_rebuild(index_dir) is True + + +def _schema_field_names(schema: tantivy.Schema) -> set[str]: + """Return the set of field names declared on a tantivy Schema. + + tantivy-py 0.26 exposes no public introspection API on Schema (no + __iter__, get_field, to_json, etc.) -- __reduce__() (used internally for + pickling) is the only way to recover the field list, so we lean on it + here for test assertions only. + """ + state = schema.__reduce__()[1][0] + return {field["name"] for field in state["inner"]} + + +class TestSchemaMatchesPublicFields: + def test_every_public_field_is_in_the_schema(self) -> None: + schema = build_schema() + schema_field_names = _schema_field_names(schema) + for field in PUBLIC_FIELDS: + assert field.name in schema_field_names, ( + f"{field.name} is in PUBLIC_FIELDS but missing from build_schema()" + ) + + def test_asn_page_count_num_notes_are_fast_unsigned_fields(self) -> None: + # Spot-check kind-derived construction for the U64 fields. + schema = build_schema() + doc = tantivy.Document() + doc.add_unsigned("id", 1) + doc.add_text("checksum", "x") + doc.add_unsigned("asn", 42) + doc.add_unsigned("page_count", 3) + doc.add_unsigned("num_notes", 0) + doc.add_date("created", datetime(2020, 1, 1, tzinfo=UTC)) + doc.add_date("modified", datetime(2020, 1, 1, tzinfo=UTC)) + doc.add_date("added", datetime(2020, 1, 1, tzinfo=UTC)) + index = tantivy.Index(schema) + register_tokenizers(index, None) + writer = index.writer() + writer.add_document(doc) + writer.commit() + index.reload() + searcher = index.searcher() + results = searcher.search(tantivy.Query.term_query(schema, "asn", 42), limit=1) + assert len(results.hits) == 1