refactor(search): derive build_schema() from shared PUBLIC_FIELDS table

src/documents/search/_schema.py

@@ -9,6 +9,9 @@ from typing import cast
 
 import tantivy
 from django.conf import settings
+from whoosh_compat import FieldKind
+
+from documents.search._fields import PUBLIC_FIELDS
 
 if TYPE_CHECKING:
     from pathlib import Path
@@ -33,17 +36,37 @@ def build_schema() -> tantivy.Schema:
     sb = tantivy.SchemaBuilder()
 
     sb.add_unsigned_field("id", stored=True, indexed=True, fast=True)
-    sb.add_text_field("checksum", stored=True, tokenizer_name="raw")
 
-    for field in (
-        "title",
-        "correspondent",
-        "document_type",
-        "storage_path",
-        "original_filename",
-        "content",
-    ):
-        sb.add_text_field(field, stored=True, tokenizer_name="paperless_text")
+    for field in PUBLIC_FIELDS:
+        if field.kind is FieldKind.TEXT:
+            sb.add_text_field(field.name, stored=True, tokenizer_name="paperless_text")
+        elif field.kind is FieldKind.KEYWORD:
+            sb.add_text_field(field.name, stored=True, tokenizer_name="raw")
+        elif field.kind is FieldKind.U64:
+            sb.add_unsigned_field(
+                field.name,
+                stored=True,
+                indexed=True,
+                fast=field.fast,
+            )
+        elif field.kind in (FieldKind.DATE, FieldKind.DATETIME):
+            sb.add_date_field(
+                field.name,
+                stored=True,
+                indexed=True,
+                fast=field.fast,
+            )
+        elif field.kind is FieldKind.JSON:
+            sb.add_json_field(field.name, stored=True, tokenizer_name="paperless_text")
+            if field.name == "notes":
+                # Plain-text companion for snippet generation — tantivy's
+                # SnippetGenerator does not support JSON fields. Schema-only,
+                # no query-syntax meaning, not in PUBLIC_FIELDS.
+                sb.add_text_field(
+                    "notes_text",
+                    stored=True,
+                    tokenizer_name="paperless_text",
+                )
 
     # Shadow sort fields - fast, not stored/indexed
     for field in ("title_sort", "correspondent_sort", "type_sort"):
@@ -86,15 +109,6 @@ def build_schema() -> tantivy.Schema:
     # The stored value is never read back, so storing it only wastes space.
     sb.add_text_field("autocomplete_word", stored=False, tokenizer_name="raw")
 
-    sb.add_text_field("tag", stored=True, tokenizer_name="paperless_text")
-
-    # JSON fields — structured queries: notes.user:alice, custom_fields.name:invoice
-    sb.add_json_field("notes", stored=True, tokenizer_name="paperless_text")
-    # Plain-text companion for notes — tantivy's SnippetGenerator does not support
-    # JSON fields, so highlights require a text field with the same content.
-    sb.add_text_field("notes_text", stored=True, tokenizer_name="paperless_text")
-    sb.add_json_field("custom_fields", stored=True, tokenizer_name="paperless_text")
-
     for field in (
         "correspondent_id",
         "document_type_id",
@@ -106,12 +120,6 @@ def build_schema() -> tantivy.Schema:
     ):
         sb.add_unsigned_field(field, stored=False, indexed=True, fast=True)
 
-    for field in ("created", "modified", "added"):
-        sb.add_date_field(field, stored=True, indexed=True, fast=True)
-
-    for field in ("asn", "page_count", "num_notes"):
-        sb.add_unsigned_field(field, stored=True, indexed=True, fast=True)
-
     return sb.build()
 
 

src/documents/tests/search/test_schema.py

@@ -1,12 +1,18 @@
 from __future__ import annotations
 
 import json
+from datetime import UTC
+from datetime import datetime
 from typing import TYPE_CHECKING
 
 import pytest
+import tantivy
 
+from documents.search._fields import PUBLIC_FIELDS
 from documents.search._schema import SCHEMA_VERSION
+from documents.search._schema import build_schema
 from documents.search._schema import needs_rebuild
+from documents.search._tokenizer import register_tokenizers
 
 if TYPE_CHECKING:
     from pathlib import Path
@@ -76,3 +82,47 @@ class TestNeedsRebuild:
             json.dumps({"schema_version": SCHEMA_VERSION, "language": "en"}),
         )
         assert needs_rebuild(index_dir) is True
+
+
+def _schema_field_names(schema: tantivy.Schema) -> set[str]:
+    """Return the set of field names declared on a tantivy Schema.
+
+    tantivy-py 0.26 exposes no public introspection API on Schema (no
+    __iter__, get_field, to_json, etc.) -- __reduce__() (used internally for
+    pickling) is the only way to recover the field list, so we lean on it
+    here for test assertions only.
+    """
+    state = schema.__reduce__()[1][0]
+    return {field["name"] for field in state["inner"]}
+
+
+class TestSchemaMatchesPublicFields:
+    def test_every_public_field_is_in_the_schema(self) -> None:
+        schema = build_schema()
+        schema_field_names = _schema_field_names(schema)
+        for field in PUBLIC_FIELDS:
+            assert field.name in schema_field_names, (
+                f"{field.name} is in PUBLIC_FIELDS but missing from build_schema()"
+            )
+
+    def test_asn_page_count_num_notes_are_fast_unsigned_fields(self) -> None:
+        # Spot-check kind-derived construction for the U64 fields.
+        schema = build_schema()
+        doc = tantivy.Document()
+        doc.add_unsigned("id", 1)
+        doc.add_text("checksum", "x")
+        doc.add_unsigned("asn", 42)
+        doc.add_unsigned("page_count", 3)
+        doc.add_unsigned("num_notes", 0)
+        doc.add_date("created", datetime(2020, 1, 1, tzinfo=UTC))
+        doc.add_date("modified", datetime(2020, 1, 1, tzinfo=UTC))
+        doc.add_date("added", datetime(2020, 1, 1, tzinfo=UTC))
+        index = tantivy.Index(schema)
+        register_tokenizers(index, None)
+        writer = index.writer()
+        writer.add_document(doc)
+        writer.commit()
+        index.reload()
+        searcher = index.searcher()
+        results = searcher.search(tantivy.Query.term_query(schema, "asn", 42), limit=1)
+        assert len(results.hits) == 1