From d0869bec01def15c4ee79ac8e6c97f211e6aae22 Mon Sep 17 00:00:00 2001 From: stumpylog <797416+stumpylog@users.noreply.github.com> Date: Tue, 25 Aug 2026 10:28:26 -0700 Subject: [PATCH] feat(search): add whoosh-compat, the shared field table and the field registry --- pyproject.toml | 1 + src/documents/search/_fields.py | 42 +++ src/documents/search/_registry.py | 91 +++++ src/documents/search/_schema.py | 321 +++++++++++++----- src/documents/search/_tokenizer.py | 53 ++- src/documents/tests/search/test_fields.py | 10 + .../search/test_json_subpath_completeness.py | 83 +++++ .../search/test_keyword_pattern_literal.py | 60 ++++ src/documents/tests/search/test_registry.py | 149 ++++++++ src/documents/tests/search/test_schema.py | 85 ++++- src/documents/tests/search/test_tokenizer.py | 4 +- uv.lock | 27 +- 12 files changed, 834 insertions(+), 92 deletions(-) create mode 100644 src/documents/search/_fields.py create mode 100644 src/documents/search/_registry.py create mode 100644 src/documents/tests/search/test_fields.py create mode 100644 src/documents/tests/search/test_json_subpath_completeness.py create mode 100644 src/documents/tests/search/test_keyword_pattern_literal.py create mode 100644 src/documents/tests/search/test_registry.py diff --git a/pyproject.toml b/pyproject.toml index 6371288bd..ccc038eb6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,6 +77,7 @@ dependencies = [ "torch~=2.13.0", "watchfiles>=1.2", "whitenoise~=6.11", + "whoosh-compat[tantivy]==0.1", "zxing-cpp~=3.1.0", ] [project.optional-dependencies] diff --git a/src/documents/search/_fields.py b/src/documents/search/_fields.py new file mode 100644 index 000000000..002a4836d --- /dev/null +++ b/src/documents/search/_fields.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +from whoosh_compat import FieldKind +from whoosh_compat import FieldSpec +from whoosh_compat import SubpathSpec + +# Internal-only schema fields with no query-syntax meaning of their own +# (sort shadow fields, bigram CJK fields, simple_title/simple_content, +# autocomplete_word, notes_text) are NOT represented here — they are +# declared in _schema.py's field_descriptors(). +# +# analyzer/pattern_normalizer are deliberately left at FieldSpec's default +# (None): they're language-specific and only meaningful to whoosh-compat's +# parser, so _registry.py attaches them per-language via dataclasses.replace() +# rather than PUBLIC_FIELDS declaring them itself. _schema.py only reads +# name/kind/fast and never sees the analyzer at all. +PUBLIC_FIELDS: tuple[FieldSpec, ...] = ( + FieldSpec("title", FieldKind.TEXT), + FieldSpec("content", FieldKind.TEXT), + FieldSpec("correspondent", FieldKind.TEXT), + FieldSpec("document_type", FieldKind.TEXT, aliases=("type",)), + FieldSpec("storage_path", FieldKind.TEXT, aliases=("path",)), + FieldSpec("original_filename", FieldKind.TEXT), + FieldSpec("tag", FieldKind.TEXT, comma_values=True), + FieldSpec("checksum", FieldKind.KEYWORD), + FieldSpec("asn", FieldKind.U64, fast=True), + FieldSpec("page_count", FieldKind.U64, fast=True), + FieldSpec("num_notes", FieldKind.U64, fast=True), + FieldSpec("created", FieldKind.DATE, date_only=True, fast=True), + FieldSpec("modified", FieldKind.DATETIME, fast=True), + FieldSpec("added", FieldKind.DATETIME, fast=True), + FieldSpec( + "notes", + FieldKind.JSON, + subpaths={"user": SubpathSpec(), "note": SubpathSpec(default=True)}, + ), + FieldSpec( + "custom_fields", + FieldKind.JSON, + subpaths={"name": SubpathSpec(), "value": SubpathSpec(default=True)}, + ), +) diff --git a/src/documents/search/_registry.py b/src/documents/search/_registry.py new file mode 100644 index 000000000..b7628db54 --- /dev/null +++ b/src/documents/search/_registry.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +import dataclasses +from typing import TYPE_CHECKING + +from whoosh_compat import FieldKind +from whoosh_compat import FieldRegistry + +from documents.search._fields import PUBLIC_FIELDS +from documents.search._tokenizer import ascii_fold +from documents.search._tokenizer import paperless_text_analyzer +from documents.search._tokenizer import stem_pattern_text + +if TYPE_CHECKING: + from whoosh_compat import PatternNormalizer + +_registry_cache: dict[str | None, FieldRegistry] = {} + + +def _identity_analyzer(text: str) -> list[str]: + """Analyzer for KEYWORD fields indexed with the raw tokenizer (no splitting).""" + return [text] + + +def _fold_normalizer(text: str) -> str: + """Wildcard/regex literal-run normalizer for fields indexed without stemming.""" + return ascii_fold(text.lower()) + + +def _make_pattern_normalizer(language: str | None) -> PatternNormalizer: + """Build the wildcard/regex literal-run normalizer for a search language.""" + + def _pattern_normalizer(text: str) -> tuple[str, ...]: + """Normalize a literal run into the forms a term may match. + + TEXT index terms go through lowercase -> ascii_fold -> stem, so a + pattern that skips stemming can never match one: "invoice*" would look + for a term starting with "invoice" while the index holds "invoic". The + run is therefore offered stemmed as well. KEYWORD fields are indexed + raw and get _fold_normalizer instead, so their patterns stay literal. + + Both forms are returned, as alternatives, because neither is a prefix + of the other in general: English stemming substitutes as well as + truncates ("copy" -> "copi"), so the stem alone loses the compounds + the typed run reaches ("copyright") while the typed run alone loses + the inflections the stem reaches ("copies"). whoosh-compat ORs the + alternatives per literal run and deduplicates them, so a run the + stemmer leaves alone costs exactly the one branch it did before. + + Inside a bracket class the emitter calls this once per character and + uses the answer only if it is a single one-character form; two forms + there leave the character as typed. A stemmer does not change a lone + character, so the two forms deduplicate to one and the class body is + folded as before. + """ + folded = ascii_fold(text.lower()) + stemmed = stem_pattern_text(folded, language) + return (folded, stemmed) + + return _pattern_normalizer + + +def get_field_registry(language: str | None) -> FieldRegistry: + """Build (or return the cached) FieldRegistry for the given search language. + + Cached keyed by language, rebuilt on the same trigger register_tokenizers() + uses (settings.SEARCH_LANGUAGE change) — a fresh call with a new language + builds and caches a new registry rather than mutating the old one. + """ + if language in _registry_cache: + return _registry_cache[language] + + text_analyzer = paperless_text_analyzer(language).analyze + pattern_normalizer = _make_pattern_normalizer(language) + + specs = [ + dataclasses.replace( + field, + analyzer=_identity_analyzer + if field.kind is FieldKind.KEYWORD + else text_analyzer, + pattern_normalizer=_fold_normalizer + if field.kind is FieldKind.KEYWORD + else pattern_normalizer, + ) + for field in PUBLIC_FIELDS + ] + + registry = FieldRegistry(specs) + _registry_cache[language] = registry + return registry diff --git a/src/documents/search/_schema.py b/src/documents/search/_schema.py index bb0361b2c..ee4ad6b5e 100644 --- a/src/documents/search/_schema.py +++ b/src/documents/search/_schema.py @@ -1,14 +1,19 @@ from __future__ import annotations +import hashlib import json import logging import shutil from typing import TYPE_CHECKING from typing import Final +from typing import NamedTuple from typing import cast import tantivy from django.conf import settings +from whoosh_compat import FieldKind + +from documents.search._fields import PUBLIC_FIELDS if TYPE_CHECKING: from pathlib import Path @@ -16,7 +21,201 @@ if TYPE_CHECKING: logger = logging.getLogger("paperless.search") # v1 - Initial tantivy schema format -SCHEMA_VERSION: Final[int] = 1 +# v2 - build_schema() derived from PUBLIC_FIELDS, changing the field declaration +# order, and the write-only correspondent/document_type/storage_path/tag id +# columns dropped. tantivy compares schemas by ordered field list, so an +# index built by v1 rejects every write against the v2 schema. +SCHEMA_VERSION: Final[int] = 2 + + +class FieldDescriptor(NamedTuple): + """One tantivy field, in declaration order. + + The descriptor vocabulary is paperless', not tantivy-py's: it is both the + input to the SchemaBuilder and the input to schema_fingerprint(), so the + persisted fingerprint cannot move under a tantivy-py upgrade. + """ + + name: str + kind: str + stored: bool + indexed: bool + fast: bool + tokenizer: str | None + + +def _public_field_descriptors() -> list[FieldDescriptor]: + """Descriptors for the query-visible fields declared in PUBLIC_FIELDS.""" + descriptors: list[FieldDescriptor] = [] + for field in PUBLIC_FIELDS: + if field.kind is FieldKind.TEXT: + descriptors.append( + FieldDescriptor( + field.name, + "text", + stored=True, + indexed=True, + fast=False, + tokenizer="paperless_text", + ), + ) + elif field.kind is FieldKind.KEYWORD: + descriptors.append( + FieldDescriptor( + field.name, + "text", + stored=True, + indexed=True, + fast=False, + tokenizer="raw", + ), + ) + elif field.kind is FieldKind.U64: + descriptors.append( + FieldDescriptor( + field.name, + "u64", + stored=True, + indexed=True, + fast=field.fast, + tokenizer=None, + ), + ) + elif field.kind in (FieldKind.DATE, FieldKind.DATETIME): + descriptors.append( + FieldDescriptor( + field.name, + "date", + stored=True, + indexed=True, + fast=field.fast, + tokenizer=None, + ), + ) + elif field.kind is FieldKind.JSON: + descriptors.append( + FieldDescriptor( + field.name, + "json", + stored=True, + indexed=True, + fast=False, + tokenizer="paperless_text", + ), + ) + if field.name == "notes": + # Plain-text companion for snippet generation — tantivy's + # SnippetGenerator does not support JSON fields. Schema-only, + # no query-syntax meaning, not in PUBLIC_FIELDS. + descriptors.append( + FieldDescriptor( + "notes_text", + "text", + stored=True, + indexed=True, + fast=False, + tokenizer="paperless_text", + ), + ) + return descriptors + + +def field_descriptors() -> list[FieldDescriptor]: + """Every field of the document index, in the order tantivy declares them. + + tantivy compares schemas by *ordered* field list, so the order here is + part of the on-disk contract: schema_fingerprint() hashes it and + needs_rebuild() acts on the result. + """ + return [ + FieldDescriptor( + "id", + "u64", + stored=True, + indexed=True, + fast=True, + tokenizer=None, + ), + *_public_field_descriptors(), + # Shadow sort fields - fast, not stored + *( + FieldDescriptor( + name, + "text", + stored=False, + indexed=True, + fast=True, + tokenizer="simple_analyzer", + ) + for name in ("title_sort", "correspondent_sort", "type_sort") + ), + # CJK support - not stored, indexed only + *( + FieldDescriptor( + name, + "text", + stored=False, + indexed=True, + fast=False, + tokenizer="bigram_analyzer", + ) + for name in ( + "bigram_content", + "bigram_title", + "bigram_correspondent", + "bigram_document_type", + "bigram_tag", + ) + ), + # Simple substring search support for title/content - not stored, + # indexed only + *( + FieldDescriptor( + name, + "text", + stored=False, + indexed=True, + fast=False, + tokenizer="simple_search_analyzer", + ) + for name in ("simple_title", "simple_content") + ), + # Autocomplete prefix scan via terms_with_prefix, which walks the + # field's term dictionary - so the field must be indexed (term dict), + # not stored. The stored value is never read back, so storing it only + # wastes space. + FieldDescriptor( + "autocomplete_word", + "text", + stored=False, + indexed=True, + fast=False, + tokenizer="raw", + ), + # Permission filter columns, read by build_permission_filter. + *( + FieldDescriptor( + name, + "u64", + stored=False, + indexed=True, + fast=True, + tokenizer=None, + ) + for name in ("owner_id", "viewer_id", "viewer_group_id") + ), + ] + + +def schema_fingerprint() -> str: + """Hash of the field descriptors, stamped into .index_settings.json. + + Changes whenever a field is added, removed, retyped, re-optioned or + reordered, so an index built from a different schema shape is detected + even when SCHEMA_VERSION was not bumped. + """ + payload = json.dumps([list(descriptor) for descriptor in field_descriptors()]) + return hashlib.blake2b(payload.encode()).hexdigest() def build_schema() -> tantivy.Schema: @@ -32,85 +231,37 @@ def build_schema() -> tantivy.Schema: """ sb = tantivy.SchemaBuilder() - sb.add_unsigned_field("id", stored=True, indexed=True, fast=True) - sb.add_text_field("checksum", stored=True, tokenizer_name="raw") - - for field in ( - "title", - "correspondent", - "document_type", - "storage_path", - "original_filename", - "content", - ): - sb.add_text_field(field, stored=True, tokenizer_name="paperless_text") - - # Shadow sort fields - fast, not stored/indexed - for field in ("title_sort", "correspondent_sort", "type_sort"): - sb.add_text_field( - field, - stored=False, - tokenizer_name="simple_analyzer", - fast=True, - ) - - # CJK support - not stored, indexed only - sb.add_text_field("bigram_content", stored=False, tokenizer_name="bigram_analyzer") - sb.add_text_field("bigram_title", stored=False, tokenizer_name="bigram_analyzer") - sb.add_text_field( - "bigram_correspondent", - stored=False, - tokenizer_name="bigram_analyzer", - ) - sb.add_text_field( - "bigram_document_type", - stored=False, - tokenizer_name="bigram_analyzer", - ) - sb.add_text_field("bigram_tag", stored=False, tokenizer_name="bigram_analyzer") - - # Simple substring search support for title/content - not stored, indexed only - sb.add_text_field( - "simple_title", - stored=False, - tokenizer_name="simple_search_analyzer", - ) - sb.add_text_field( - "simple_content", - stored=False, - tokenizer_name="simple_search_analyzer", - ) - - # Autocomplete prefix scan via terms_with_prefix, which walks the field's - # term dictionary - so the field must be indexed (term dict), not stored. - # The stored value is never read back, so storing it only wastes space. - sb.add_text_field("autocomplete_word", stored=False, tokenizer_name="raw") - - sb.add_text_field("tag", stored=True, tokenizer_name="paperless_text") - - # JSON fields — structured queries: notes.user:alice, custom_fields.name:invoice - sb.add_json_field("notes", stored=True, tokenizer_name="paperless_text") - # Plain-text companion for notes — tantivy's SnippetGenerator does not support - # JSON fields, so highlights require a text field with the same content. - sb.add_text_field("notes_text", stored=True, tokenizer_name="paperless_text") - sb.add_json_field("custom_fields", stored=True, tokenizer_name="paperless_text") - - for field in ( - "correspondent_id", - "document_type_id", - "storage_path_id", - "tag_id", - "owner_id", - "viewer_id", - "viewer_group_id", - ): - sb.add_unsigned_field(field, stored=False, indexed=True, fast=True) - - for field in ("created", "modified", "added"): - sb.add_date_field(field, stored=True, indexed=True, fast=True) - - for field in ("asn", "page_count", "num_notes"): - sb.add_unsigned_field(field, stored=True, indexed=True, fast=True) + for descriptor in field_descriptors(): + if descriptor.kind == "text": + sb.add_text_field( + descriptor.name, + stored=descriptor.stored, + fast=descriptor.fast, + tokenizer_name=cast("str", descriptor.tokenizer), + ) + elif descriptor.kind == "json": + sb.add_json_field( + descriptor.name, + stored=descriptor.stored, + fast=descriptor.fast, + tokenizer_name=cast("str", descriptor.tokenizer), + ) + elif descriptor.kind == "u64": + sb.add_unsigned_field( + descriptor.name, + stored=descriptor.stored, + indexed=descriptor.indexed, + fast=descriptor.fast, + ) + elif descriptor.kind == "date": + sb.add_date_field( + descriptor.name, + stored=descriptor.stored, + indexed=descriptor.indexed, + fast=descriptor.fast, + ) + else: + raise ValueError(f"Unknown schema field kind: {descriptor.kind}") return sb.build() @@ -119,9 +270,9 @@ def needs_rebuild(index_dir: Path) -> bool: """ Check if the search index needs rebuilding. - Reads .index_settings.json to compare the stored schema version and - search language against the current configuration. Returns True if the - file is missing, unparsable, or either value mismatches. + Reads .index_settings.json to compare the stored schema version, search + language and schema fingerprint against the current configuration. Returns + True if the file is missing, unparsable, or any value mismatches. Args: index_dir: Path to the search index directory @@ -140,6 +291,9 @@ def needs_rebuild(index_dir: Path) -> bool: if "language" not in data or data["language"] != settings.SEARCH_LANGUAGE: logger.info("Search index language changed - rebuilding.") return True + if data.get("schema_fingerprint") != schema_fingerprint(): + logger.info("Search index schema fingerprint mismatch - rebuilding.") + return True except ValueError: return True return False @@ -170,6 +324,7 @@ def _write_sentinels(index_dir: Path) -> None: { "schema_version": SCHEMA_VERSION, "language": settings.SEARCH_LANGUAGE, + "schema_fingerprint": schema_fingerprint(), }, ), ) diff --git a/src/documents/search/_tokenizer.py b/src/documents/search/_tokenizer.py index c84dd9093..beadc0312 100644 --- a/src/documents/search/_tokenizer.py +++ b/src/documents/search/_tokenizer.py @@ -1,6 +1,7 @@ from __future__ import annotations import logging +from functools import cache from typing import Final import tantivy @@ -71,7 +72,7 @@ def register_tokenizers(index: tantivy.Index, language: str | None) -> None: use fast=True and Tantivy requires fast-field tokenizers to exist even for documents that omit those fields. """ - index.register_tokenizer("paperless_text", _paperless_text(language)) + index.register_tokenizer("paperless_text", paperless_text_analyzer(language)) index.register_tokenizer("simple_analyzer", _simple_analyzer()) index.register_tokenizer("bigram_analyzer", _bigram_analyzer()) index.register_tokenizer("simple_search_analyzer", _simple_search_analyzer()) @@ -79,7 +80,7 @@ def register_tokenizers(index: tantivy.Index, language: str | None) -> None: index.register_fast_field_tokenizer("simple_analyzer", _simple_analyzer()) -def _paperless_text(language: str | None) -> tantivy.TextAnalyzer: +def paperless_text_analyzer(language: str | None) -> tantivy.TextAnalyzer: """Main full-text tokenizer for content, title, etc: simple -> remove_long(129) -> lowercase -> ascii_fold [-> stemmer]""" builder = ( tantivy.TextAnalyzerBuilder(tantivy.Tokenizer.simple()) @@ -100,6 +101,54 @@ def _paperless_text(language: str | None) -> tantivy.TextAnalyzer: return builder.build() +@cache +def _pattern_stemmer(language: str | None) -> tantivy.TextAnalyzer | None: + """The stemming tail of paperless_text_analyzer, over a whole literal run. + + Same language gate and same Snowball stemmer paperless_text_analyzer + applies at index time, so query patterns follow SEARCH_LANGUAGE. Returns + None when that gate disables stemming; paperless_text_analyzer already + warns about an unsupported language, so this stays quiet. + + The raw tokenizer keeps the run whole (a wildcard literal is a fragment, + not necessarily a word), and remove_long is kept so an over-long run is + treated the same way the index treats it. + """ + if not language: + return None + tantivy_lang = _LANGUAGE_MAP.get(language.lower()) + if tantivy_lang is None: + return None + return ( + tantivy.TextAnalyzerBuilder(tantivy.Tokenizer.raw()) + .filter(tantivy.Filter.remove_long(_TOKEN_REMOVE_LONG_LIMIT)) + .filter(tantivy.Filter.stemmer(tantivy_lang)) + .build() + ) + + +def stem_pattern_text(text: str, language: str | None) -> str: + """Stem an already lowercased/ascii-folded run the way index terms are. + + Returns text unchanged when stemming is disabled for language, and also + when the stem step does not yield exactly one token: remove_long drops a run + past the length limit, leaving no stem to substitute. Falling back to the + text as typed is the safe direction for a pattern prefix, since it can only + be as narrow as it was before stemming was considered. + + The raw tokenizer emits one token whatever the input and the stemmer is + 1-to-1, so only the zero-token case can fire today; the guard covers both + counts so a tokenizer change cannot turn this into an IndexError. + """ + analyzer = _pattern_stemmer(language) + if analyzer is None: + return text + tokens = analyzer.analyze(text) + if len(tokens) != 1: + return text + return tokens[0] + + def _simple_analyzer() -> tantivy.TextAnalyzer: """Tokenizer for shadow sort fields (title_sort, correspondent_sort, type_sort): simple -> lowercase -> ascii_fold.""" return ( diff --git a/src/documents/tests/search/test_fields.py b/src/documents/tests/search/test_fields.py new file mode 100644 index 000000000..39c9d176e --- /dev/null +++ b/src/documents/tests/search/test_fields.py @@ -0,0 +1,10 @@ +from whoosh_compat import FieldKind + +from documents.search._fields import PUBLIC_FIELDS + + +class TestPublicFields: + def test_json_fields_have_subpaths(self) -> None: + for field in PUBLIC_FIELDS: + if field.kind is FieldKind.JSON: + assert field.subpaths, f"{field.name} is JSON but has no subpaths" diff --git a/src/documents/tests/search/test_json_subpath_completeness.py b/src/documents/tests/search/test_json_subpath_completeness.py new file mode 100644 index 000000000..66e3335b5 --- /dev/null +++ b/src/documents/tests/search/test_json_subpath_completeness.py @@ -0,0 +1,83 @@ +"""Every declared JSON subpath must actually be written to the index. + +PUBLIC_FIELDS declares each JSON field's subpaths (e.g. ``notes`` -> +{"user", "note"}), but nothing coupled that declaration to what +``_backend.py``'s document builder actually writes into the JSON blob at +index time. A subpath declared but never written would be +queryable-but-always-empty -- syntactically valid, silently matching +nothing -- with no test failure anywhere. + +This indexes one real document carrying values for every JSON field +(a Note, a CustomFieldInstance) and inspects the document's own stored +JSON payload, rather than running field-specific queries: that way a +future JSON field's subpaths are covered automatically, without a new +per-subpath query having to be added by hand each time. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest +import tantivy +from django.contrib.auth.models import User +from whoosh_compat import FieldKind + +from documents.models import CustomField +from documents.models import CustomFieldInstance +from documents.models import Document +from documents.models import Note +from documents.search._fields import PUBLIC_FIELDS + +if TYPE_CHECKING: + from documents.search._backend import TantivyBackend + +pytestmark = [pytest.mark.search, pytest.mark.django_db] + + +class TestJsonSubpathsAreWrittenAtIndexTime: + def test_every_declared_json_subpath_appears_in_the_stored_document( + self, + backend: TantivyBackend, + ) -> None: + user = User.objects.create_user(username="completeness-user") + field = CustomField.objects.create( + name="Completeness Field", + data_type=CustomField.FieldDataType.STRING, + ) + doc = Document.objects.create( + title="Completeness doc", + content="x", + checksum="json-subpath-completeness", + ) + Note.objects.create(document=doc, user=user, note="a note") + CustomFieldInstance.objects.create( + document=doc, + field=field, + value_text="a value", + ) + backend.add_or_update(doc) + + index = backend._index + searcher = index.searcher() + hits = searcher.search( + tantivy.Query.term_query(index.schema, "id", doc.pk), + limit=1, + ).hits + assert hits, "the document was not indexed" + stored = searcher.doc(hits[0][1]).to_dict() + + json_fields = [f for f in PUBLIC_FIELDS if f.kind is FieldKind.JSON] + assert json_fields, "no JSON fields declared - fixture is stale" + for field_spec in json_fields: + stored_values = stored.get(field_spec.name) + assert stored_values, ( + f"{field_spec.name} was not written to the index at all" + ) + written_keys = stored_values[0].keys() + for subpath in field_spec.subpaths: + assert subpath in written_keys, ( + f"{field_spec.name}.{subpath} is declared in PUBLIC_FIELDS " + "but _backend.py's document builder never writes it - it " + "would be queryable but always empty" + ) diff --git a/src/documents/tests/search/test_keyword_pattern_literal.py b/src/documents/tests/search/test_keyword_pattern_literal.py new file mode 100644 index 000000000..412a5d2e6 --- /dev/null +++ b/src/documents/tests/search/test_keyword_pattern_literal.py @@ -0,0 +1,60 @@ +"""Wildcard patterns on KEYWORD fields must stay literal. + +``checksum`` is the only KEYWORD field: it is indexed with the raw tokenizer, +so its terms are never lowercased, folded or stemmed. Running its wildcard +patterns through the stemming normalizer rewrote hex prefixes ("ceded" -> +"cede") and returned documents whose checksum did not start with what the user +typed, which for an identity field is a wrong answer. + +This covers only the registry-level normalizer, which is all that exists to +prove at this point in the stack: user queries are not yet routed through +whoosh-compat (that lands with the query-layer PR), so the same fact proven +end to end against real indexed documents lives in +``test_checksum_prefix_queries.py``. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from documents.search._registry import get_field_registry + +if TYPE_CHECKING: + from whoosh_compat import FieldRegistry + from whoosh_compat import PatternNormalizer + +pytestmark = [pytest.mark.search, pytest.mark.django_db] + + +def _normalizer(registry: FieldRegistry, name: str) -> PatternNormalizer: + ref = registry.make_ref(name) + assert ref is not None + resolved = registry.resolve(ref) + assert resolved is not None + assert resolved.spec.pattern_normalizer is not None + return resolved.spec.pattern_normalizer + + +class TestKeywordPatternNormalizer: + @pytest.mark.parametrize( + "run", + [ + pytest.param("ceded", id="stems_to_cede"), + pytest.param("added", id="stems_to_ad"), + pytest.param("cafed", id="stems_to_cafe"), + ], + ) + def test_keyword_runs_are_folded_not_stemmed(self, run: str) -> None: + """One form, the run as typed: a KEYWORD pattern must never be widened + to a stem, which would return checksums that do not start with what + the user typed.""" + normalize = _normalizer(get_field_registry("en"), "checksum") + assert normalize(run) == run + + def test_text_runs_still_offer_their_stem(self) -> None: + """A TEXT field offers the stem alongside the typed run, so a term + matching either one is reachable.""" + normalize = _normalizer(get_field_registry("en"), "title") + assert tuple(normalize("Running")) == ("running", "run") diff --git a/src/documents/tests/search/test_registry.py b/src/documents/tests/search/test_registry.py new file mode 100644 index 000000000..b591bde0b --- /dev/null +++ b/src/documents/tests/search/test_registry.py @@ -0,0 +1,149 @@ +from collections.abc import Sequence + +import pytest +from whoosh_compat import FieldKind +from whoosh_compat import FieldRegistry +from whoosh_compat.fields import ResolvedField + +from documents.search._fields import PUBLIC_FIELDS +from documents.search._registry import get_field_registry + + +@pytest.fixture +def registry() -> FieldRegistry: + return get_field_registry(None) + + +def _resolve(registry: FieldRegistry, name: str) -> ResolvedField: + ref = registry.make_ref(name) + assert ref is not None, f"{name} is not a valid field ref" + resolved = registry.resolve(ref) + assert resolved is not None, f"{name} did not resolve" + return resolved + + +def _distinct_forms(result: str | Sequence[str]) -> tuple[str, ...]: + """The forms a term may match, in order, the way whoosh-compat's emitter + reads a pattern_normalizer's answer: a bare str is one form, a sequence is + several, deduplicated.""" + if isinstance(result, str): + return (result,) + return tuple(dict.fromkeys(result)) + + +class TestFieldRegistry: + def test_internal_id_fields_are_not_registered( + self, + registry: FieldRegistry, + ) -> None: + for name in ( + "tag_id", + "owner_id", + "viewer_id", + "correspondent_id", + "document_type_id", + "storage_path_id", + "viewer_group_id", + ): + assert name not in registry + + def test_no_queryable_field_name_ends_in_id(self) -> None: + # The list above names the seven that were dropped; this catches the + # eighth. Internal *_id columns are written for permission filtering + # and joins, and whoosh only exposed them as query fields by accident, + # so a new one reaching the query surface is a leak rather than a + # feature. Checked against PUBLIC_FIELDS rather than the registry so + # an internal field is caught where it is declared. + leaked = [f.name for f in PUBLIC_FIELDS if f.name.endswith("_id")] + assert not leaked, f"internal id fields reached the query surface: {leaked}" + + def test_type_alias_resolves_to_document_type( + self, + registry: FieldRegistry, + ) -> None: + assert _resolve(registry, "type").spec.name == "document_type" + + def test_path_alias_resolves_to_storage_path(self, registry: FieldRegistry) -> None: + assert _resolve(registry, "path").spec.name == "storage_path" + + def test_notes_json_subpaths_resolve(self, registry: FieldRegistry) -> None: + resolved = _resolve(registry, "notes.user") + assert resolved.spec.name == "notes" + assert resolved.json_path == "user" + assert resolved.is_subpath is True + + def test_custom_fields_json_subpaths_resolve(self, registry: FieldRegistry) -> None: + for raw in ("custom_fields.name", "custom_fields.value"): + _resolve(registry, raw) + + def test_unregistered_json_subpath_does_not_resolve( + self, + registry: FieldRegistry, + ) -> None: + # An unregistered subpath is not even a valid FieldRef: make_ref + # returns None for a dotted name whose subpath isn't registered + # (it doesn't produce a ref for resolve() to then reject). + assert registry.make_ref("notes.bogus") is None + + def test_tag_is_comma_values(self, registry: FieldRegistry) -> None: + assert _resolve(registry, "tag").spec.comma_values is True + + def test_correspondent_is_not_comma_values(self, registry: FieldRegistry) -> None: + # "tag" is the only field that opts in. This is only observable here: + # end to end the two readings of "correspondent:foo,bar" agree, + # because the analyzer splits the literal value on the comma anyway, + # so a result-level test cannot tell a value list from literal text. + assert _resolve(registry, "correspondent").spec.comma_values is False + + def test_created_is_date_kind(self, registry: FieldRegistry) -> None: + resolved = _resolve(registry, "created") + assert resolved.spec.kind is FieldKind.DATE + assert resolved.spec.date_only is True + + def test_analyzer_lowercases_and_ascii_folds(self, registry: FieldRegistry) -> None: + # title uses the paperless_text analyzer: simple -> remove_long -> + # lowercase -> ascii_fold [-> stemmer]. With no language configured + # (None), no stemmer runs, so "Café" folds to the single token "cafe". + resolved = _resolve(registry, "title") + assert resolved.spec.analyzer is not None + assert resolved.spec.analyzer("Café") == ["cafe"] + + def test_checksum_analyzer_is_identity_single_token( + self, + registry: FieldRegistry, + ) -> None: + # checksum uses the raw tokenizer at index time (no splitting). + resolved = _resolve(registry, "checksum") + assert resolved.spec.analyzer is not None + assert resolved.spec.analyzer("ABC-123") == ["ABC-123"] + + def test_pattern_normalizer_follows_the_registry_language( + self, + registry: FieldRegistry, + ) -> None: + # Index terms are stemmed, so patterns offer their stem too, using the + # registry's own language: "Running" has to reach the indexed "run". + # Without a language the index holds surface forms, so there is no + # second form and the run is only case/accent-folded. + resolved = _resolve(registry, "title") + assert resolved.spec.pattern_normalizer is not None + assert _distinct_forms(resolved.spec.pattern_normalizer("Running")) == ( + "running", + ) + + resolved_en = _resolve(get_field_registry("en"), "title") + assert resolved_en.spec.pattern_normalizer is not None + assert _distinct_forms(resolved_en.spec.pattern_normalizer("Running")) == ( + "running", + "run", + ) + + def test_registry_is_cached_per_language(self) -> None: + a = get_field_registry("en") + b = get_field_registry("en") + assert a is b + + def test_registry_rebuilds_on_language_change(self) -> None: + a = get_field_registry("en") + b = get_field_registry("de") + assert a is not b diff --git a/src/documents/tests/search/test_schema.py b/src/documents/tests/search/test_schema.py index 7219df580..1d5a51bf3 100644 --- a/src/documents/tests/search/test_schema.py +++ b/src/documents/tests/search/test_schema.py @@ -1,12 +1,20 @@ from __future__ import annotations import json +from datetime import UTC +from datetime import datetime from typing import TYPE_CHECKING import pytest +import tantivy +from documents.search._fields import PUBLIC_FIELDS from documents.search._schema import SCHEMA_VERSION +from documents.search._schema import build_schema +from documents.search._schema import field_descriptors from documents.search._schema import needs_rebuild +from documents.search._schema import schema_fingerprint +from documents.search._tokenizer import register_tokenizers if TYPE_CHECKING: from pathlib import Path @@ -29,7 +37,13 @@ class TestNeedsRebuild: ) -> None: settings.SEARCH_LANGUAGE = "en" (index_dir / ".index_settings.json").write_text( - json.dumps({"schema_version": SCHEMA_VERSION, "language": "en"}), + json.dumps( + { + "schema_version": SCHEMA_VERSION, + "language": "en", + "schema_fingerprint": schema_fingerprint(), + }, + ), ) assert needs_rebuild(index_dir) is False @@ -76,3 +90,72 @@ class TestNeedsRebuild: json.dumps({"schema_version": SCHEMA_VERSION, "language": "en"}), ) assert needs_rebuild(index_dir) is True + + +def _schema_fields(schema: tantivy.Schema) -> dict[str, dict]: + """{name: field-state} for every field declared on a tantivy Schema. + + tantivy-py 0.26 exposes no public introspection API on Schema (no + __iter__, get_field, to_json, etc.) -- __reduce__() (used internally for + pickling) is the only way to recover the field list, so we lean on it + here for test assertions only. + """ + state = schema.__reduce__()[1][0] + return {field["name"]: field for field in state["inner"]} + + +class TestSchemaMatchesPublicFields: + def test_every_public_field_is_in_the_schema(self) -> None: + schema = build_schema() + schema_field_names = set(_schema_fields(schema)) + for field in PUBLIC_FIELDS: + assert field.name in schema_field_names, ( + f"{field.name} is in PUBLIC_FIELDS but missing from build_schema()" + ) + + def test_asn_page_count_num_notes_are_fast_unsigned_fields(self) -> None: + # Spot-check kind-derived construction for the U64 fields. + schema = build_schema() + doc = tantivy.Document() + doc.add_unsigned("id", 1) + doc.add_text("checksum", "x") + doc.add_unsigned("asn", 42) + doc.add_unsigned("page_count", 3) + doc.add_unsigned("num_notes", 0) + doc.add_date("created", datetime(2020, 1, 1, tzinfo=UTC)) + doc.add_date("modified", datetime(2020, 1, 1, tzinfo=UTC)) + doc.add_date("added", datetime(2020, 1, 1, tzinfo=UTC)) + index = tantivy.Index(schema) + register_tokenizers(index, None) + writer = index.writer() + writer.add_document(doc) + writer.commit() + index.reload() + searcher = index.searcher() + results = searcher.search(tantivy.Query.term_query(schema, "asn", 42), limit=1) + assert len(results.hits) == 1 + + +class TestFastFlagAgreement: + def test_every_public_field_fast_flag_matches_the_built_schema(self) -> None: + # whoosh-compat's registry trusts PUBLIC_FIELDS' fast flag when resolving + # field:* existence checks (its FAST_FIELD strategy); a fast=True + # entry whose actual tantivy column is not fast would make those + # searches silently match nothing at search time. Only the U64 and + # DATE descriptors can carry the flag today, so this + # pins the agreement for EVERY kind: a future fast=True + # TEXT/KEYWORD/JSON entry the builder silently ignores fails here + # instead of at a user's query. + # + # field_descriptors() (not tantivy-py's __reduce__() pickling + # internals) is used as the probe here: it is exactly the input + # build_schema()'s SchemaBuilder consumes for the `fast` kwarg on + # every field kind, so it pins the same agreement without depending + # on a private pickled representation surviving a tantivy-py + # upgrade. + descriptor_fast = {d.name: d.fast for d in field_descriptors()} + for public_field in PUBLIC_FIELDS: + assert descriptor_fast[public_field.name] == public_field.fast, ( + f"{public_field.name}: PUBLIC_FIELDS says fast={public_field.fast} but" + f" field_descriptors() says fast={descriptor_fast[public_field.name]}" + ) diff --git a/src/documents/tests/search/test_tokenizer.py b/src/documents/tests/search/test_tokenizer.py index bd03cdf09..0bcf1ce3a 100644 --- a/src/documents/tests/search/test_tokenizer.py +++ b/src/documents/tests/search/test_tokenizer.py @@ -7,8 +7,8 @@ import pytest import tantivy from documents.search._tokenizer import _bigram_analyzer -from documents.search._tokenizer import _paperless_text from documents.search._tokenizer import _simple_search_analyzer +from documents.search._tokenizer import paperless_text_analyzer from documents.search._tokenizer import register_tokenizers if TYPE_CHECKING: @@ -25,7 +25,7 @@ class TestTokenizers: sb.add_text_field("content", stored=True, tokenizer_name="paperless_text") schema = sb.build() idx = tantivy.Index(schema, path=None) - idx.register_tokenizer("paperless_text", _paperless_text("")) + idx.register_tokenizer("paperless_text", paperless_text_analyzer("")) return idx @pytest.fixture diff --git a/uv.lock b/uv.lock index 3c08f0f55..3294f96e6 100644 --- a/uv.lock +++ b/uv.lock @@ -4,11 +4,11 @@ requires-python = ">=3.11" resolution-markers = [ "python_full_version >= '3.15' and sys_platform == 'darwin'", "python_full_version >= '3.15' and sys_platform == 'linux'", + "python_full_version == '3.14.*' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "python_full_version == '3.14.*' and platform_machine == 'aarch64' and sys_platform == 'linux'", "python_full_version >= '3.12' and python_full_version < '3.15' and sys_platform == 'darwin'", "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'", "python_full_version == '3.12.*' and platform_machine == 'aarch64' and sys_platform == 'linux'", - "python_full_version == '3.14.*' and platform_machine == 'x86_64' and sys_platform == 'linux'", - "python_full_version == '3.14.*' and platform_machine == 'aarch64' and sys_platform == 'linux'", "(python_full_version >= '3.12' and python_full_version < '3.15' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version == '3.13.*' and platform_machine == 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.13.*' and platform_machine == 'x86_64' and sys_platform == 'linux')", "python_full_version < '3.12' and sys_platform == 'darwin'", "python_full_version < '3.12' and sys_platform == 'linux'", @@ -2933,6 +2933,7 @@ dependencies = [ { name = "torch", version = "2.13.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform == 'linux'" }, { name = "watchfiles" }, { name = "whitenoise" }, + { name = "whoosh-compat", extra = ["tantivy"] }, { name = "zxing-cpp" }, ] @@ -3091,6 +3092,7 @@ requires-dist = [ { name = "torch", specifier = "~=2.13.0", index = "https://download.pytorch.org/whl/cpu" }, { name = "watchfiles", specifier = ">=1.2" }, { name = "whitenoise", specifier = "~=6.11" }, + { name = "whoosh-compat", extras = ["tantivy"], specifier = "==0.1.0" }, { name = "zxing-cpp", specifier = "~=3.1.0" }, ] provides-extras = ["mariadb", "postgres", "webserver"] @@ -5014,10 +5016,10 @@ version = "2.13.0+cpu" source = { registry = "https://download.pytorch.org/whl/cpu" } resolution-markers = [ "python_full_version >= '3.15' and sys_platform == 'linux'", - "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'", - "python_full_version == '3.12.*' and platform_machine == 'aarch64' and sys_platform == 'linux'", "python_full_version == '3.14.*' and platform_machine == 'x86_64' and sys_platform == 'linux'", "python_full_version == '3.14.*' and platform_machine == 'aarch64' and sys_platform == 'linux'", + "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "python_full_version == '3.12.*' and platform_machine == 'aarch64' and sys_platform == 'linux'", "(python_full_version >= '3.12' and python_full_version < '3.15' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version == '3.13.*' and platform_machine == 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.13.*' and platform_machine == 'x86_64' and sys_platform == 'linux')", "python_full_version < '3.12' and sys_platform == 'linux'", ] @@ -5652,6 +5654,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/db/eb/d5583a11486211f3ebd4b385545ae787f32363d453c19fffd81106c9c138/whitenoise-6.12.0-py3-none-any.whl", hash = "sha256:fc5e8c572e33ebf24795b47b6a7da8da3c00cff2349f5b04c02f28d0cc5a3cc2", size = 20302, upload-time = "2026-02-27T00:05:40.086Z" }, ] +[[package]] +name = "whoosh-compat" +version = "0.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dateutil" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5d/f7/3e45f4a484afa174cd42e424ce2b8c514ae54f564830122656ad17b765e2/whoosh_compat-0.1.0.tar.gz", hash = "sha256:86935bdc159ed9b0a06a4661d17f1251d8280340b84e218cc73d915a2edaddf7", size = 577543, upload-time = "2026-08-25T15:22:42.316Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/51/9a8399d0f472814e136a2884bf910c6c59843a5a2c66afc4b5133eb531ea/whoosh_compat-0.1.0-py3-none-any.whl", hash = "sha256:3e7c5f519b4d397dbf4f8d7bbe4ca1eb6004da24c8892bc701844ed393bc97e7", size = 153875, upload-time = "2026-08-25T15:22:40.787Z" }, +] + +[package.optional-dependencies] +tantivy = [ + { name = "tantivy" }, +] + [[package]] name = "wrapt" version = "2.0.1"