diff --git a/src/documents/search/_backend.py b/src/documents/search/_backend.py index 8c47ae817..9bac95ba5 100644 --- a/src/documents/search/_backend.py +++ b/src/documents/search/_backend.py @@ -506,7 +506,6 @@ class TantivyBackend: doc.add_text("correspondent_sort", document.correspondent.name) if cjk_corr := extract_cjk_text(document.correspondent.name): doc.add_text("bigram_correspondent", cjk_corr) - doc.add_unsigned("correspondent_id", document.correspondent_id) # Document type if document.document_type: @@ -514,12 +513,10 @@ class TantivyBackend: doc.add_text("type_sort", document.document_type.name) if cjk_type := extract_cjk_text(document.document_type.name): doc.add_text("bigram_document_type", cjk_type) - doc.add_unsigned("document_type_id", document.document_type_id) # Storage path if document.storage_path: doc.add_text("storage_path", document.storage_path.name) - doc.add_unsigned("storage_path_id", document.storage_path_id) # Tags — collect names for autocomplete in the same pass tag_names: list[str] = [] @@ -527,7 +524,6 @@ class TantivyBackend: doc.add_text("tag", tag.name) if cjk_tag := extract_cjk_text(tag.name): doc.add_text("bigram_tag", cjk_tag) - doc.add_unsigned("tag_id", tag.pk) tag_names.append(tag.name) # Notes — JSON for structured queries (notes.user:alice, notes.note:text). diff --git a/src/documents/search/_schema.py b/src/documents/search/_schema.py index e850a2e98..6a6c7bc60 100644 --- a/src/documents/search/_schema.py +++ b/src/documents/search/_schema.py @@ -19,7 +19,11 @@ if TYPE_CHECKING: logger = logging.getLogger("paperless.search") # v1 - Initial tantivy schema format -SCHEMA_VERSION: Final[int] = 1 +# v2 - build_schema() derived from PUBLIC_FIELDS, changing the field declaration +# order, and the write-only correspondent/document_type/storage_path/tag id +# columns dropped. tantivy compares schemas by ordered field list, so an +# index built by v1 rejects every write against the v2 schema. +SCHEMA_VERSION: Final[int] = 2 def build_schema() -> tantivy.Schema: @@ -109,15 +113,8 @@ def build_schema() -> tantivy.Schema: # The stored value is never read back, so storing it only wastes space. sb.add_text_field("autocomplete_word", stored=False, tokenizer_name="raw") - for field in ( - "correspondent_id", - "document_type_id", - "storage_path_id", - "tag_id", - "owner_id", - "viewer_id", - "viewer_group_id", - ): + # Permission filter columns, read by build_permission_filter. + for field in ("owner_id", "viewer_id", "viewer_group_id"): sb.add_unsigned_field(field, stored=False, indexed=True, fast=True) return sb.build() diff --git a/src/documents/tests/search/test_permission_field_isolation.py b/src/documents/tests/search/test_permission_field_isolation.py new file mode 100644 index 000000000..dd196ef44 --- /dev/null +++ b/src/documents/tests/search/test_permission_field_isolation.py @@ -0,0 +1,148 @@ +"""Permission filtering must hold against the real indexed document shape. + +Only three of the index's unsigned ``*_id`` columns are load-bearing: +``owner_id``, ``viewer_id`` and ``viewer_group_id``, all read by +build_permission_filter. The rest (correspondent/document_type/storage_path/tag +ids) were written on every document and read by nothing, and were dropped. + +These tests index real Documents through the backend's own document builder and +assert result-level visibility per user, so a mistake about which columns are +load-bearing shows up as documents leaking across users rather than as a passing +unit test over a hand-built index. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest +from django.contrib.auth.models import Group +from django.contrib.auth.models import User +from guardian.shortcuts import assign_perm + +from documents.models import Correspondent +from documents.models import Document +from documents.models import DocumentType +from documents.models import StoragePath +from documents.models import Tag + +if TYPE_CHECKING: + from documents.search._backend import TantivyBackend + +pytestmark = [pytest.mark.search, pytest.mark.django_db] + + +@pytest.fixture +def owner() -> User: + return User.objects.create_user(username="owner") + + +@pytest.fixture +def stranger() -> User: + return User.objects.create_user(username="stranger") + + +@pytest.fixture +def viewer() -> User: + return User.objects.create_user(username="viewer") + + +@pytest.fixture +def group_member() -> User: + user = User.objects.create_user(username="group_member") + user.groups.add(Group.objects.create(name="accounting")) + return user + + +class TestPermissionFilteringOnIndexedDocuments: + def test_unowned_document_is_visible_to_everyone( + self, + backend: TantivyBackend, + stranger: User, + ) -> None: + doc = Document.objects.create( + title="Public Invoice", + content="invoice total due", + checksum="perm-unowned", + ) + backend.add_or_update(doc) + + assert backend.search_ids("invoice", user=stranger) == [doc.pk] + + def test_owned_document_is_visible_only_to_its_owner( + self, + backend: TantivyBackend, + owner: User, + stranger: User, + ) -> None: + doc = Document.objects.create( + title="Private Invoice", + content="invoice total due", + checksum="perm-owned", + owner=owner, + ) + backend.add_or_update(doc) + + assert backend.search_ids("invoice", user=owner) == [doc.pk] + assert backend.search_ids("invoice", user=stranger) == [] + + def test_explicitly_shared_document_is_visible_to_the_viewer( + self, + backend: TantivyBackend, + owner: User, + viewer: User, + stranger: User, + ) -> None: + doc = Document.objects.create( + title="Shared Invoice", + content="invoice total due", + checksum="perm-shared-user", + owner=owner, + ) + assign_perm("view_document", viewer, doc) + backend.add_or_update(doc) + + assert backend.search_ids("invoice", user=viewer) == [doc.pk] + assert backend.search_ids("invoice", user=stranger) == [] + + def test_group_shared_document_is_visible_to_group_members( + self, + backend: TantivyBackend, + owner: User, + group_member: User, + stranger: User, + ) -> None: + doc = Document.objects.create( + title="Group Invoice", + content="invoice total due", + checksum="perm-shared-group", + owner=owner, + ) + assign_perm("view_document", group_member.groups.first(), doc) + backend.add_or_update(doc) + + assert backend.search_ids("invoice", user=group_member) == [doc.pk] + assert backend.search_ids("invoice", user=stranger) == [] + + def test_metadata_does_not_widen_visibility( + self, + backend: TantivyBackend, + owner: User, + stranger: User, + ) -> None: + """A document carrying correspondent/type/storage-path/tag metadata is + still filtered by owner alone.""" + doc = Document.objects.create( + title="Tagged Invoice", + content="invoice total due", + checksum="perm-metadata", + owner=owner, + correspondent=Correspondent.objects.create(name="ACME"), + document_type=DocumentType.objects.create(name="Bill"), + storage_path=StoragePath.objects.create(name="Archive", path="archive/"), + ) + doc.tags.add(Tag.objects.create(name="paid")) + backend.add_or_update(doc) + + assert backend.search_ids("invoice", user=owner) == [doc.pk] + assert backend.search_ids("invoice", user=stranger) == [] diff --git a/src/documents/tests/search/test_schema_version.py b/src/documents/tests/search/test_schema_version.py new file mode 100644 index 000000000..902256c48 --- /dev/null +++ b/src/documents/tests/search/test_schema_version.py @@ -0,0 +1,164 @@ +"""SCHEMA_VERSION must change whenever build_schema()'s field list or order does. + +tantivy compares schemas by *ordered* field list. ``Index.open()`` loads the +schema from the index's own ``meta.json``, so reads against an index built by an +older release keep working after a field reorder. Writes do not: +``WriteBatch.__enter__`` calls ``tantivy.Index(build_schema(), path=...)``, an +open-or-create that raises ``ValueError`` on any schema difference. Nothing +catches that ValueError, so consumption, index_document and bulk edit all +hard-fail while ``/api/status/`` still reports the index healthy. + +The only thing that saves such an install is ``needs_rebuild()`` noticing the +version stamped in ``.index_settings.json`` is stale. +""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING + +import pytest +import tantivy +from django.conf import settings as django_settings + +from documents.search._schema import build_schema +from documents.search._schema import needs_rebuild +from documents.search._schema import open_or_rebuild_index + +if TYPE_CHECKING: + from pathlib import Path + +pytestmark = [pytest.mark.search] + +RELEASED_V1_SCHEMA_VERSION = 1 + + +def _build_released_v1_schema() -> tantivy.Schema: + """Frozen copy of build_schema() as shipped in v3.0.x (schema version 1). + + Deliberately duplicated rather than imported: it must keep describing the + on-disk layout of already-deployed indexes even as build_schema() evolves. + """ + sb = tantivy.SchemaBuilder() + + sb.add_unsigned_field("id", stored=True, indexed=True, fast=True) + sb.add_text_field("checksum", stored=True, tokenizer_name="raw") + + for field in ( + "title", + "correspondent", + "document_type", + "storage_path", + "original_filename", + "content", + ): + sb.add_text_field(field, stored=True, tokenizer_name="paperless_text") + + for field in ("title_sort", "correspondent_sort", "type_sort"): + sb.add_text_field( + field, + stored=False, + tokenizer_name="simple_analyzer", + fast=True, + ) + + for field in ( + "bigram_content", + "bigram_title", + "bigram_correspondent", + "bigram_document_type", + "bigram_tag", + ): + sb.add_text_field(field, stored=False, tokenizer_name="bigram_analyzer") + + for field in ("simple_title", "simple_content"): + sb.add_text_field(field, stored=False, tokenizer_name="simple_search_analyzer") + + sb.add_text_field("autocomplete_word", stored=False, tokenizer_name="raw") + sb.add_text_field("tag", stored=True, tokenizer_name="paperless_text") + + sb.add_json_field("notes", stored=True, tokenizer_name="paperless_text") + sb.add_text_field("notes_text", stored=True, tokenizer_name="paperless_text") + sb.add_json_field("custom_fields", stored=True, tokenizer_name="paperless_text") + + for field in ( + "correspondent_id", + "document_type_id", + "storage_path_id", + "tag_id", + "owner_id", + "viewer_id", + "viewer_group_id", + ): + sb.add_unsigned_field(field, stored=False, indexed=True, fast=True) + + for field in ("created", "modified", "added"): + sb.add_date_field(field, stored=True, indexed=True, fast=True) + + for field in ("asn", "page_count", "num_notes"): + sb.add_unsigned_field(field, stored=True, indexed=True, fast=True) + + return sb.build() + + +@pytest.fixture +def released_v1_index(tmp_path: Path) -> Path: + """An index directory as a v3.0.x install would leave it on disk.""" + index_dir = tmp_path / "index" + index_dir.mkdir() + tantivy.Index(_build_released_v1_schema(), path=str(index_dir)) + (index_dir / ".index_settings.json").write_text( + json.dumps( + { + "schema_version": RELEASED_V1_SCHEMA_VERSION, + "language": django_settings.SEARCH_LANGUAGE, + }, + ), + ) + return index_dir + + +class TestUpgradeFromReleasedV1Index: + def test_released_v1_index_is_flagged_for_rebuild( + self, + released_v1_index: Path, + ) -> None: + """The current schema differs from v1's, so the sentinel must be stale. + + If this fails, `document_index reindex --if-needed` prints "Search index + is up to date" and skips, leaving the mismatched index in place. + """ + assert needs_rebuild(released_v1_index) is True + + def test_v1_index_rejects_writes_against_the_current_schema( + self, + released_v1_index: Path, + ) -> None: + """The failure mode the version bump exists to prevent. + + This is exactly what WriteBatch.__enter__ does on every index write. + """ + with pytest.raises(ValueError, match="schema does not match"): + tantivy.Index(build_schema(), path=str(released_v1_index)) + + def test_opening_a_v1_index_leaves_it_writable( + self, + released_v1_index: Path, + ) -> None: + """End to end: open_or_rebuild_index must hand back an index that the + write path can reopen. Before the version bump, needs_rebuild() returned + False here, the stale directory survived untouched, and every subsequent + write raised the ValueError above.""" + open_or_rebuild_index(released_v1_index) + + tantivy.Index(build_schema(), path=str(released_v1_index)) + + def test_rebuilt_index_is_not_rebuilt_again( + self, + released_v1_index: Path, + ) -> None: + """The rebuild must stamp the version it actually wrote, otherwise every + startup wipes and reindexes the whole corpus.""" + open_or_rebuild_index(released_v1_index) + + assert needs_rebuild(released_v1_index) is False