Performance: Tantivy indexing optimization (#13053)

* Tantivy: get permissions by chunks

-40% indexing time compared to previous commit

* Make progress bar process one by one with chunk

-15% indexing time compared to previous commit

* Prefetch FK + iterate over chunk from SQL

Prefetch additional needed data (note user, custom field content)

-20% indexing time compared to previous commit

* Reindex: increase Tantivy heap size from 128 to 512MB

Gains probably vary depending on the machine,
but it seems a sweet spot compatible with low-end hardware.

* Reindex: optimization on permission fetching and autocomplete word set

-10% indexing time compared to previous commit

* Autocomplete analyzer python->rust

Splits words with underscore compared to the python analyzer.
E.g.: "blue_print" -> ["blue", "print"]
It can still be found with the "blue_print" keyword,
as the search string is also split in two words.

-50% indexing time compared to previous commit (indexing is twice faster!)

* Index bigram for CJK content only

Inedxing time slightly longer (~3%),
but since the non-CJK content is not indexed,
bigram searchs will be slightly optimized.

* Fix group-based view_document permissions missing from bulk rebuild

_bulk_get_viewer_ids only queried UserObjectPermission, dropping the
group-permission expansion that get_users_with_perms(with_group_users=True)
performs for the non-batched per-document indexing path. A user who could
only see a document via group membership would lose search access to it
after any full reindex.

Also query GroupObjectPermission and expand group membership to user ids,
matching the existing single-document behavior.

* Yield (document, viewer_ids) pairs from _DocumentViewerStream

Previously _DocumentViewerStream.__iter__ yielded plain Document objects
while the matching viewer ids were exposed through a separate mutable
attribute (viewer_ids_by_pk), overwritten each time the generator crossed
a chunk boundary. rebuild() read that attribute out-of-band per document.

This only worked because the current iter_wrapper (a plain progress-bar
passthrough) happens to consume the stream in strict lock-step with no
lookahead. Any wrapper that buffers, batches, or reorders would silently
pair a document with the wrong chunk's viewer ids. Yield the pair directly
so the association travels with the document regardless of how iter_wrapper
consumes the stream, and drop the now-unneeded viewer_ids_by_pk attribute.

* Add --heap-size-mb CLI arg to document_index reindex

writer_heap_bytes was hardcoded at 512MB with no way to tune it. Expose it
as a manual-rebuild-only CLI arg rather than a settings/env var, per review
feedback, so lower-memory hosts can reduce it without a wider config
surface. Defaults to unset so TantivyBackend.rebuild's own default stays
the single source of truth.

---------

Co-authored-by: stumpylog <797416+stumpylog@users.noreply.github.com>

src/documents/tests/search/test_backend.py

@@ -1,5 +1,7 @@
 import pytest
+from django.contrib.auth.models import Group
 from django.contrib.auth.models import User
+from guardian.shortcuts import assign_perm
 from pytest_mock import MockerFixture
 
 from documents.models import CustomField
@@ -15,6 +17,7 @@ from documents.tests.factories import CorrespondentFactory
 from documents.tests.factories import DocumentFactory
 from documents.tests.factories import DocumentTypeFactory
 from documents.tests.factories import TagFactory
+from documents.tests.factories import UserFactory
 
 pytestmark = [pytest.mark.search, pytest.mark.django_db]
 
@@ -561,18 +564,48 @@ class TestRebuild:
     """Test index rebuilding functionality."""
 
     def test_with_iter_wrapper_called(self, backend: TantivyBackend) -> None:
-        """Index rebuild must pass documents through iter_wrapper for progress tracking."""
+        """Index rebuild must pass (document, viewer_ids) pairs through iter_wrapper."""
         seen = []
 
-        def wrapper(docs):
-            for doc in docs:
+        def wrapper(pairs):
+            for doc, viewer_ids in pairs:
                 seen.append(doc.pk)
-                yield doc
+                yield doc, viewer_ids
 
         Document.objects.create(title="Tracked", content="x", checksum="TW1", pk=30)
         backend.rebuild(Document.objects.all(), iter_wrapper=wrapper)
         assert 30 in seen
 
+    def test_includes_group_granted_viewers(self, backend: TantivyBackend) -> None:
+        """Rebuild must index viewer ids for group-only grants, not just direct ones.
+
+        The batched viewer-id lookup used during rebuild() must mirror
+        get_users_with_perms(with_group_users=True), which is the default
+        used by the non-batched per-document indexing path. Without it, a
+        user who can only see a document via group membership would lose
+        search access to it after any full reindex.
+        """
+        owner = UserFactory()
+        group_member = UserFactory()
+        group = Group.objects.create(name="viewers")
+        group_member.groups.add(group)
+
+        doc = DocumentFactory(
+            title="Group shared doc",
+            content="group secret keyword",
+            owner=owner,
+        )
+        assign_perm("view_document", group, doc)
+
+        backend.rebuild(Document.objects.all())
+
+        ids = backend.search_ids(
+            "group secret",
+            user=group_member,
+            search_mode=SearchMode.QUERY,
+        )
+        assert ids == [doc.pk]
+
 
 class TestAutocomplete:
     """Test autocomplete functionality."""

src/documents/tests/test_management.py

@@ -176,6 +176,35 @@ class TestMakeIndex:
         call_command("document_index", "reindex", if_needed=True, skip_checks=True)
         mock_get_backend.return_value.rebuild.assert_called_once()
 
+    def test_reindex_default_heap_size_not_overridden(
+        self,
+        mocker: MockerFixture,
+    ) -> None:
+        """Without --heap-size-mb, rebuild must use its own default heap size."""
+        mock_get_backend = mocker.patch(
+            "documents.management.commands.document_index.get_backend",
+        )
+        call_command("document_index", "reindex", skip_checks=True)
+        _, kwargs = mock_get_backend.return_value.rebuild.call_args
+        assert "writer_heap_bytes" not in kwargs
+
+    def test_reindex_heap_size_mb_passed_to_rebuild(
+        self,
+        mocker: MockerFixture,
+    ) -> None:
+        """--heap-size-mb must convert to bytes and pass through to rebuild."""
+        mock_get_backend = mocker.patch(
+            "documents.management.commands.document_index.get_backend",
+        )
+        call_command(
+            "document_index",
+            "reindex",
+            heap_size_mb=128,
+            skip_checks=True,
+        )
+        _, kwargs = mock_get_backend.return_value.rebuild.call_args
+        assert kwargs["writer_heap_bytes"] == 128_000_000
+
 
 @pytest.mark.management
 class TestRenamer(DirectoriesMixin, FileSystemAssertsMixin, TestCase):