Chore: use document type ids in the bulk edit selection_data test

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

src-ui/messages.xlf

@@ -10334,18 +10334,18 @@
           <context context-type="linenumber">294,295</context>
         </context-group>
       </trans-unit>
-      <trans-unit id="3420321797707163677" datatype="html">
+      <trans-unit id="2179847500064178686" datatype="html">
-        <source>Preview document</source>
+        <source>Edit document</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/document-list/document-list.component.html</context>
           <context context-type="linenumber">328</context>
         </context-group>
       </trans-unit>
-      <trans-unit id="2179847500064178686" datatype="html">
+      <trans-unit id="3420321797707163677" datatype="html">
-        <source>Edit document</source>
+        <source>Preview document</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/document-list/document-list.component.html</context>
-          <context context-type="linenumber">331</context>
+          <context context-type="linenumber">329</context>
         </context-group>
       </trans-unit>
       <trans-unit id="4512084577073831437" datatype="html">

src-ui/src/app/components/common/custom-fields-dropdown/custom-fields-dropdown.component.scss

@@ -1,3 +1,10 @@
 .custom-fields-dropdown {
     min-width: 300px;
+
+    // correct position on mobile
+    @media (max-width: 575.98px) {
+        &.show {
+            margin-left: -245px !important;
+        }
+    }
 }

src-ui/src/app/components/document-detail/document-detail.component.html

@@ -308,7 +308,7 @@
                   </tr>
                   <tr class="align-middle">
                     <td i18n>Original SHA256 checksum</td>
-                    <td><span class="badge bg-light text-dark font-monospace text-wrap text-break text-start">{{metadata()?.original_checksum}}</span></td>
+                    <td><span class="badge bg-light font-monospace text-wrap text-break text-start">{{metadata()?.original_checksum}}</span></td>
                   </tr>
                   <tr>
                     <td i18n>Original file size</td>
@@ -465,7 +465,7 @@
     @switch (archiveContentRenderType) {
       @case (ContentRenderType.PDF) {
         @if (!useNativePdfViewer) {
-          <div class="preview-sticky pdf-viewer-container" tabindex="-1">
+          <div class="preview-sticky pdf-viewer-container">
             <pngx-pdf-viewer
               [src]="pdfSource()"
               [sourceRevision]="previewRevision()"
@@ -477,7 +477,7 @@
               (loadError)="onError($event)"
               (afterLoadComplete)="pdfPreviewLoaded($event)">
             </pngx-pdf-viewer>
-            <div class="preview-controls position-absolute bottom-0 start-50 translate-middle-x z-1 mb-3 d-flex gap-2 p-1 bg-body border rounded shadow">
+            <div class="position-absolute bottom-0 start-50 translate-middle-x z-1 mb-3 d-flex gap-2 p-1 bg-body border rounded shadow">
               @if (previewNumPages()) {
                 <div class="input-group input-group-sm w-auto flex-nowrap">
                   <input class="form-control flex-grow-0 w-auto" type="number" min="1" [max]="previewNumPages()" [ngModelOptions]="{standalone: true}" [ngModel]="previewCurrentPage()" (ngModelChange)="previewCurrentPage.set($event)" aria-label="Page" i18n-aria-label />

src-ui/src/app/components/document-detail/document-detail.component.scss

@@ -12,22 +12,6 @@
     width: 100%;
     height: 100%;
   }
-
-  &:focus {
-    outline: none;
-  }
-
-  .preview-controls {
-    opacity: 0;
-    pointer-events: none;
-    transition: opacity 0.15s ease-in-out;
-  }
-
-  &:hover .preview-controls,
-  &:focus-within .preview-controls {
-    opacity: 1;
-    pointer-events: auto;
-  }
 }
 
 .btn-group .dropdown-toggle-split {

src-ui/src/app/components/document-list/document-list.component.html

@@ -208,7 +208,7 @@
                   i18n>Correspondent</th>
               }
               @if (activeDisplayFields.includes(DisplayField.TITLE)) {
-                <th class="cursor-pointer ps-5"
+                <th class="cursor-pointer"
                   pngxSortable="title"
                   title="Sort by title" i18n-title
                   [currentSortField]="list.sortField"
@@ -322,13 +322,13 @@
                   </td>
                 }
                 @if (activeDisplayFields.includes(DisplayField.TITLE) || activeDisplayFields.includes(DisplayField.TAGS)) {
-                  <td width="30%" class="position-relative" [class.ps-5]="activeDisplayFields.includes(DisplayField.TITLE)">
+                  <td width="30%">
                     @if (activeDisplayFields.includes(DisplayField.TITLE)) {
                       <div class="d-inline-block" (mouseleave)="popupPreview.close()">
-                        <pngx-preview-popup class="position-absolute start-0" [document]="d" linkClasses="btn btn-sm btn-link text-secondary" linkTitle="Preview document" (click)="$event.stopPropagation()" i18n-linkTitle #popupPreview>
+                        <a routerLink="/documents/{{d.id}}" title="Edit document" i18n-title style="overflow-wrap: anywhere;">{{d.title | documentTitle}}</a>
+                        <pngx-preview-popup [document]="d" linkClasses="btn btn-sm btn-link text-secondary" linkTitle="Preview document" (click)="$event.stopPropagation()" i18n-linkTitle #popupPreview>
                           <i-bs name="eye"></i-bs>
                         </pngx-preview-popup>
-                        <a routerLink="/documents/{{d.id}}" title="Edit document" i18n-title style="overflow-wrap: anywhere;">{{d.title | documentTitle}}</a>
                       </div>
                     }
                     @if (activeDisplayFields.includes(DisplayField.TAGS)) {

src-ui/src/app/components/document-list/document-list.component.scss

@@ -87,10 +87,6 @@ $paperless-card-breakpoints: (
   }
 }
 
-.table .ps-5 {
-  padding-left: 2.2rem !important;
-}
-
 a {
   cursor: pointer;
 }

src/documents/permissions.py

@@ -576,12 +576,9 @@ def annotate_document_count_by_ids(
                        or CustomFieldInstance)
         related_object_field: field on the relation pointing back to queryset pk
         document_ids: the document ids to count against -- a concrete list/set,
-                       or a simple (already resolved) queryset of ids. Callers
+                       or a simple (already resolved) queryset of ids. Keep
-                       that need this filtered by a complex condition (e.g. a
+                       the list small: it is embedded in the statement as
-                       permission check) should resolve it to a concrete list
+                       literal parameters.
-                       first if the same ids will be reused across multiple
-                       calls, rather than passing the complex queryset itself
-                       into each -- see ``_get_selection_data_for_queryset``.
         target_field: field on the relation pointing to Document id
     """
 

src/documents/tests/test_api_bulk_edit.py

@@ -1141,8 +1141,8 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.assertCountEqual(
             response.data["selected_document_types"],
             [
-                {"id": self.c1.id, "document_count": 1},
+                {"id": self.dt1.id, "document_count": 1},
-                {"id": self.c2.id, "document_count": 0},
+                {"id": self.dt2.id, "document_count": 0},
             ],
         )
 

src/documents/tests/test_api_documents.py

@@ -49,6 +49,7 @@ from documents.models import WorkflowTrigger
 from documents.search import get_backend
 from documents.signals.handlers import run_workflows
 from documents.tests.utils import ConsumeTaskMixin
+from documents.views import DocumentViewSet
 from paperless_testing.dirs import DirectoriesMixin
 from paperless_testing.factories import DocumentFactory
 from paperless_testing.factories import TagFactory
@@ -1755,6 +1756,106 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         }
         self.assertEqual(counts_by_field[custom_field.id], 1)
 
+    def test_selection_data_counts_with_shared_by_filter(self) -> None:
+        """
+        GIVEN:
+            - Several documents owned by one user with the same correspondent
+              and document type, only one of which is shared
+        WHEN:
+            - Requesting the document list filtered by shared_by__id with
+              include_selection_data=true
+        THEN:
+            - Counts reflect only the shared document, not every document the
+              owner has with that correspondent or document type, however the
+              link-table lookups are batched
+        """
+        owner = User.objects.create_user(username="owner")
+        other = User.objects.create_user(username="other")
+        correspondent = Correspondent.objects.create(name="c1")
+        doc_type = DocumentType.objects.create(name="dt1")
+        tag = Tag.objects.create(name="tag")
+        docs = [
+            Document.objects.create(
+                checksum=str(i),
+                owner=owner,
+                correspondent=correspondent,
+                document_type=doc_type,
+            )
+            for i in range(3)
+        ]
+        for doc in docs:
+            doc.tags.add(tag)
+        grant_object(other, docs[0], "view_document")
+
+        for batch_size in (10_000, 1):
+            with (
+                self.subTest(link_batch_size=batch_size),
+                mock.patch.object(
+                    DocumentViewSet,
+                    "selection_data_link_batch_size",
+                    batch_size,
+                ),
+            ):
+                response = self.client.get(
+                    f"/api/documents/?shared_by__id={owner.id}"
+                    "&include_selection_data=true",
+                )
+
+                self.assertEqual(response.status_code, status.HTTP_200_OK)
+                self.assertEqual(response.data["count"], 1)
+                selection_data = response.data["selection_data"]
+                for key, obj in (
+                    ("selected_correspondents", correspondent),
+                    ("selected_document_types", doc_type),
+                    ("selected_tags", tag),
+                ):
+                    counts = {
+                        item["id"]: item["document_count"]
+                        for item in selection_data[key]
+                    }
+                    self.assertEqual(counts[obj.id], 1, key)
+
+    def test_selection_data_counts_documents_once_when_filter_joins(self) -> None:
+        """
+        GIVEN:
+            - A document matching a multi-valued filter through two tags
+        WHEN:
+            - Requesting the document list with include_selection_data=true
+        THEN:
+            - The document is counted once per correspondent, not once per
+              matching tag, however the link-table lookups are batched
+        """
+        correspondent = Correspondent.objects.create(name="c1")
+        tag_a = Tag.objects.create(name="a")
+        tag_b = Tag.objects.create(name="b")
+        both = Document.objects.create(checksum="1", correspondent=correspondent)
+        both.tags.add(tag_a, tag_b)
+        only_a = Document.objects.create(checksum="2", correspondent=correspondent)
+        only_a.tags.add(tag_a)
+
+        for batch_size in (10_000, 1):
+            with (
+                self.subTest(link_batch_size=batch_size),
+                mock.patch.object(
+                    DocumentViewSet,
+                    "selection_data_link_batch_size",
+                    batch_size,
+                ),
+            ):
+                response = self.client.get(
+                    f"/api/documents/?tags__id__in={tag_a.id},{tag_b.id}"
+                    "&include_selection_data=true",
+                )
+
+                self.assertEqual(response.status_code, status.HTTP_200_OK)
+                counts = {
+                    item["id"]: item["document_count"]
+                    for item in response.data["selection_data"][
+                        "selected_correspondents"
+                    ]
+                }
+                self.assertEqual(counts[correspondent.id], 2)
+
     def test_statistics(self) -> None:
         doc1 = Document.objects.create(
             title="none1",

src/documents/views.py

@@ -5,6 +5,7 @@ import platform
 import re
 import tempfile
 import zipfile
+from collections import Counter
 from collections import deque
 from collections.abc import Callable
 from datetime import datetime
@@ -168,7 +169,6 @@ from documents.permissions import PaperlessNotePermissions
 from documents.permissions import PaperlessObjectPermissions
 from documents.permissions import TrashPermissions
 from documents.permissions import ViewDocumentsPermissions
-from documents.permissions import annotate_document_count_by_ids
 from documents.permissions import annotate_document_count_for_related_queryset
 from documents.permissions import get_document_count_filter_for_user
 from documents.permissions import get_permission_grantees
@@ -1022,71 +1022,83 @@ class DocumentViewSet(
         "custom_field_",
     )
 
-    def _get_selection_data_for_queryset(self, queryset):
+    # Matching document ids are looked up in the link tables in batches of this
-        # Resolve once instead of once per model below. `queryset` can carry an
+    # size: one literal IN (...) over every match is a multi-MB statement per
-        # arbitrarily expensive WHERE clause (user filters plus the permission
+    # relation and, on SQLite, exceeds the bound-parameter limit.
-        # filter); re-embedding it as a subquery inside 5 separate Count(...)
+    selection_data_link_batch_size = 10_000
-        # calls forces the database to re-evaluate that whole thing 5 times, and
-        # -- for FK relations especially -- can defeat semi-join planning
-        # entirely at scale. A concrete id list is cheap to reuse.
-        # order_by() drops the default/user ordering -- irrelevant for a plain
-        # id list, but left in place it forces a sort over the full filtered
-        # set before the ids can even be collected.
-        document_ids = list(queryset.order_by().values_list("pk", flat=True))
 
-        correspondents = Correspondent.objects.annotate(
+    def _get_selection_data_for_queryset(self, queryset):
-            document_count=Count(
+        # The filtered queryset is evaluated once, as the page's own filter is,
-                "documents",
+        # and never wrapped in a subquery: re-deriving a join or a HAVING
-                filter=Q(documents__id__in=document_ids),
+        # (shared_by__id) per statement is very slow on some databases, and
-                distinct=True,
+        # regrouping the queryset by another column would miscount.
-            ),
+        # order_by() drops the default/user ordering, which only costs a sort.
-        )
+        correspondents: Counter = Counter()
-        document_types = DocumentType.objects.annotate(
+        document_types: Counter = Counter()
-            document_count=Count(
+        storage_paths: Counter = Counter()
-                "documents",
+        document_ids: list[int] = []
-                filter=Q(documents__id__in=document_ids),
+        seen: set[int] = set()
-                distinct=True,
+        rows = queryset.order_by().values_list(
-            ),
+            "pk",
-        )
+            "correspondent_id",
-        storage_paths = StoragePath.objects.annotate(
+            "document_type_id",
-            document_count=Count(
+            "storage_path_id",
-                "documents",
-                filter=Q(documents__id__in=document_ids),
-                distinct=True,
-            ),
-        )
-        # Tag and CustomField reach Document through an M2M/through-model table;
-        # a plain Count(filter=...) there is a much more expensive plan than the
-        # FK relations above once the bridge table is large -- see
-        # annotate_document_count_by_ids() for why.
-        tags = annotate_document_count_by_ids(
-            Tag.objects.all(),
-            through_model=Document.tags.through,
-            related_object_field="tag_id",
-            document_ids=document_ids,
-        )
-        custom_fields = annotate_document_count_by_ids(
-            CustomField.objects.all(),
-            through_model=CustomFieldInstance,
-            related_object_field="field_id",
-            document_ids=document_ids,
         )
+        for pk, correspondent, document_type, storage_path in rows:
+            # A filter can join, repeating a document row.
+            if pk in seen:
+                continue
+            seen.add(pk)
+            document_ids.append(pk)
+            # Documents without a value count under None, which selection()
+            # never looks up.
+            correspondents[correspondent] += 1
+            document_types[document_type] += 1
+            storage_paths[storage_path] += 1
+
+        def link_table_counts(link_model, field_name: str) -> Counter:
+            result: Counter = Counter()
+            batch_size = self.selection_data_link_batch_size
+            for start in range(0, len(document_ids), batch_size):
+                result.update(
+                    dict(
+                        link_model.objects.filter(
+                            document_id__in=document_ids[start : start + batch_size],
+                        )
+                        .values_list(field_name)
+                        .annotate(document_count=Count("document_id")),
+                    ),
+                )
+            return result
+
+        def selection(model, counts: Counter) -> list[dict[str, int]]:
+            # Every object is listed, with 0 for those with no matching document.
+            return [
+                {"id": pk, "document_count": counts.get(pk, 0)}
+                for pk in model.objects.values_list("pk", flat=True)
+            ]
+
         return {
-            "selected_correspondents": [
+            "selected_correspondents": selection(
-                {"id": t.id, "document_count": t.document_count} for t in correspondents
+                Correspondent,
-            ],
+                correspondents,
-            "selected_tags": [
+            ),
-                {"id": t.id, "document_count": t.document_count} for t in tags
+            "selected_tags": selection(
-            ],
+                Tag,
-            "selected_document_types": [
+                link_table_counts(Document.tags.through, "tag_id"),
-                {"id": t.id, "document_count": t.document_count} for t in document_types
+            ),
-            ],
+            "selected_document_types": selection(
-            "selected_storage_paths": [
+                DocumentType,
-                {"id": t.id, "document_count": t.document_count} for t in storage_paths
+                document_types,
-            ],
+            ),
-            "selected_custom_fields": [
+            "selected_storage_paths": selection(
-                {"id": t.id, "document_count": t.document_count} for t in custom_fields
+                StoragePath,
-            ],
+                storage_paths,
+            ),
+            "selected_custom_fields": selection(
+                CustomField,
+                link_table_counts(CustomFieldInstance, "field_id"),
+            ),
         }
 
     @classmethod

src/paperless_ai/embedding.py

@@ -20,18 +20,7 @@ OCR_LEADER_REGEX = re.compile(r"[._\-\u00b7]{4,}")
 HORIZONTAL_WHITESPACE_REGEX = re.compile(r"[ \t\u00a0]+")
 
 
-def get_embedding_model(
+def get_embedding_model(config: AIConfig) -> "BaseEmbedding":
-    config: AIConfig,
-    *,
-    refresh: bool = False,
-) -> "BaseEmbedding":
-    """Build the configured embedding model.
-
-    For the Hugging Face backend, a model already in the local cache is loaded
-    without contacting the Hub, which saves seconds per call. ``refresh=True``
-    checks the Hub for a newer revision first; index rebuilds use it so a new
-    model release is picked up together with re-embedding every document.
-    """
     match config.llm_embedding_backend:
         case LLMEmbeddingBackend.OPENAI_LIKE:
             from llama_index.embeddings.openai_like import OpenAILikeEmbedding
@@ -63,18 +52,11 @@ def get_embedding_model(
         case LLMEmbeddingBackend.HUGGINGFACE:
             from llama_index.embeddings.huggingface import HuggingFaceEmbedding
 
-            hf_kwargs = {
+            return HuggingFaceEmbedding(
-                "model_name": config.llm_embedding_model
+                model_name=config.llm_embedding_model
                 or "sentence-transformers/all-MiniLM-L6-v2",
-                "cache_folder": str(settings.DATA_DIR / "hf_cache"),
+                cache_folder=str(settings.DATA_DIR / "hf_cache"),
-            }
+            )
-            if refresh:
-                return HuggingFaceEmbedding(**hf_kwargs)
-            try:
-                return HuggingFaceEmbedding(**hf_kwargs, local_files_only=True)
-            except OSError:
-                # Not cached yet, download it
-                return HuggingFaceEmbedding(**hf_kwargs)
         case LLMEmbeddingBackend.OLLAMA:
             from llama_index.embeddings.ollama import OllamaEmbedding
             from ollama import AsyncClient

src/paperless_ai/indexing.py

@@ -470,7 +470,7 @@ def update_llm_index(
         logger.warning("No documents found to index.")
 
     chunk_size = config.llm_embedding_chunk_size
-    embed_model = get_embedding_model(config, refresh=rebuild)
+    embed_model = get_embedding_model(config)
 
     with write_store(embed_model_name=model_name) as store:
         if rebuild or not store.table_exists():

src/paperless_ai/tests/test_embedding.py

@@ -167,7 +167,6 @@ def test_get_embedding_model_huggingface(mock_ai_config):
         MockHuggingFaceEmbedding.assert_called_once_with(
             model_name="sentence-transformers/all-MiniLM-L6-v2",
             cache_folder=str(settings.DATA_DIR / "hf_cache"),
-            local_files_only=True,
         )
         assert model == MockHuggingFaceEmbedding.return_value
 

src/paperless_ai/tests/test_embedding_hf_local_files.py

@@ -1,106 +0,0 @@
-from pathlib import Path
-from unittest.mock import MagicMock
-from unittest.mock import call
-from unittest.mock import patch
-
-import pytest
-import pytest_mock
-from django.conf import settings
-
-from paperless.models import LLMEmbeddingBackend
-from paperless_ai import indexing
-from paperless_ai.embedding import get_embedding_model
-from paperless_ai.tests.conftest import FakeEmbedding
-from paperless_testing.factories import DocumentFactory
-
-MODEL = "sentence-transformers/all-MiniLM-L6-v2"
-CACHE = str(settings.DATA_DIR / "hf_cache")
-
-
-@pytest.fixture
-def hf_config() -> MagicMock:
-    config = MagicMock()
-    config.llm_embedding_backend = LLMEmbeddingBackend.HUGGINGFACE
-    config.llm_embedding_model = MODEL
-    return config
-
-
-@pytest.fixture
-def hf_embedding(mocker: pytest_mock.MockerFixture) -> MagicMock:
-    return mocker.patch("llama_index.embeddings.huggingface.HuggingFaceEmbedding")
-
-
-class TestHuggingFaceLocalFirst:
-    def test_uses_cached_model_without_hub(
-        self,
-        hf_config: MagicMock,
-        hf_embedding: MagicMock,
-    ) -> None:
-        model = get_embedding_model(hf_config)
-
-        hf_embedding.assert_called_once_with(
-            model_name=MODEL,
-            cache_folder=CACHE,
-            local_files_only=True,
-        )
-        assert model is hf_embedding.return_value
-
-    def test_downloads_when_not_cached(
-        self,
-        hf_config: MagicMock,
-        hf_embedding: MagicMock,
-    ) -> None:
-        downloaded = MagicMock()
-        hf_embedding.side_effect = [OSError("not cached"), downloaded]
-
-        model = get_embedding_model(hf_config)
-
-        assert hf_embedding.call_args_list == [
-            call(model_name=MODEL, cache_folder=CACHE, local_files_only=True),
-            call(model_name=MODEL, cache_folder=CACHE),
-        ]
-        assert model is downloaded
-
-    def test_refresh_checks_the_hub(
-        self,
-        hf_config: MagicMock,
-        hf_embedding: MagicMock,
-    ) -> None:
-        get_embedding_model(hf_config, refresh=True)
-
-        hf_embedding.assert_called_once_with(model_name=MODEL, cache_folder=CACHE)
-
-
-@pytest.mark.django_db
-class TestIndexUpdateRefresh:
-    @pytest.fixture(autouse=True)
-    def _index_inputs(self, temp_llm_index_dir: Path) -> None:
-        DocumentFactory.create()
-
-    def _update(self, mock_embed_model: FakeEmbedding, *, rebuild: bool) -> MagicMock:
-        mock_config = MagicMock()
-        mock_config.llm_embedding_chunk_size = 512
-        with (
-            patch("paperless_ai.indexing.AIConfig", return_value=mock_config),
-            patch("paperless_ai.indexing.build_document_node", return_value=[]),
-            patch(
-                "paperless_ai.indexing.get_embedding_model",
-                return_value=mock_embed_model,
-            ) as get_model,
-        ):
-            indexing.update_llm_index(rebuild=rebuild)
-        get_model.assert_called_once()
-        return get_model
-
-    def test_rebuild_refreshes(self, mock_embed_model: FakeEmbedding) -> None:
-        get_model = self._update(mock_embed_model, rebuild=True)
-
-        assert get_model.call_args.kwargs == {"refresh": True}
-
-    def test_incremental_update_stays_local(
-        self,
-        mock_embed_model: FakeEmbedding,
-    ) -> None:
-        get_model = self._update(mock_embed_model, rebuild=False)
-
-        assert get_model.call_args.kwargs == {"refresh": False}