Compare commits

..
Author SHA1 Message Date
Trenton HandClaude Sonnet 5.5 7adc03dd66 Chore: use document type ids in the bulk edit selection_data test
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-30 14:08:37 -07:00
Trenton HandClaude Sonnet 5.5 01f5c81bc2 Fix: count selection_data from one pass over the filtered documents
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-30 14:08:37 -07:00
13 changed files with 200 additions and 301 deletions

No files matched your search

+5 -5
View File
@@ -10334,18 +10334,18 @@
<context context-type="linenumber">294,295</context>
</context-group>
</trans-unit>
<trans-unit id="3420321797707163677" datatype="html">
<source>Preview document</source>
<trans-unit id="2179847500064178686" datatype="html">
<source>Edit document</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-list/document-list.component.html</context>
<context context-type="linenumber">328</context>
</context-group>
</trans-unit>
<trans-unit id="2179847500064178686" datatype="html">
<source>Edit document</source>
<trans-unit id="3420321797707163677" datatype="html">
<source>Preview document</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-list/document-list.component.html</context>
<context context-type="linenumber">331</context>
<context context-type="linenumber">329</context>
</context-group>
</trans-unit>
<trans-unit id="4512084577073831437" datatype="html">
@@ -1,3 +1,10 @@
.custom-fields-dropdown {
min-width: 300px;
// correct position on mobile
@media (max-width: 575.98px) {
&.show {
margin-left: -245px !important;
}
}
}
@@ -308,7 +308,7 @@
</tr>
<tr class="align-middle">
<td i18n>Original SHA256 checksum</td>
<td><span class="badge bg-light text-dark font-monospace text-wrap text-break text-start">{{metadata()?.original_checksum}}</span></td>
<td><span class="badge bg-light font-monospace text-wrap text-break text-start">{{metadata()?.original_checksum}}</span></td>
</tr>
<tr>
<td i18n>Original file size</td>
@@ -465,7 +465,7 @@
@switch (archiveContentRenderType) {
@case (ContentRenderType.PDF) {
@if (!useNativePdfViewer) {
<div class="preview-sticky pdf-viewer-container" tabindex="-1">
<div class="preview-sticky pdf-viewer-container">
<pngx-pdf-viewer
[src]="pdfSource()"
[sourceRevision]="previewRevision()"
@@ -477,7 +477,7 @@
(loadError)="onError($event)"
(afterLoadComplete)="pdfPreviewLoaded($event)">
</pngx-pdf-viewer>
<div class="preview-controls position-absolute bottom-0 start-50 translate-middle-x z-1 mb-3 d-flex gap-2 p-1 bg-body border rounded shadow">
<div class="position-absolute bottom-0 start-50 translate-middle-x z-1 mb-3 d-flex gap-2 p-1 bg-body border rounded shadow">
@if (previewNumPages()) {
<div class="input-group input-group-sm w-auto flex-nowrap">
<input class="form-control flex-grow-0 w-auto" type="number" min="1" [max]="previewNumPages()" [ngModelOptions]="{standalone: true}" [ngModel]="previewCurrentPage()" (ngModelChange)="previewCurrentPage.set($event)" aria-label="Page" i18n-aria-label />
@@ -12,22 +12,6 @@
width: 100%;
height: 100%;
}
&:focus {
outline: none;
}
.preview-controls {
opacity: 0;
pointer-events: none;
transition: opacity 0.15s ease-in-out;
}
&:hover .preview-controls,
&:focus-within .preview-controls {
opacity: 1;
pointer-events: auto;
}
}
.btn-group .dropdown-toggle-split {
@@ -208,7 +208,7 @@
i18n>Correspondent</th>
}
@if (activeDisplayFields.includes(DisplayField.TITLE)) {
<th class="cursor-pointer ps-5"
<th class="cursor-pointer"
pngxSortable="title"
title="Sort by title" i18n-title
[currentSortField]="list.sortField"
@@ -322,13 +322,13 @@
</td>
}
@if (activeDisplayFields.includes(DisplayField.TITLE) || activeDisplayFields.includes(DisplayField.TAGS)) {
<td width="30%" class="position-relative" [class.ps-5]="activeDisplayFields.includes(DisplayField.TITLE)">
<td width="30%">
@if (activeDisplayFields.includes(DisplayField.TITLE)) {
<div class="d-inline-block" (mouseleave)="popupPreview.close()">
<pngx-preview-popup class="position-absolute start-0" [document]="d" linkClasses="btn btn-sm btn-link text-secondary" linkTitle="Preview document" (click)="$event.stopPropagation()" i18n-linkTitle #popupPreview>
<a routerLink="/documents/{{d.id}}" title="Edit document" i18n-title style="overflow-wrap: anywhere;">{{d.title | documentTitle}}</a>
<pngx-preview-popup [document]="d" linkClasses="btn btn-sm btn-link text-secondary" linkTitle="Preview document" (click)="$event.stopPropagation()" i18n-linkTitle #popupPreview>
<i-bs name="eye"></i-bs>
</pngx-preview-popup>
<a routerLink="/documents/{{d.id}}" title="Edit document" i18n-title style="overflow-wrap: anywhere;">{{d.title | documentTitle}}</a>
</div>
}
@if (activeDisplayFields.includes(DisplayField.TAGS)) {
@@ -87,10 +87,6 @@ $paperless-card-breakpoints: (
}
}
.table .ps-5 {
padding-left: 2.2rem !important;
}
a {
cursor: pointer;
}
+3 -6
View File
@@ -576,12 +576,9 @@ def annotate_document_count_by_ids(
or CustomFieldInstance)
related_object_field: field on the relation pointing back to queryset pk
document_ids: the document ids to count against -- a concrete list/set,
or a simple (already resolved) queryset of ids. Callers
that need this filtered by a complex condition (e.g. a
permission check) should resolve it to a concrete list
first if the same ids will be reused across multiple
calls, rather than passing the complex queryset itself
into each -- see ``_get_selection_data_for_queryset``.
or a simple (already resolved) queryset of ids. Keep
the list small: it is embedded in the statement as
literal parameters.
target_field: field on the relation pointing to Document id
"""
+2 -2
View File
@@ -1141,8 +1141,8 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.assertCountEqual(
response.data["selected_document_types"],
[
{"id": self.c1.id, "document_count": 1},
{"id": self.c2.id, "document_count": 0},
{"id": self.dt1.id, "document_count": 1},
{"id": self.dt2.id, "document_count": 0},
],
)
+101
View File
@@ -49,6 +49,7 @@ from documents.models import WorkflowTrigger
from documents.search import get_backend
from documents.signals.handlers import run_workflows
from documents.tests.utils import ConsumeTaskMixin
from documents.views import DocumentViewSet
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import DocumentFactory
from paperless_testing.factories import TagFactory
@@ -1755,6 +1756,106 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
}
self.assertEqual(counts_by_field[custom_field.id], 1)
def test_selection_data_counts_with_shared_by_filter(self) -> None:
"""
GIVEN:
- Several documents owned by one user with the same correspondent
and document type, only one of which is shared
WHEN:
- Requesting the document list filtered by shared_by__id with
include_selection_data=true
THEN:
- Counts reflect only the shared document, not every document the
owner has with that correspondent or document type, however the
link-table lookups are batched
"""
owner = User.objects.create_user(username="owner")
other = User.objects.create_user(username="other")
correspondent = Correspondent.objects.create(name="c1")
doc_type = DocumentType.objects.create(name="dt1")
tag = Tag.objects.create(name="tag")
docs = [
Document.objects.create(
checksum=str(i),
owner=owner,
correspondent=correspondent,
document_type=doc_type,
)
for i in range(3)
]
for doc in docs:
doc.tags.add(tag)
grant_object(other, docs[0], "view_document")
for batch_size in (10_000, 1):
with (
self.subTest(link_batch_size=batch_size),
mock.patch.object(
DocumentViewSet,
"selection_data_link_batch_size",
batch_size,
),
):
response = self.client.get(
f"/api/documents/?shared_by__id={owner.id}"
"&include_selection_data=true",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data["count"], 1)
selection_data = response.data["selection_data"]
for key, obj in (
("selected_correspondents", correspondent),
("selected_document_types", doc_type),
("selected_tags", tag),
):
counts = {
item["id"]: item["document_count"]
for item in selection_data[key]
}
self.assertEqual(counts[obj.id], 1, key)
def test_selection_data_counts_documents_once_when_filter_joins(self) -> None:
"""
GIVEN:
- A document matching a multi-valued filter through two tags
WHEN:
- Requesting the document list with include_selection_data=true
THEN:
- The document is counted once per correspondent, not once per
matching tag, however the link-table lookups are batched
"""
correspondent = Correspondent.objects.create(name="c1")
tag_a = Tag.objects.create(name="a")
tag_b = Tag.objects.create(name="b")
both = Document.objects.create(checksum="1", correspondent=correspondent)
both.tags.add(tag_a, tag_b)
only_a = Document.objects.create(checksum="2", correspondent=correspondent)
only_a.tags.add(tag_a)
for batch_size in (10_000, 1):
with (
self.subTest(link_batch_size=batch_size),
mock.patch.object(
DocumentViewSet,
"selection_data_link_batch_size",
batch_size,
),
):
response = self.client.get(
f"/api/documents/?tags__id__in={tag_a.id},{tag_b.id}"
"&include_selection_data=true",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
counts = {
item["id"]: item["document_count"]
for item in response.data["selection_data"][
"selected_correspondents"
]
}
self.assertEqual(counts[correspondent.id], 2)
def test_statistics(self) -> None:
doc1 = Document.objects.create(
title="none1",
+74 -62
View File
@@ -5,6 +5,7 @@ import platform
import re
import tempfile
import zipfile
from collections import Counter
from collections import deque
from collections.abc import Callable
from datetime import datetime
@@ -168,7 +169,6 @@ from documents.permissions import PaperlessNotePermissions
from documents.permissions import PaperlessObjectPermissions
from documents.permissions import TrashPermissions
from documents.permissions import ViewDocumentsPermissions
from documents.permissions import annotate_document_count_by_ids
from documents.permissions import annotate_document_count_for_related_queryset
from documents.permissions import get_document_count_filter_for_user
from documents.permissions import get_permission_grantees
@@ -1022,71 +1022,83 @@ class DocumentViewSet(
"custom_field_",
)
def _get_selection_data_for_queryset(self, queryset):
# Resolve once instead of once per model below. `queryset` can carry an
# arbitrarily expensive WHERE clause (user filters plus the permission
# filter); re-embedding it as a subquery inside 5 separate Count(...)
# calls forces the database to re-evaluate that whole thing 5 times, and
# -- for FK relations especially -- can defeat semi-join planning
# entirely at scale. A concrete id list is cheap to reuse.
# order_by() drops the default/user ordering -- irrelevant for a plain
# id list, but left in place it forces a sort over the full filtered
# set before the ids can even be collected.
document_ids = list(queryset.order_by().values_list("pk", flat=True))
# Matching document ids are looked up in the link tables in batches of this
# size: one literal IN (...) over every match is a multi-MB statement per
# relation and, on SQLite, exceeds the bound-parameter limit.
selection_data_link_batch_size = 10_000
correspondents = Correspondent.objects.annotate(
document_count=Count(
"documents",
filter=Q(documents__id__in=document_ids),
distinct=True,
def _get_selection_data_for_queryset(self, queryset):
# The filtered queryset is evaluated once, as the page's own filter is,
# and never wrapped in a subquery: re-deriving a join or a HAVING
# (shared_by__id) per statement is very slow on some databases, and
# regrouping the queryset by another column would miscount.
# order_by() drops the default/user ordering, which only costs a sort.
correspondents: Counter = Counter()
document_types: Counter = Counter()
storage_paths: Counter = Counter()
document_ids: list[int] = []
seen: set[int] = set()
rows = queryset.order_by().values_list(
"pk",
"correspondent_id",
"document_type_id",
"storage_path_id",
)
for pk, correspondent, document_type, storage_path in rows:
# A filter can join, repeating a document row.
if pk in seen:
continue
seen.add(pk)
document_ids.append(pk)
# Documents without a value count under None, which selection()
# never looks up.
correspondents[correspondent] += 1
document_types[document_type] += 1
storage_paths[storage_path] += 1
def link_table_counts(link_model, field_name: str) -> Counter:
result: Counter = Counter()
batch_size = self.selection_data_link_batch_size
for start in range(0, len(document_ids), batch_size):
result.update(
dict(
link_model.objects.filter(
document_id__in=document_ids[start : start + batch_size],
)
.values_list(field_name)
.annotate(document_count=Count("document_id")),
),
)
document_types = DocumentType.objects.annotate(
document_count=Count(
"documents",
filter=Q(documents__id__in=document_ids),
distinct=True,
),
)
storage_paths = StoragePath.objects.annotate(
document_count=Count(
"documents",
filter=Q(documents__id__in=document_ids),
distinct=True,
),
)
# Tag and CustomField reach Document through an M2M/through-model table;
# a plain Count(filter=...) there is a much more expensive plan than the
# FK relations above once the bridge table is large -- see
# annotate_document_count_by_ids() for why.
tags = annotate_document_count_by_ids(
Tag.objects.all(),
through_model=Document.tags.through,
related_object_field="tag_id",
document_ids=document_ids,
)
custom_fields = annotate_document_count_by_ids(
CustomField.objects.all(),
through_model=CustomFieldInstance,
related_object_field="field_id",
document_ids=document_ids,
)
return result
def selection(model, counts: Counter) -> list[dict[str, int]]:
# Every object is listed, with 0 for those with no matching document.
return [
{"id": pk, "document_count": counts.get(pk, 0)}
for pk in model.objects.values_list("pk", flat=True)
]
return {
"selected_correspondents": [
{"id": t.id, "document_count": t.document_count} for t in correspondents
],
"selected_tags": [
{"id": t.id, "document_count": t.document_count} for t in tags
],
"selected_document_types": [
{"id": t.id, "document_count": t.document_count} for t in document_types
],
"selected_storage_paths": [
{"id": t.id, "document_count": t.document_count} for t in storage_paths
],
"selected_custom_fields": [
{"id": t.id, "document_count": t.document_count} for t in custom_fields
],
"selected_correspondents": selection(
Correspondent,
correspondents,
),
"selected_tags": selection(
Tag,
link_table_counts(Document.tags.through, "tag_id"),
),
"selected_document_types": selection(
DocumentType,
document_types,
),
"selected_storage_paths": selection(
StoragePath,
storage_paths,
),
"selected_custom_fields": selection(
CustomField,
link_table_counts(CustomFieldInstance, "field_id"),
),
}
@classmethod
@@ -1,93 +0,0 @@
import sqlite3
from collections.abc import Generator
from pathlib import Path
from unittest.mock import patch
import pytest
from llama_index.core.schema import TextNode
from paperless_ai.tables import DocumentChunksTable
from paperless_ai.vector_store import DB_FILENAME
from paperless_ai.vector_store import DEFAULT_TABLE_NAME
from paperless_ai.vector_store import VEC0_CHUNK_SIZE
from paperless_ai.vector_store import PaperlessSqliteVecVectorStore
DIM = 16
def make_node(node_id: str, document_id: int) -> TextNode:
node = TextNode(
id_=node_id,
text="some text",
metadata={"document_id": document_id, "modified": "2026-06-10T00:00:00"},
)
node.relationships = {}
node.embedding = [document_id + i / 100 for i in range(DIM)]
return node
def vec_table_sql(db_path: Path) -> str:
conn = sqlite3.connect(db_path)
try:
return conn.execute(
"SELECT sql FROM sqlite_master WHERE name = ?",
(DEFAULT_TABLE_NAME,),
).fetchone()[0]
finally:
conn.close()
def create_legacy_vec_table(conn: sqlite3.Connection, dim: int) -> None:
"""The table definition from before chunk_size was set explicitly."""
conn.execute(
f"CREATE VIRTUAL TABLE {DEFAULT_TABLE_NAME} USING vec0("
"id TEXT PRIMARY KEY, document_id INTEGER, +node_content TEXT, "
f"embedding float[{dim}] distance_metric=cosine)",
)
@pytest.fixture
def store(tmp_path: Path) -> Generator[PaperlessSqliteVecVectorStore, None, None]:
with PaperlessSqliteVecVectorStore(uri=str(tmp_path)) as store:
yield store
class TestVecChunkSize:
def test_new_table_uses_configured_chunk_size(
self,
store: PaperlessSqliteVecVectorStore,
tmp_path: Path,
) -> None:
"""
GIVEN an empty store
WHEN the first document is indexed and the vec0 table is created
THEN the table definition carries the configured chunk size
"""
store.upsert_document(1, [make_node("1-a", 1)])
assert f"chunk_size={VEC0_CHUNK_SIZE}" in vec_table_sql(tmp_path / DB_FILENAME)
def test_compact_moves_legacy_table_to_configured_chunk_size(
self,
store: PaperlessSqliteVecVectorStore,
tmp_path: Path,
) -> None:
"""
GIVEN an index whose vec0 table was created with the default chunk size
WHEN the index is compacted
THEN the rebuilt table uses the configured chunk size
AND no indexed chunks are lost
"""
with patch.object(
PaperlessSqliteVecVectorStore,
"_create_vec_table",
staticmethod(create_legacy_vec_table),
):
store.upsert_document(1, [make_node("1-a", 1), make_node("1-b", 1)])
store.upsert_document(2, [make_node("2-a", 2)])
assert "chunk_size" not in vec_table_sql(tmp_path / DB_FILENAME)
store.compact(force=True)
assert f"chunk_size={VEC0_CHUNK_SIZE}" in vec_table_sql(tmp_path / DB_FILENAME)
assert DocumentChunksTable.count(store.client) == 3
@@ -1,89 +0,0 @@
import sqlite3
from collections.abc import Generator
from pathlib import Path
import pytest
from llama_index.core.schema import TextNode
from paperless_ai.tables import DocumentChunksTable
from paperless_ai.vector_store import DB_FILENAME
from paperless_ai.vector_store import DB_PAGE_SIZE
from paperless_ai.vector_store import PaperlessSqliteVecVectorStore
DIM = 16
LEGACY_PAGE_SIZE = 4096
def make_node(node_id: str, document_id: int) -> TextNode:
node = TextNode(
id_=node_id,
text="some text",
metadata={"document_id": document_id, "modified": "2026-06-10T00:00:00"},
)
node.relationships = {}
node.embedding = [document_id + i / 100 for i in range(DIM)]
return node
def page_size_of(db_path: Path) -> int:
conn = sqlite3.connect(db_path)
try:
return conn.execute("PRAGMA page_size").fetchone()[0]
finally:
conn.close()
@pytest.fixture
def legacy_page_size_db(tmp_path: Path) -> Path:
"""An existing index file created before the page size was set."""
db_path = tmp_path / DB_FILENAME
conn = sqlite3.connect(db_path)
conn.execute(f"PRAGMA page_size={LEGACY_PAGE_SIZE}")
conn.execute("CREATE TABLE _seed(x)")
conn.execute("DROP TABLE _seed")
conn.close()
assert page_size_of(db_path) == LEGACY_PAGE_SIZE
return db_path
@pytest.fixture
def store(tmp_path: Path) -> Generator[PaperlessSqliteVecVectorStore, None, None]:
with PaperlessSqliteVecVectorStore(uri=str(tmp_path)) as store:
yield store
class TestIndexPageSize:
def test_new_index_file_uses_configured_page_size(
self,
store: PaperlessSqliteVecVectorStore,
tmp_path: Path,
) -> None:
"""
GIVEN no existing index file
WHEN a store is opened and a document is indexed
THEN the database file uses the configured page size
"""
store.upsert_document(1, [make_node("1-a", 1)])
assert page_size_of(tmp_path / DB_FILENAME) == DB_PAGE_SIZE
def test_compact_moves_legacy_file_to_configured_page_size(
self,
legacy_page_size_db: Path,
) -> None:
"""
GIVEN an existing index file created with a different page size
WHEN the index is compacted
THEN the rebuilt file uses the configured page size
AND no indexed chunks are lost
"""
with PaperlessSqliteVecVectorStore(
uri=str(legacy_page_size_db.parent),
) as store:
store.upsert_document(1, [make_node("1-a", 1), make_node("1-b", 1)])
assert page_size_of(legacy_page_size_db) == LEGACY_PAGE_SIZE
store.compact(force=True)
assert page_size_of(legacy_page_size_db) == DB_PAGE_SIZE
assert DocumentChunksTable.count(store.client) == 2
+1 -17
View File
@@ -47,20 +47,6 @@ SCHEMA_VERSION = 2
# a rebuild copies the live rows into a fresh table.
COMPACT_BLOAT_RATIO = 2.0
# Rows per vec0 vector chunk (vec0's default is 1024, which makes each chunk a
# multi-megabyte blob). On spinning disks, inserts and re-indexing were dozens
# of times faster with small chunks, at no cost in query speed or file size.
# Only applies to tables created from here on; an existing index picks it up
# the next time it is rebuilt (compact or rebuild).
VEC0_CHUNK_SIZE = 32
# SQLite page size for the index file, in bytes. With the small vec0 chunks above,
# 64 KiB pages made cold scans about 35% faster and warm ones about 45% faster,
# at the cost of a larger write-ahead log (up to about 70 MB). Only takes effect
# on a newly created file, so like the chunk size it reaches an existing index
# when it is rebuilt.
DB_PAGE_SIZE = 65536
# Number of rows fetched/copied per batch whenever this module streams rows
# instead of materializing them all at once, keeping memory bounded regardless
# of index size -- used by compact()'s rebuild, m0001_v1_to_v2's migration
@@ -236,7 +222,6 @@ class PaperlessSqliteVecVectorStore(BasePydanticVectorStore):
conn.enable_load_extension(True) # noqa: FBT003
sqlite_vec.load(conn)
conn.enable_load_extension(False) # noqa: FBT003
conn.execute(f"PRAGMA page_size={DB_PAGE_SIZE}")
conn.execute("PRAGMA journal_mode=WAL")
conn.execute("PRAGMA synchronous=NORMAL")
IndexMetaTable.create(conn)
@@ -344,8 +329,7 @@ class PaperlessSqliteVecVectorStore(BasePydanticVectorStore):
+ " +node_content TEXT,"
+ " embedding float["
+ str(int(dim))
+ "] distance_metric=cosine, chunk_size="
+ str(VEC0_CHUNK_SIZE)
+ "] distance_metric=cosine"
+ ")",
)