mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-11-01 12:35:43 +00:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7adc03dd66 | ||
|
|
01f5c81bc2 |
No files matched your search
+5
-5
@@ -10334,18 +10334,18 @@
|
||||
<context context-type="linenumber">294,295</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="3420321797707163677" datatype="html">
|
||||
<source>Preview document</source>
|
||||
<trans-unit id="2179847500064178686" datatype="html">
|
||||
<source>Edit document</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-list/document-list.component.html</context>
|
||||
<context context-type="linenumber">328</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="2179847500064178686" datatype="html">
|
||||
<source>Edit document</source>
|
||||
<trans-unit id="3420321797707163677" datatype="html">
|
||||
<source>Preview document</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-list/document-list.component.html</context>
|
||||
<context context-type="linenumber">331</context>
|
||||
<context context-type="linenumber">329</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="4512084577073831437" datatype="html">
|
||||
|
||||
+7
@@ -1,3 +1,10 @@
|
||||
.custom-fields-dropdown {
|
||||
min-width: 300px;
|
||||
|
||||
// correct position on mobile
|
||||
@media (max-width: 575.98px) {
|
||||
&.show {
|
||||
margin-left: -245px !important;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -308,7 +308,7 @@
|
||||
</tr>
|
||||
<tr class="align-middle">
|
||||
<td i18n>Original SHA256 checksum</td>
|
||||
<td><span class="badge bg-light text-dark font-monospace text-wrap text-break text-start">{{metadata()?.original_checksum}}</span></td>
|
||||
<td><span class="badge bg-light font-monospace text-wrap text-break text-start">{{metadata()?.original_checksum}}</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td i18n>Original file size</td>
|
||||
@@ -465,7 +465,7 @@
|
||||
@switch (archiveContentRenderType) {
|
||||
@case (ContentRenderType.PDF) {
|
||||
@if (!useNativePdfViewer) {
|
||||
<div class="preview-sticky pdf-viewer-container" tabindex="-1">
|
||||
<div class="preview-sticky pdf-viewer-container">
|
||||
<pngx-pdf-viewer
|
||||
[src]="pdfSource()"
|
||||
[sourceRevision]="previewRevision()"
|
||||
@@ -477,7 +477,7 @@
|
||||
(loadError)="onError($event)"
|
||||
(afterLoadComplete)="pdfPreviewLoaded($event)">
|
||||
</pngx-pdf-viewer>
|
||||
<div class="preview-controls position-absolute bottom-0 start-50 translate-middle-x z-1 mb-3 d-flex gap-2 p-1 bg-body border rounded shadow">
|
||||
<div class="position-absolute bottom-0 start-50 translate-middle-x z-1 mb-3 d-flex gap-2 p-1 bg-body border rounded shadow">
|
||||
@if (previewNumPages()) {
|
||||
<div class="input-group input-group-sm w-auto flex-nowrap">
|
||||
<input class="form-control flex-grow-0 w-auto" type="number" min="1" [max]="previewNumPages()" [ngModelOptions]="{standalone: true}" [ngModel]="previewCurrentPage()" (ngModelChange)="previewCurrentPage.set($event)" aria-label="Page" i18n-aria-label />
|
||||
|
||||
@@ -12,22 +12,6 @@
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
}
|
||||
|
||||
&:focus {
|
||||
outline: none;
|
||||
}
|
||||
|
||||
.preview-controls {
|
||||
opacity: 0;
|
||||
pointer-events: none;
|
||||
transition: opacity 0.15s ease-in-out;
|
||||
}
|
||||
|
||||
&:hover .preview-controls,
|
||||
&:focus-within .preview-controls {
|
||||
opacity: 1;
|
||||
pointer-events: auto;
|
||||
}
|
||||
}
|
||||
|
||||
.btn-group .dropdown-toggle-split {
|
||||
|
||||
@@ -208,7 +208,7 @@
|
||||
i18n>Correspondent</th>
|
||||
}
|
||||
@if (activeDisplayFields.includes(DisplayField.TITLE)) {
|
||||
<th class="cursor-pointer ps-5"
|
||||
<th class="cursor-pointer"
|
||||
pngxSortable="title"
|
||||
title="Sort by title" i18n-title
|
||||
[currentSortField]="list.sortField"
|
||||
@@ -322,13 +322,13 @@
|
||||
</td>
|
||||
}
|
||||
@if (activeDisplayFields.includes(DisplayField.TITLE) || activeDisplayFields.includes(DisplayField.TAGS)) {
|
||||
<td width="30%" class="position-relative" [class.ps-5]="activeDisplayFields.includes(DisplayField.TITLE)">
|
||||
<td width="30%">
|
||||
@if (activeDisplayFields.includes(DisplayField.TITLE)) {
|
||||
<div class="d-inline-block" (mouseleave)="popupPreview.close()">
|
||||
<pngx-preview-popup class="position-absolute start-0" [document]="d" linkClasses="btn btn-sm btn-link text-secondary" linkTitle="Preview document" (click)="$event.stopPropagation()" i18n-linkTitle #popupPreview>
|
||||
<a routerLink="/documents/{{d.id}}" title="Edit document" i18n-title style="overflow-wrap: anywhere;">{{d.title | documentTitle}}</a>
|
||||
<pngx-preview-popup [document]="d" linkClasses="btn btn-sm btn-link text-secondary" linkTitle="Preview document" (click)="$event.stopPropagation()" i18n-linkTitle #popupPreview>
|
||||
<i-bs name="eye"></i-bs>
|
||||
</pngx-preview-popup>
|
||||
<a routerLink="/documents/{{d.id}}" title="Edit document" i18n-title style="overflow-wrap: anywhere;">{{d.title | documentTitle}}</a>
|
||||
</div>
|
||||
}
|
||||
@if (activeDisplayFields.includes(DisplayField.TAGS)) {
|
||||
|
||||
@@ -87,10 +87,6 @@ $paperless-card-breakpoints: (
|
||||
}
|
||||
}
|
||||
|
||||
.table .ps-5 {
|
||||
padding-left: 2.2rem !important;
|
||||
}
|
||||
|
||||
a {
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
@@ -576,12 +576,9 @@ def annotate_document_count_by_ids(
|
||||
or CustomFieldInstance)
|
||||
related_object_field: field on the relation pointing back to queryset pk
|
||||
document_ids: the document ids to count against -- a concrete list/set,
|
||||
or a simple (already resolved) queryset of ids. Callers
|
||||
that need this filtered by a complex condition (e.g. a
|
||||
permission check) should resolve it to a concrete list
|
||||
first if the same ids will be reused across multiple
|
||||
calls, rather than passing the complex queryset itself
|
||||
into each -- see ``_get_selection_data_for_queryset``.
|
||||
or a simple (already resolved) queryset of ids. Keep
|
||||
the list small: it is embedded in the statement as
|
||||
literal parameters.
|
||||
target_field: field on the relation pointing to Document id
|
||||
"""
|
||||
|
||||
|
||||
@@ -1141,8 +1141,8 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
|
||||
self.assertCountEqual(
|
||||
response.data["selected_document_types"],
|
||||
[
|
||||
{"id": self.c1.id, "document_count": 1},
|
||||
{"id": self.c2.id, "document_count": 0},
|
||||
{"id": self.dt1.id, "document_count": 1},
|
||||
{"id": self.dt2.id, "document_count": 0},
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@@ -49,6 +49,7 @@ from documents.models import WorkflowTrigger
|
||||
from documents.search import get_backend
|
||||
from documents.signals.handlers import run_workflows
|
||||
from documents.tests.utils import ConsumeTaskMixin
|
||||
from documents.views import DocumentViewSet
|
||||
from paperless_testing.dirs import DirectoriesMixin
|
||||
from paperless_testing.factories import DocumentFactory
|
||||
from paperless_testing.factories import TagFactory
|
||||
@@ -1755,6 +1756,106 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
|
||||
}
|
||||
self.assertEqual(counts_by_field[custom_field.id], 1)
|
||||
|
||||
def test_selection_data_counts_with_shared_by_filter(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- Several documents owned by one user with the same correspondent
|
||||
and document type, only one of which is shared
|
||||
WHEN:
|
||||
- Requesting the document list filtered by shared_by__id with
|
||||
include_selection_data=true
|
||||
THEN:
|
||||
- Counts reflect only the shared document, not every document the
|
||||
owner has with that correspondent or document type, however the
|
||||
link-table lookups are batched
|
||||
"""
|
||||
owner = User.objects.create_user(username="owner")
|
||||
other = User.objects.create_user(username="other")
|
||||
correspondent = Correspondent.objects.create(name="c1")
|
||||
doc_type = DocumentType.objects.create(name="dt1")
|
||||
tag = Tag.objects.create(name="tag")
|
||||
docs = [
|
||||
Document.objects.create(
|
||||
checksum=str(i),
|
||||
owner=owner,
|
||||
correspondent=correspondent,
|
||||
document_type=doc_type,
|
||||
)
|
||||
for i in range(3)
|
||||
]
|
||||
for doc in docs:
|
||||
doc.tags.add(tag)
|
||||
grant_object(other, docs[0], "view_document")
|
||||
|
||||
for batch_size in (10_000, 1):
|
||||
with (
|
||||
self.subTest(link_batch_size=batch_size),
|
||||
mock.patch.object(
|
||||
DocumentViewSet,
|
||||
"selection_data_link_batch_size",
|
||||
batch_size,
|
||||
),
|
||||
):
|
||||
response = self.client.get(
|
||||
f"/api/documents/?shared_by__id={owner.id}"
|
||||
"&include_selection_data=true",
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
self.assertEqual(response.data["count"], 1)
|
||||
selection_data = response.data["selection_data"]
|
||||
for key, obj in (
|
||||
("selected_correspondents", correspondent),
|
||||
("selected_document_types", doc_type),
|
||||
("selected_tags", tag),
|
||||
):
|
||||
counts = {
|
||||
item["id"]: item["document_count"]
|
||||
for item in selection_data[key]
|
||||
}
|
||||
self.assertEqual(counts[obj.id], 1, key)
|
||||
|
||||
def test_selection_data_counts_documents_once_when_filter_joins(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A document matching a multi-valued filter through two tags
|
||||
WHEN:
|
||||
- Requesting the document list with include_selection_data=true
|
||||
THEN:
|
||||
- The document is counted once per correspondent, not once per
|
||||
matching tag, however the link-table lookups are batched
|
||||
"""
|
||||
correspondent = Correspondent.objects.create(name="c1")
|
||||
tag_a = Tag.objects.create(name="a")
|
||||
tag_b = Tag.objects.create(name="b")
|
||||
both = Document.objects.create(checksum="1", correspondent=correspondent)
|
||||
both.tags.add(tag_a, tag_b)
|
||||
only_a = Document.objects.create(checksum="2", correspondent=correspondent)
|
||||
only_a.tags.add(tag_a)
|
||||
|
||||
for batch_size in (10_000, 1):
|
||||
with (
|
||||
self.subTest(link_batch_size=batch_size),
|
||||
mock.patch.object(
|
||||
DocumentViewSet,
|
||||
"selection_data_link_batch_size",
|
||||
batch_size,
|
||||
),
|
||||
):
|
||||
response = self.client.get(
|
||||
f"/api/documents/?tags__id__in={tag_a.id},{tag_b.id}"
|
||||
"&include_selection_data=true",
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
counts = {
|
||||
item["id"]: item["document_count"]
|
||||
for item in response.data["selection_data"][
|
||||
"selected_correspondents"
|
||||
]
|
||||
}
|
||||
self.assertEqual(counts[correspondent.id], 2)
|
||||
|
||||
def test_statistics(self) -> None:
|
||||
doc1 = Document.objects.create(
|
||||
title="none1",
|
||||
|
||||
+75
-63
@@ -5,6 +5,7 @@ import platform
|
||||
import re
|
||||
import tempfile
|
||||
import zipfile
|
||||
from collections import Counter
|
||||
from collections import deque
|
||||
from collections.abc import Callable
|
||||
from datetime import datetime
|
||||
@@ -168,7 +169,6 @@ from documents.permissions import PaperlessNotePermissions
|
||||
from documents.permissions import PaperlessObjectPermissions
|
||||
from documents.permissions import TrashPermissions
|
||||
from documents.permissions import ViewDocumentsPermissions
|
||||
from documents.permissions import annotate_document_count_by_ids
|
||||
from documents.permissions import annotate_document_count_for_related_queryset
|
||||
from documents.permissions import get_document_count_filter_for_user
|
||||
from documents.permissions import get_permission_grantees
|
||||
@@ -1022,71 +1022,83 @@ class DocumentViewSet(
|
||||
"custom_field_",
|
||||
)
|
||||
|
||||
def _get_selection_data_for_queryset(self, queryset):
|
||||
# Resolve once instead of once per model below. `queryset` can carry an
|
||||
# arbitrarily expensive WHERE clause (user filters plus the permission
|
||||
# filter); re-embedding it as a subquery inside 5 separate Count(...)
|
||||
# calls forces the database to re-evaluate that whole thing 5 times, and
|
||||
# -- for FK relations especially -- can defeat semi-join planning
|
||||
# entirely at scale. A concrete id list is cheap to reuse.
|
||||
# order_by() drops the default/user ordering -- irrelevant for a plain
|
||||
# id list, but left in place it forces a sort over the full filtered
|
||||
# set before the ids can even be collected.
|
||||
document_ids = list(queryset.order_by().values_list("pk", flat=True))
|
||||
# Matching document ids are looked up in the link tables in batches of this
|
||||
# size: one literal IN (...) over every match is a multi-MB statement per
|
||||
# relation and, on SQLite, exceeds the bound-parameter limit.
|
||||
selection_data_link_batch_size = 10_000
|
||||
|
||||
correspondents = Correspondent.objects.annotate(
|
||||
document_count=Count(
|
||||
"documents",
|
||||
filter=Q(documents__id__in=document_ids),
|
||||
distinct=True,
|
||||
),
|
||||
)
|
||||
document_types = DocumentType.objects.annotate(
|
||||
document_count=Count(
|
||||
"documents",
|
||||
filter=Q(documents__id__in=document_ids),
|
||||
distinct=True,
|
||||
),
|
||||
)
|
||||
storage_paths = StoragePath.objects.annotate(
|
||||
document_count=Count(
|
||||
"documents",
|
||||
filter=Q(documents__id__in=document_ids),
|
||||
distinct=True,
|
||||
),
|
||||
)
|
||||
# Tag and CustomField reach Document through an M2M/through-model table;
|
||||
# a plain Count(filter=...) there is a much more expensive plan than the
|
||||
# FK relations above once the bridge table is large -- see
|
||||
# annotate_document_count_by_ids() for why.
|
||||
tags = annotate_document_count_by_ids(
|
||||
Tag.objects.all(),
|
||||
through_model=Document.tags.through,
|
||||
related_object_field="tag_id",
|
||||
document_ids=document_ids,
|
||||
)
|
||||
custom_fields = annotate_document_count_by_ids(
|
||||
CustomField.objects.all(),
|
||||
through_model=CustomFieldInstance,
|
||||
related_object_field="field_id",
|
||||
document_ids=document_ids,
|
||||
def _get_selection_data_for_queryset(self, queryset):
|
||||
# The filtered queryset is evaluated once, as the page's own filter is,
|
||||
# and never wrapped in a subquery: re-deriving a join or a HAVING
|
||||
# (shared_by__id) per statement is very slow on some databases, and
|
||||
# regrouping the queryset by another column would miscount.
|
||||
# order_by() drops the default/user ordering, which only costs a sort.
|
||||
correspondents: Counter = Counter()
|
||||
document_types: Counter = Counter()
|
||||
storage_paths: Counter = Counter()
|
||||
document_ids: list[int] = []
|
||||
seen: set[int] = set()
|
||||
rows = queryset.order_by().values_list(
|
||||
"pk",
|
||||
"correspondent_id",
|
||||
"document_type_id",
|
||||
"storage_path_id",
|
||||
)
|
||||
for pk, correspondent, document_type, storage_path in rows:
|
||||
# A filter can join, repeating a document row.
|
||||
if pk in seen:
|
||||
continue
|
||||
seen.add(pk)
|
||||
document_ids.append(pk)
|
||||
# Documents without a value count under None, which selection()
|
||||
# never looks up.
|
||||
correspondents[correspondent] += 1
|
||||
document_types[document_type] += 1
|
||||
storage_paths[storage_path] += 1
|
||||
|
||||
def link_table_counts(link_model, field_name: str) -> Counter:
|
||||
result: Counter = Counter()
|
||||
batch_size = self.selection_data_link_batch_size
|
||||
for start in range(0, len(document_ids), batch_size):
|
||||
result.update(
|
||||
dict(
|
||||
link_model.objects.filter(
|
||||
document_id__in=document_ids[start : start + batch_size],
|
||||
)
|
||||
.values_list(field_name)
|
||||
.annotate(document_count=Count("document_id")),
|
||||
),
|
||||
)
|
||||
return result
|
||||
|
||||
def selection(model, counts: Counter) -> list[dict[str, int]]:
|
||||
# Every object is listed, with 0 for those with no matching document.
|
||||
return [
|
||||
{"id": pk, "document_count": counts.get(pk, 0)}
|
||||
for pk in model.objects.values_list("pk", flat=True)
|
||||
]
|
||||
|
||||
return {
|
||||
"selected_correspondents": [
|
||||
{"id": t.id, "document_count": t.document_count} for t in correspondents
|
||||
],
|
||||
"selected_tags": [
|
||||
{"id": t.id, "document_count": t.document_count} for t in tags
|
||||
],
|
||||
"selected_document_types": [
|
||||
{"id": t.id, "document_count": t.document_count} for t in document_types
|
||||
],
|
||||
"selected_storage_paths": [
|
||||
{"id": t.id, "document_count": t.document_count} for t in storage_paths
|
||||
],
|
||||
"selected_custom_fields": [
|
||||
{"id": t.id, "document_count": t.document_count} for t in custom_fields
|
||||
],
|
||||
"selected_correspondents": selection(
|
||||
Correspondent,
|
||||
correspondents,
|
||||
),
|
||||
"selected_tags": selection(
|
||||
Tag,
|
||||
link_table_counts(Document.tags.through, "tag_id"),
|
||||
),
|
||||
"selected_document_types": selection(
|
||||
DocumentType,
|
||||
document_types,
|
||||
),
|
||||
"selected_storage_paths": selection(
|
||||
StoragePath,
|
||||
storage_paths,
|
||||
),
|
||||
"selected_custom_fields": selection(
|
||||
CustomField,
|
||||
link_table_counts(CustomFieldInstance, "field_id"),
|
||||
),
|
||||
}
|
||||
|
||||
@classmethod
|
||||
|
||||
@@ -1,93 +0,0 @@
|
||||
import sqlite3
|
||||
from collections.abc import Generator
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from llama_index.core.schema import TextNode
|
||||
|
||||
from paperless_ai.tables import DocumentChunksTable
|
||||
from paperless_ai.vector_store import DB_FILENAME
|
||||
from paperless_ai.vector_store import DEFAULT_TABLE_NAME
|
||||
from paperless_ai.vector_store import VEC0_CHUNK_SIZE
|
||||
from paperless_ai.vector_store import PaperlessSqliteVecVectorStore
|
||||
|
||||
DIM = 16
|
||||
|
||||
|
||||
def make_node(node_id: str, document_id: int) -> TextNode:
|
||||
node = TextNode(
|
||||
id_=node_id,
|
||||
text="some text",
|
||||
metadata={"document_id": document_id, "modified": "2026-06-10T00:00:00"},
|
||||
)
|
||||
node.relationships = {}
|
||||
node.embedding = [document_id + i / 100 for i in range(DIM)]
|
||||
return node
|
||||
|
||||
|
||||
def vec_table_sql(db_path: Path) -> str:
|
||||
conn = sqlite3.connect(db_path)
|
||||
try:
|
||||
return conn.execute(
|
||||
"SELECT sql FROM sqlite_master WHERE name = ?",
|
||||
(DEFAULT_TABLE_NAME,),
|
||||
).fetchone()[0]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def create_legacy_vec_table(conn: sqlite3.Connection, dim: int) -> None:
|
||||
"""The table definition from before chunk_size was set explicitly."""
|
||||
conn.execute(
|
||||
f"CREATE VIRTUAL TABLE {DEFAULT_TABLE_NAME} USING vec0("
|
||||
"id TEXT PRIMARY KEY, document_id INTEGER, +node_content TEXT, "
|
||||
f"embedding float[{dim}] distance_metric=cosine)",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def store(tmp_path: Path) -> Generator[PaperlessSqliteVecVectorStore, None, None]:
|
||||
with PaperlessSqliteVecVectorStore(uri=str(tmp_path)) as store:
|
||||
yield store
|
||||
|
||||
|
||||
class TestVecChunkSize:
|
||||
def test_new_table_uses_configured_chunk_size(
|
||||
self,
|
||||
store: PaperlessSqliteVecVectorStore,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN an empty store
|
||||
WHEN the first document is indexed and the vec0 table is created
|
||||
THEN the table definition carries the configured chunk size
|
||||
"""
|
||||
store.upsert_document(1, [make_node("1-a", 1)])
|
||||
|
||||
assert f"chunk_size={VEC0_CHUNK_SIZE}" in vec_table_sql(tmp_path / DB_FILENAME)
|
||||
|
||||
def test_compact_moves_legacy_table_to_configured_chunk_size(
|
||||
self,
|
||||
store: PaperlessSqliteVecVectorStore,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN an index whose vec0 table was created with the default chunk size
|
||||
WHEN the index is compacted
|
||||
THEN the rebuilt table uses the configured chunk size
|
||||
AND no indexed chunks are lost
|
||||
"""
|
||||
with patch.object(
|
||||
PaperlessSqliteVecVectorStore,
|
||||
"_create_vec_table",
|
||||
staticmethod(create_legacy_vec_table),
|
||||
):
|
||||
store.upsert_document(1, [make_node("1-a", 1), make_node("1-b", 1)])
|
||||
store.upsert_document(2, [make_node("2-a", 2)])
|
||||
assert "chunk_size" not in vec_table_sql(tmp_path / DB_FILENAME)
|
||||
|
||||
store.compact(force=True)
|
||||
|
||||
assert f"chunk_size={VEC0_CHUNK_SIZE}" in vec_table_sql(tmp_path / DB_FILENAME)
|
||||
assert DocumentChunksTable.count(store.client) == 3
|
||||
@@ -1,89 +0,0 @@
|
||||
import sqlite3
|
||||
from collections.abc import Generator
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from llama_index.core.schema import TextNode
|
||||
|
||||
from paperless_ai.tables import DocumentChunksTable
|
||||
from paperless_ai.vector_store import DB_FILENAME
|
||||
from paperless_ai.vector_store import DB_PAGE_SIZE
|
||||
from paperless_ai.vector_store import PaperlessSqliteVecVectorStore
|
||||
|
||||
DIM = 16
|
||||
LEGACY_PAGE_SIZE = 4096
|
||||
|
||||
|
||||
def make_node(node_id: str, document_id: int) -> TextNode:
|
||||
node = TextNode(
|
||||
id_=node_id,
|
||||
text="some text",
|
||||
metadata={"document_id": document_id, "modified": "2026-06-10T00:00:00"},
|
||||
)
|
||||
node.relationships = {}
|
||||
node.embedding = [document_id + i / 100 for i in range(DIM)]
|
||||
return node
|
||||
|
||||
|
||||
def page_size_of(db_path: Path) -> int:
|
||||
conn = sqlite3.connect(db_path)
|
||||
try:
|
||||
return conn.execute("PRAGMA page_size").fetchone()[0]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def legacy_page_size_db(tmp_path: Path) -> Path:
|
||||
"""An existing index file created before the page size was set."""
|
||||
db_path = tmp_path / DB_FILENAME
|
||||
conn = sqlite3.connect(db_path)
|
||||
conn.execute(f"PRAGMA page_size={LEGACY_PAGE_SIZE}")
|
||||
conn.execute("CREATE TABLE _seed(x)")
|
||||
conn.execute("DROP TABLE _seed")
|
||||
conn.close()
|
||||
assert page_size_of(db_path) == LEGACY_PAGE_SIZE
|
||||
return db_path
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def store(tmp_path: Path) -> Generator[PaperlessSqliteVecVectorStore, None, None]:
|
||||
with PaperlessSqliteVecVectorStore(uri=str(tmp_path)) as store:
|
||||
yield store
|
||||
|
||||
|
||||
class TestIndexPageSize:
|
||||
def test_new_index_file_uses_configured_page_size(
|
||||
self,
|
||||
store: PaperlessSqliteVecVectorStore,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN no existing index file
|
||||
WHEN a store is opened and a document is indexed
|
||||
THEN the database file uses the configured page size
|
||||
"""
|
||||
store.upsert_document(1, [make_node("1-a", 1)])
|
||||
|
||||
assert page_size_of(tmp_path / DB_FILENAME) == DB_PAGE_SIZE
|
||||
|
||||
def test_compact_moves_legacy_file_to_configured_page_size(
|
||||
self,
|
||||
legacy_page_size_db: Path,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN an existing index file created with a different page size
|
||||
WHEN the index is compacted
|
||||
THEN the rebuilt file uses the configured page size
|
||||
AND no indexed chunks are lost
|
||||
"""
|
||||
with PaperlessSqliteVecVectorStore(
|
||||
uri=str(legacy_page_size_db.parent),
|
||||
) as store:
|
||||
store.upsert_document(1, [make_node("1-a", 1), make_node("1-b", 1)])
|
||||
assert page_size_of(legacy_page_size_db) == LEGACY_PAGE_SIZE
|
||||
|
||||
store.compact(force=True)
|
||||
|
||||
assert page_size_of(legacy_page_size_db) == DB_PAGE_SIZE
|
||||
assert DocumentChunksTable.count(store.client) == 2
|
||||
@@ -47,20 +47,6 @@ SCHEMA_VERSION = 2
|
||||
# a rebuild copies the live rows into a fresh table.
|
||||
COMPACT_BLOAT_RATIO = 2.0
|
||||
|
||||
# Rows per vec0 vector chunk (vec0's default is 1024, which makes each chunk a
|
||||
# multi-megabyte blob). On spinning disks, inserts and re-indexing were dozens
|
||||
# of times faster with small chunks, at no cost in query speed or file size.
|
||||
# Only applies to tables created from here on; an existing index picks it up
|
||||
# the next time it is rebuilt (compact or rebuild).
|
||||
VEC0_CHUNK_SIZE = 32
|
||||
|
||||
# SQLite page size for the index file, in bytes. With the small vec0 chunks above,
|
||||
# 64 KiB pages made cold scans about 35% faster and warm ones about 45% faster,
|
||||
# at the cost of a larger write-ahead log (up to about 70 MB). Only takes effect
|
||||
# on a newly created file, so like the chunk size it reaches an existing index
|
||||
# when it is rebuilt.
|
||||
DB_PAGE_SIZE = 65536
|
||||
|
||||
# Number of rows fetched/copied per batch whenever this module streams rows
|
||||
# instead of materializing them all at once, keeping memory bounded regardless
|
||||
# of index size -- used by compact()'s rebuild, m0001_v1_to_v2's migration
|
||||
@@ -236,7 +222,6 @@ class PaperlessSqliteVecVectorStore(BasePydanticVectorStore):
|
||||
conn.enable_load_extension(True) # noqa: FBT003
|
||||
sqlite_vec.load(conn)
|
||||
conn.enable_load_extension(False) # noqa: FBT003
|
||||
conn.execute(f"PRAGMA page_size={DB_PAGE_SIZE}")
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
conn.execute("PRAGMA synchronous=NORMAL")
|
||||
IndexMetaTable.create(conn)
|
||||
@@ -344,8 +329,7 @@ class PaperlessSqliteVecVectorStore(BasePydanticVectorStore):
|
||||
+ " +node_content TEXT,"
|
||||
+ " embedding float["
|
||||
+ str(int(dim))
|
||||
+ "] distance_metric=cosine, chunk_size="
|
||||
+ str(VEC0_CHUNK_SIZE)
|
||||
+ "] distance_metric=cosine"
|
||||
+ ")",
|
||||
)
|
||||
|
||||
|
||||
Reference in new issue
Block a user