From 24b86770df6bac33cc416747ccb590937cbb365c Mon Sep 17 00:00:00 2001 From: stumpylog <797416+stumpylog@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:39:02 -0700 Subject: [PATCH] Perf: store document_chunks.document_id as INTEGER document_chunks is a plain SQLite table, not a vec0 virtual table, so (unlike vec0's own TEXT metadata column) it gets normal type-affinity coercion between the TEXT document ids used elsewhere in this module and an INTEGER column -- verified empirically. INTEGER keys make the per-document delete lookup this table exists for cheaper: smaller index entries and integer rather than text B-tree comparisons. --- src/paperless_ai/tests/test_vector_store.py | 2 +- src/paperless_ai/vector_store.py | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/paperless_ai/tests/test_vector_store.py b/src/paperless_ai/tests/test_vector_store.py index e11dde1ea..6509a56d3 100644 --- a/src/paperless_ai/tests/test_vector_store.py +++ b/src/paperless_ai/tests/test_vector_store.py @@ -103,7 +103,7 @@ def _chunk_index_rows( sql += " WHERE document_id = ?" params.append(document_id) rows = store.client.execute(sql + " ORDER BY chunk_id", params).fetchall() - return [(r["chunk_id"], r["document_id"]) for r in rows] + return [(r["chunk_id"], str(r["document_id"])) for r in rows] @contextmanager diff --git a/src/paperless_ai/vector_store.py b/src/paperless_ai/vector_store.py index fc94fe6f2..27b320d2e 100644 --- a/src/paperless_ai/vector_store.py +++ b/src/paperless_ai/vector_store.py @@ -241,9 +241,15 @@ class PaperlessSqliteVecVectorStore(BasePydanticVectorStore): # (MATCH) query; a plain `WHERE document_id = ?` is a full table scan # regardless of index size. This plain, indexed table is how delete()/ # upsert_document() find a document's chunk ids without that scan. + # document_id is INTEGER here (unlike vec0's own TEXT metadata + # column): this is a normal SQLite table, so standard type affinity + # correctly coerces the TEXT document ids written/looked-up + # elsewhere in this module -- it does not share vec0's own + # metadata-column comparison code, which silently mismatches + # non-TEXT bound values instead of coercing them. conn.execute( "CREATE TABLE IF NOT EXISTS document_chunks " - "(chunk_id TEXT PRIMARY KEY, document_id TEXT NOT NULL)", + "(chunk_id TEXT PRIMARY KEY, document_id INTEGER NOT NULL)", ) conn.execute( "CREATE INDEX IF NOT EXISTS idx_document_chunks_document_id "