Perf: store document_chunks.document_id as INTEGER

document_chunks is a plain SQLite table, not a vec0 virtual table, so
(unlike vec0's own TEXT metadata column) it gets normal type-affinity
coercion between the TEXT document ids used elsewhere in this module and
an INTEGER column -- verified empirically. INTEGER keys make the
per-document delete lookup this table exists for cheaper: smaller index
entries and integer rather than text B-tree comparisons.
This commit is contained in:
stumpylog
2026-07-28 09:39:02 -07:00
parent 6a2972f313
commit 24b86770df
2 changed files with 8 additions and 2 deletions
+1 -1
View File
@@ -103,7 +103,7 @@ def _chunk_index_rows(
sql += " WHERE document_id = ?"
params.append(document_id)
rows = store.client.execute(sql + " ORDER BY chunk_id", params).fetchall()
return [(r["chunk_id"], r["document_id"]) for r in rows]
return [(r["chunk_id"], str(r["document_id"])) for r in rows]
@contextmanager
+7 -1
View File
@@ -241,9 +241,15 @@ class PaperlessSqliteVecVectorStore(BasePydanticVectorStore):
# (MATCH) query; a plain `WHERE document_id = ?` is a full table scan
# regardless of index size. This plain, indexed table is how delete()/
# upsert_document() find a document's chunk ids without that scan.
# document_id is INTEGER here (unlike vec0's own TEXT metadata
# column): this is a normal SQLite table, so standard type affinity
# correctly coerces the TEXT document ids written/looked-up
# elsewhere in this module -- it does not share vec0's own
# metadata-column comparison code, which silently mismatches
# non-TEXT bound values instead of coercing them.
conn.execute(
"CREATE TABLE IF NOT EXISTS document_chunks "
"(chunk_id TEXT PRIMARY KEY, document_id TEXT NOT NULL)",
"(chunk_id TEXT PRIMARY KEY, document_id INTEGER NOT NULL)",
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_document_chunks_document_id "