mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-07-29 23:34:56 +00:00
Perf: store document_chunks.document_id as INTEGER
document_chunks is a plain SQLite table, not a vec0 virtual table, so (unlike vec0's own TEXT metadata column) it gets normal type-affinity coercion between the TEXT document ids used elsewhere in this module and an INTEGER column -- verified empirically. INTEGER keys make the per-document delete lookup this table exists for cheaper: smaller index entries and integer rather than text B-tree comparisons.
This commit is contained in:
@@ -103,7 +103,7 @@ def _chunk_index_rows(
|
||||
sql += " WHERE document_id = ?"
|
||||
params.append(document_id)
|
||||
rows = store.client.execute(sql + " ORDER BY chunk_id", params).fetchall()
|
||||
return [(r["chunk_id"], r["document_id"]) for r in rows]
|
||||
return [(r["chunk_id"], str(r["document_id"])) for r in rows]
|
||||
|
||||
|
||||
@contextmanager
|
||||
|
||||
@@ -241,9 +241,15 @@ class PaperlessSqliteVecVectorStore(BasePydanticVectorStore):
|
||||
# (MATCH) query; a plain `WHERE document_id = ?` is a full table scan
|
||||
# regardless of index size. This plain, indexed table is how delete()/
|
||||
# upsert_document() find a document's chunk ids without that scan.
|
||||
# document_id is INTEGER here (unlike vec0's own TEXT metadata
|
||||
# column): this is a normal SQLite table, so standard type affinity
|
||||
# correctly coerces the TEXT document ids written/looked-up
|
||||
# elsewhere in this module -- it does not share vec0's own
|
||||
# metadata-column comparison code, which silently mismatches
|
||||
# non-TEXT bound values instead of coercing them.
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS document_chunks "
|
||||
"(chunk_id TEXT PRIMARY KEY, document_id TEXT NOT NULL)",
|
||||
"(chunk_id TEXT PRIMARY KEY, document_id INTEGER NOT NULL)",
|
||||
)
|
||||
conn.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_document_chunks_document_id "
|
||||
|
||||
Reference in New Issue
Block a user