From feac5263189434ab828a7aba887a3e042c587294 Mon Sep 17 00:00:00 2001 From: stumpylog <797416+stumpylog@users.noreply.github.com> Date: Tue, 28 Jul 2026 15:36:42 -0700 Subject: [PATCH] Perf: count compact()'s live rows via document_chunks, not a vec0 scan compact() ran SELECT count(*) on the vec0 table itself purely to compute the bloat ratio, on every update_llm_index() call -- including small scoped ones from bulk_update_documents. Like any other document_id-scale lookup, count(*) has no point-lookup fast path on vec0's own table, so this is a full scan regardless of index size. document_chunks is kept in lockstep with the vec0 table by construction and is a plain indexed table, so it gives an identical count far more cheaply. Only affects the bloat-ratio heuristic and a log line -- the actual rebuild already streams and counts real vec0 rows independently (_copy_rows()'s own return value), so this can't desync compaction's correctness even in the brief window before a store's document_chunks is fully backfilled by migration. Co-Authored-By: Claude Sonnet 5 --- src/paperless_ai/vector_store.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/paperless_ai/vector_store.py b/src/paperless_ai/vector_store.py index 5a866488f..06a1cae69 100644 --- a/src/paperless_ai/vector_store.py +++ b/src/paperless_ai/vector_store.py @@ -599,9 +599,12 @@ class PaperlessSqliteVecVectorStore(BasePydanticVectorStore): """ if not self.table_exists(): return - live = self._conn.execute( - "SELECT count(*) FROM " + DEFAULT_TABLE_NAME, - ).fetchone()[0] + # document_chunks is a plain indexed table kept in lockstep with the + # vec0 table (see _index_chunks()/_delete_chunks_by_document_id()), + # so it gives an identical live-row count to `SELECT count(*) FROM + # documents` without vec0's full table scan -- count(*) has no + # equivalent point-lookup fast path, unlike an EQ constraint on `id`. + live = self._conn.execute("SELECT count(*) FROM document_chunks").fetchone()[0] total = int(self._meta_get("total_inserts") or str(live)) if not force and total <= max(live, 1) * COMPACT_BLOAT_RATIO: return