From 4f7f280f32d6c67eeca926e5b09e358bb0641a7e Mon Sep 17 00:00:00 2001 From: stumpylog <797416+stumpylog@users.noreply.github.com> Date: Thu, 30 Jul 2026 07:58:27 -0700 Subject: [PATCH] Refactor: rename COMPACT_BATCH_SIZE to BATCH_SIZE, no longer compact()-specific --- src/paperless_ai/migrations/m0001_v1_to_v2.py | 4 ++-- src/paperless_ai/tables.py | 2 +- src/paperless_ai/tests/test_vector_store.py | 2 +- src/paperless_ai/vector_store.py | 14 ++++++++------ 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/paperless_ai/migrations/m0001_v1_to_v2.py b/src/paperless_ai/migrations/m0001_v1_to_v2.py index c7edf7f41..18409d5fc 100644 --- a/src/paperless_ai/migrations/m0001_v1_to_v2.py +++ b/src/paperless_ai/migrations/m0001_v1_to_v2.py @@ -7,7 +7,7 @@ from paperless_ai.tables import DocumentChunksTable from paperless_ai.tables import DocumentMetaRow from paperless_ai.tables import DocumentMetaTable from paperless_ai.tables import IndexMetaTable -from paperless_ai.vector_store import COMPACT_BATCH_SIZE +from paperless_ai.vector_store import BATCH_SIZE from paperless_ai.vector_store import DEFAULT_TABLE_NAME # v1's vec0 shape has never changed since it first shipped and is the ONLY @@ -75,7 +75,7 @@ def _migrate_v1_to_v2( dst_conn.execute("BEGIN IMMEDIATE") src_cursor = src_conn.execute(_V1_SELECT) live = 0 - while batch := src_cursor.fetchmany(COMPACT_BATCH_SIZE): + while batch := src_cursor.fetchmany(BATCH_SIZE): vec0_rows = [] chunk_rows = [] meta_by_document: dict[int, str] = {} diff --git a/src/paperless_ai/tables.py b/src/paperless_ai/tables.py index f3ac5710f..c8d6d838c 100644 --- a/src/paperless_ai/tables.py +++ b/src/paperless_ai/tables.py @@ -131,7 +131,7 @@ class DocumentMetaTable: connections (compact()/migrations) -- an unbounded fetchall here would defeat the same OOM-avoidance the vec0 row copy already relies on. batch_size has no default: forces the call site to think about - it (pass COMPACT_BATCH_SIZE).""" + it (pass BATCH_SIZE).""" cursor = src_conn.execute( "SELECT document_id, modified FROM document_meta", ) diff --git a/src/paperless_ai/tests/test_vector_store.py b/src/paperless_ai/tests/test_vector_store.py index e4cf883bb..3d2dc13f5 100644 --- a/src/paperless_ai/tests/test_vector_store.py +++ b/src/paperless_ai/tests/test_vector_store.py @@ -508,7 +508,7 @@ class TestCompact: A tiny batch size forces several fetchmany()/executemany() cycles so a regression in the streaming loop (dropped tail, off-by-one) surfaces. """ - monkeypatch.setattr("paperless_ai.vector_store.COMPACT_BATCH_SIZE", 3) + monkeypatch.setattr("paperless_ai.vector_store.BATCH_SIZE", 3) store.add([make_node(f"n{i}", 1, seed=float(i)) for i in range(10)]) store.compact(force=True) ids = {n.node_id for n in store.get_nodes(filters=_in_filter([1]))} diff --git a/src/paperless_ai/vector_store.py b/src/paperless_ai/vector_store.py index bb3007323..975b6c57b 100644 --- a/src/paperless_ai/vector_store.py +++ b/src/paperless_ai/vector_store.py @@ -47,10 +47,12 @@ SCHEMA_VERSION = 2 # a rebuild copies the live rows into a fresh table. COMPACT_BLOAT_RATIO = 2.0 -# compact(): number of rows copied per executemany() when rebuilding the file. -# Rows are streamed from the source cursor in batches of this size rather than -# materialized all at once, keeping memory bounded regardless of index size. -COMPACT_BATCH_SIZE = 500 +# Number of rows fetched/copied per batch whenever this module streams rows +# instead of materializing them all at once, keeping memory bounded regardless +# of index size -- used by compact()'s rebuild, m0001_v1_to_v2's migration +# copy, and DocumentMetaTable.copy_all(). No longer compact()-specific, hence +# the plain name. +BATCH_SIZE = 500 # Filterable vec0 metadata columns. _build_where() only ever receives filter # keys we construct ourselves, but allowlisting keeps SQL identifiers safe by @@ -605,7 +607,7 @@ class PaperlessSqliteVecVectorStore(BasePydanticVectorStore): + DEFAULT_TABLE_NAME, ) copied = 0 - while batch := src_cursor.fetchmany(COMPACT_BATCH_SIZE): + while batch := src_cursor.fetchmany(BATCH_SIZE): dst_conn.executemany( _INSERT, [ @@ -623,7 +625,7 @@ class PaperlessSqliteVecVectorStore(BasePydanticVectorStore): (ChunkRow(r["id"], r["document_id"]) for r in batch), ) copied += len(batch) - DocumentMetaTable.copy_all(src_conn, dst_conn, COMPACT_BATCH_SIZE) + DocumentMetaTable.copy_all(src_conn, dst_conn, BATCH_SIZE) # Reset the cumulative counter: after a rebuild, total_inserts == live. IndexMetaTable.reset_total_inserts(dst_conn, copied) dst_conn.execute("COMMIT")