From 15938945d7f371e573eec6ef5ff4d686e453e47b Mon Sep 17 00:00:00 2001 From: stumpylog <797416+stumpylog@users.noreply.github.com> Date: Tue, 28 Jul 2026 12:32:59 -0700 Subject: [PATCH] Simplify: dedupe file-swap rebuild, migration-check locking, and test mocks Prompted by an automated simplification review of the branch. Applies the highest-value findings, all behavior-preserving (full suite still green): - vector_store.py: extract _rebuild_file() (temp-file lifecycle: open, populate, swap-in-or-discard-on-failure) and _rebuild_into() (create table, copy meta, stream rows) out of compact() and the v1->v2 migration, which had duplicated both almost verbatim. The migration module shrinks to a single _rebuild_into() call instead of reaching into four PaperlessSqliteVecVectorStore privates. - vector_store.py: extract _stored_schema_version() out of has_pending_migration() and check_and_run_migrations(), which duplicated the same schema_version read and its "missing key means current" default. - indexing.py: extract _with_exclusive_access(), collapsing three identical _exclude_readers()/Timeout/log-and-skip blocks (compaction in update_llm_index() and llm_index_compact(), the migration check in _check_and_run_migrations()) to one line each. Also trims _check_and_run_migrations()'s docstring, which had grown to restate content already documented on has_pending_migration() and check_and_run_migrations(), including a claim that was now stale (llm_index_migrate() is a second consumer of the re-embed signal). - test_ai_indexing.py: add a mock_store fixture for the write_store()-yields-a-MagicMock pattern that 8 tests were hand-rolling identically. - migrations/__init__.py: consolidate the "how to add a migration" instructions to one place instead of three. - docs/administration.md: fix two now-inaccurate claims -- the bare-metal migrate step doesn't run "automatically" (it's the manual step being documented), and the self-contradictory "if enabled... no-op if disabled" phrasing on the same step. Co-Authored-By: Claude Sonnet 5 --- docs/administration.md | 19 ++- src/paperless_ai/indexing.py | 70 ++++----- src/paperless_ai/migrations/__init__.py | 33 +++-- .../migrations/m0001_add_document_chunks.py | 26 ++-- src/paperless_ai/tests/test_ai_indexing.py | 94 ++++-------- src/paperless_ai/vector_store.py | 138 +++++++++++------- 6 files changed, 174 insertions(+), 206 deletions(-) diff --git a/docs/administration.md b/docs/administration.md index 2911b2442..004ef005f 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -212,16 +212,15 @@ following: This is a no-op if the index is already up to date, so it is safe to run on every upgrade. -5. If the LLM index is enabled, apply any pending LLM index schema - migrations. +5. Apply any pending LLM index schema migrations. ```shell-session cd src python3 manage.py document_llmindex migrate ``` - This is a no-op if the index is already up to date (or the LLM index - is disabled), so it is safe to run on every upgrade. + This is a no-op if the index is already up to date, or if the LLM index + is disabled, so it is safe to run on every upgrade. ### Database Upgrades @@ -557,12 +556,12 @@ Specify `compact` to reclaim space and optimize the on-disk vector store. Specify `migrate` to apply any pending index schema migrations without a full reindex. This is a no-op if the index is already up to date, so it is safe to run on every -startup or upgrade; it is what the container's startup sequence and the -[bare-metal upgrade steps](#bare-metal-updating) run automatically. If a pending -migration would require re-embedding every document, `migrate` only logs a warning and -leaves the index as-is -- re-embedding can be slow and, for a metered embedding backend, -cost money, so it is never triggered automatically. Run `rebuild` yourself when you are -ready. +startup or upgrade; the container's startup sequence runs it automatically, and the +[bare-metal upgrade steps](#bare-metal-updating) include it as a manual step. If a +pending migration would require re-embedding every document, `migrate` only logs a +warning and leaves the index as-is -- re-embedding can be slow and, for a metered +embedding backend, cost money, so it is never triggered automatically. Run `rebuild` +yourself when you are ready. !!! note diff --git a/src/paperless_ai/indexing.py b/src/paperless_ai/indexing.py index 2d8d8d5b5..69d99aa55 100644 --- a/src/paperless_ai/indexing.py +++ b/src/paperless_ai/indexing.py @@ -144,6 +144,24 @@ def _exclude_readers(): lock.close() +def _with_exclusive_access(operation: str, fn): + """Run ``fn()`` with exclusive index access (see ``_exclude_readers()``), + for compaction/migration file swaps that must not run while readers are + active. Returns ``fn()``'s result, or None (after logging) if active + readers do not drain within ``LLM_INDEX_COMPACTION_LOCK_TIMEOUT`` -- + callers skip the operation this run; it retries next time. + """ + try: + with _exclude_readers(): + return fn() + except Timeout: + logger.info( + "Skipping LLM index %s: index readers are active; will retry next run.", + operation, + ) + return None + + @contextmanager def write_store(embed_model_name: str | None = None): """Acquire the write lock and yield the vector store. @@ -325,34 +343,18 @@ def _exclude_document_id_filter(document_id: int | str): def _check_and_run_migrations(store: "PaperlessSqliteVecVectorStore") -> bool: - """Run any pending structural migrations before a write. Returns True - when a pending re-embed migration was encountered, signaling the - caller must force a full rebuild. - - check_and_run_migrations() never re-embeds anything itself: it only - applies cheap, local structural migrations, and for a re-embed - migration returns True as a pure signal with no side effects. Actual - re-embedding only happens in update_llm_index()'s explicit rebuild - path below, so it is safe to call this before any write -- including - delete()/upsert_document() -- without risking an unwanted (and - possibly costly, if the embedding backend is metered) re-embed. - - has_pending_migration() gates the exclusive-access check: in the - common case (already at SCHEMA_VERSION) this returns immediately - without ever contending with readers or a concurrent compaction -- - normal writes must not be gated by that lock. + """Run any pending structural migrations, returning True if a pending + re-embed migration needs the caller to force a rebuild -- never + triggered automatically here. Safe to call before any write, including + delete()/upsert_document(): has_pending_migration() (see its docstring) + keeps this a no-op, with no exclusive access taken, once the store is + current. """ if not store.has_pending_migration(): return False - try: - with _exclude_readers(): - return store.check_and_run_migrations() - except Timeout: - logger.info( - "Skipping LLM index migration check: index readers are active; " - "will retry next run.", - ) - return False + return bool( + _with_exclusive_access("migration check", store.check_and_run_migrations), + ) def update_llm_index( @@ -447,14 +449,7 @@ def update_llm_index( else "No changes detected in LLM index." ) - try: - with _exclude_readers(): - store.compact() - except Timeout: - logger.info( - "Skipping LLM index compaction: index readers are active; " - "will retry next run.", - ) + _with_exclusive_access("compaction", store.compact) return msg @@ -501,14 +496,7 @@ def llm_index_migrate() -> None: def llm_index_compact() -> None: """Compact the index immediately, rebuilding the table to reclaim space.""" with write_store() as store: - try: - with _exclude_readers(): - store.compact(force=True) - except Timeout: - logger.info( - "Skipping LLM index compaction: index readers are active; " - "will retry next run.", - ) + _with_exclusive_access("compaction", lambda: store.compact(force=True)) def llm_index_remove_document(document: Document): diff --git a/src/paperless_ai/migrations/__init__.py b/src/paperless_ai/migrations/__init__.py index 72b07f121..98a2f5496 100644 --- a/src/paperless_ai/migrations/__init__.py +++ b/src/paperless_ai/migrations/__init__.py @@ -2,20 +2,20 @@ Each migration lives in its own module here, named ``mNNNN_description.py`` (e.g. ``m0001_add_document_chunks.py`` -- a leading digit isn't a valid -Python identifier, hence the ``m`` prefix, unlike Django's own -``NNNN_description.py`` migrations, which get away with it because Django's -migration loader uses ``importlib.import_module()`` with a dynamic string -path instead of a static import statement), and registers itself into +Python identifier, hence the ``m`` prefix, unlike Django's own numbered +migrations, which load via a dynamic ``importlib.import_module()`` call +rather than a static import statement), and registers itself into ``MIGRATIONS`` at import time. ``vector_store.py`` imports those modules at the bottom of the file, purely for that registration side effect, after -``PaperlessSqliteVecVectorStore`` and ``_copy_rows`` are fully defined -- -migrations need both to implement ``apply()``. +``PaperlessSqliteVecVectorStore`` is fully defined -- migrations need it to +implement ``apply()`` (see ``Migration`` below). -Add a new migration by adding a new ``mNNNN_description.py`` module here that -imports ``PaperlessSqliteVecVectorStore``/``_copy_rows`` from -``paperless_ai.vector_store``, defines its ``apply()``, and appends a -``Migration`` to ``MIGRATIONS``; then import that module at the bottom of -``vector_store.py`` and bump ``SCHEMA_VERSION``. +To add a new migration: add a new ``mNNNN_description.py`` module here that +imports ``PaperlessSqliteVecVectorStore`` from ``paperless_ai.vector_store``, +defines its ``apply()`` (most likely just a call to +``PaperlessSqliteVecVectorStore._rebuild_into()``, see ``Migration`` below), +and appends a ``Migration`` to ``MIGRATIONS``; then import that module at the +bottom of ``vector_store.py`` and bump ``SCHEMA_VERSION`` there. """ import sqlite3 @@ -30,11 +30,12 @@ class Migration: """A schema migration for the sqlite-vec vector store. kind="structural": rows are copied into a new-schema file with no - re-embedding needed. Supply ``apply(src_conn, dst_conn, dim)`` which - must create the vec0 table in ``dst_conn``, copy all rows from - ``src_conn``, and write ``dim`` / ``embed_model`` / ``total_inserts`` to - ``dst_conn``'s ``index_meta``. ``schema_version`` is written by the - migration runner after ``apply`` returns. + re-embedding needed. Supply ``apply(src_conn, dst_conn, dim)``, which + must create the vec0 table in ``dst_conn`` and copy ``src_conn``'s rows + and index_meta into it -- usually just a call to + ``PaperlessSqliteVecVectorStore._rebuild_into(src_conn, dst_conn, dim)``. + ``schema_version`` is written by the migration runner after ``apply`` + returns, not by ``apply`` itself. kind="re-embed": the new schema requires fresh embeddings. ``check_and_run_migrations()`` returns True when it encounters one of diff --git a/src/paperless_ai/migrations/m0001_add_document_chunks.py b/src/paperless_ai/migrations/m0001_add_document_chunks.py index fae96edf1..beb0141ad 100644 --- a/src/paperless_ai/migrations/m0001_add_document_chunks.py +++ b/src/paperless_ai/migrations/m0001_add_document_chunks.py @@ -3,7 +3,6 @@ import sqlite3 from paperless_ai.migrations import MIGRATIONS from paperless_ai.migrations import Migration from paperless_ai.vector_store import PaperlessSqliteVecVectorStore -from paperless_ai.vector_store import _copy_rows def _migrate_v1_to_v2_add_document_chunks( @@ -18,22 +17,17 @@ def _migrate_v1_to_v2_add_document_chunks( full table scan on the document_id metadata column. Every row written before this migration predates that table, so without backfilling, deleting a pre-migration document would find zero chunk ids and leave its - vec0 rows permanently orphaned. Backfilling is a plain row copy into the - new-schema file (``_copy_rows``, the same helper compact() uses), which - records every copied row in document_chunks as it goes. + vec0 rows permanently orphaned. Backfilling is just the same rebuild + compact() uses (_rebuild_into), minus "schema_version" -- the caller + (_run_structural_migration) sets that to this migration's target version + instead of preserving the source's. """ - PaperlessSqliteVecVectorStore._create_vec_table(dst_conn, dim) - PaperlessSqliteVecVectorStore._meta_set_on(dst_conn, "dim", str(dim)) - embed_model = PaperlessSqliteVecVectorStore._meta_get_on(src_conn, "embed_model") - if embed_model is not None: - PaperlessSqliteVecVectorStore._meta_set_on(dst_conn, "embed_model", embed_model) - - dst_conn.execute("BEGIN IMMEDIATE") - live = _copy_rows(src_conn, dst_conn) - # This migration only ever copies live rows (like compact()), so the - # cumulative counter resets to match -- the new file has no bloat yet. - PaperlessSqliteVecVectorStore._meta_set_on(dst_conn, "total_inserts", str(live)) - dst_conn.execute("COMMIT") + PaperlessSqliteVecVectorStore._rebuild_into( + src_conn, + dst_conn, + dim, + meta_keys=("dim", "embed_model"), + ) MIGRATIONS.append( diff --git a/src/paperless_ai/tests/test_ai_indexing.py b/src/paperless_ai/tests/test_ai_indexing.py index 895d5113a..f5b606a71 100644 --- a/src/paperless_ai/tests/test_ai_indexing.py +++ b/src/paperless_ai/tests/test_ai_indexing.py @@ -37,6 +37,23 @@ def real_document(db: None) -> Document: ) +@pytest.fixture +def mock_store(mocker: pytest_mock.MockerFixture) -> MagicMock: + """The MagicMock store yielded by every ``with write_store() as store:`` + block, for tests that only care what indexing.py does with the store, + not what the store itself does. + """ + store = mocker.MagicMock() + mocker.patch( + "paperless_ai.indexing.write_store", + return_value=mocker.MagicMock( + __enter__=mocker.MagicMock(return_value=store), + __exit__=mocker.MagicMock(return_value=False), + ), + ) + return store + + @pytest.mark.django_db def test_build_document_node(real_document: Document) -> None: nodes = indexing.build_document_node(real_document) @@ -762,18 +779,9 @@ class TestLlmIndexAddOrUpdateDocumentEmptyContent: @pytest.mark.django_db def test_llm_index_compact_uses_force( temp_llm_index_dir: Path, - mocker: pytest_mock.MockerFixture, + mock_store: MagicMock, ) -> None: """compact must use force=True to rebuild the table and reclaim space immediately.""" - mock_store = mocker.MagicMock() - mocker.patch( - "paperless_ai.indexing.write_store", - return_value=mocker.MagicMock( - __enter__=mocker.MagicMock(return_value=mock_store), - __exit__=mocker.MagicMock(return_value=False), - ), - ) - indexing.llm_index_compact() mock_store.compact.assert_called_once_with(force=True) @@ -812,6 +820,7 @@ class TestLlmIndexMigrate: self, temp_llm_index_dir: Path, mocker: pytest_mock.MockerFixture, + mock_store: MagicMock, caplog: pytest.LogCaptureFixture, ) -> None: """ @@ -826,16 +835,8 @@ class TestLlmIndexMigrate: mock_config = mocker.MagicMock() mock_config.llm_index_enabled = True mocker.patch("paperless_ai.indexing.AIConfig", return_value=mock_config) - mock_store = mocker.MagicMock() mock_store.has_pending_migration.return_value = True mock_store.check_and_run_migrations.return_value = False - mocker.patch( - "paperless_ai.indexing.write_store", - return_value=mocker.MagicMock( - __enter__=mocker.MagicMock(return_value=mock_store), - __exit__=mocker.MagicMock(return_value=False), - ), - ) with caplog.at_level("WARNING"): indexing.llm_index_migrate() @@ -847,6 +848,7 @@ class TestLlmIndexMigrate: self, temp_llm_index_dir: Path, mocker: pytest_mock.MockerFixture, + mock_store: MagicMock, caplog: pytest.LogCaptureFixture, ) -> None: """ @@ -864,16 +866,8 @@ class TestLlmIndexMigrate: mock_config = mocker.MagicMock() mock_config.llm_index_enabled = True mocker.patch("paperless_ai.indexing.AIConfig", return_value=mock_config) - mock_store = mocker.MagicMock() mock_store.has_pending_migration.return_value = True mock_store.check_and_run_migrations.return_value = True - mocker.patch( - "paperless_ai.indexing.write_store", - return_value=mocker.MagicMock( - __enter__=mocker.MagicMock(return_value=mock_store), - __exit__=mocker.MagicMock(return_value=False), - ), - ) with caplog.at_level("WARNING"): indexing.llm_index_migrate() @@ -895,16 +889,9 @@ class TestLlmIndexLocking: self, temp_llm_index_dir: Path, mock_embed_model: FakeEmbedding, + mock_store: MagicMock, mocker: pytest_mock.MockerFixture, ) -> None: - mock_store = MagicMock() - mocker.patch( - "paperless_ai.indexing.write_store", - return_value=mocker.MagicMock( - __enter__=mocker.MagicMock(return_value=mock_store), - __exit__=mocker.MagicMock(return_value=False), - ), - ) mock_node = MagicMock() mock_node.get_content.return_value = "fake node text" mocker.patch( @@ -923,6 +910,7 @@ class TestLlmIndexLocking: self, temp_llm_index_dir: Path, mock_embed_model: FakeEmbedding, + mock_store: MagicMock, mocker: pytest_mock.MockerFixture, *, has_pending: bool, @@ -939,15 +927,7 @@ class TestLlmIndexLocking: that check_and_run_migrations() requires - upsert_document() is called either way """ - mock_store = MagicMock() mock_store.has_pending_migration.return_value = has_pending - mocker.patch( - "paperless_ai.indexing.write_store", - return_value=mocker.MagicMock( - __enter__=mocker.MagicMock(return_value=mock_store), - __exit__=mocker.MagicMock(return_value=False), - ), - ) mock_node = MagicMock() mock_node.get_content.return_value = "fake node text" mocker.patch( @@ -964,17 +944,8 @@ class TestLlmIndexLocking: def test_remove_document_uses_write_store( self, temp_llm_index_dir: Path, - mocker: pytest_mock.MockerFixture, + mock_store: MagicMock, ) -> None: - mock_store = MagicMock() - mocker.patch( - "paperless_ai.indexing.write_store", - return_value=mocker.MagicMock( - __enter__=mocker.MagicMock(return_value=mock_store), - __exit__=mocker.MagicMock(return_value=False), - ), - ) - doc = MagicMock(spec=Document) doc.id = 1 indexing.llm_index_remove_document(doc) @@ -985,7 +956,7 @@ class TestLlmIndexLocking: def test_remove_document_runs_migration_check_only_when_pending( self, temp_llm_index_dir: Path, - mocker: pytest_mock.MockerFixture, + mock_store: MagicMock, *, has_pending: bool, ) -> None: @@ -1002,15 +973,7 @@ class TestLlmIndexLocking: test_normal_write_is_not_gated_by_the_compaction_lock) - delete() is called either way """ - mock_store = MagicMock() mock_store.has_pending_migration.return_value = has_pending - mocker.patch( - "paperless_ai.indexing.write_store", - return_value=mocker.MagicMock( - __enter__=mocker.MagicMock(return_value=mock_store), - __exit__=mocker.MagicMock(return_value=False), - ), - ) doc = DocumentFactory.build(id=1) indexing.llm_index_remove_document(doc) @@ -1022,16 +985,9 @@ class TestLlmIndexLocking: self, temp_llm_index_dir: Path, mock_embed_model: FakeEmbedding, + mock_store: MagicMock, mocker: pytest_mock.MockerFixture, ) -> None: - mock_store = MagicMock() - mocker.patch( - "paperless_ai.indexing.write_store", - return_value=mocker.MagicMock( - __enter__=mocker.MagicMock(return_value=mock_store), - __exit__=mocker.MagicMock(return_value=False), - ), - ) mock_qs = MagicMock() mock_qs.exists.return_value = True mock_qs.__iter__ = MagicMock(return_value=iter([])) diff --git a/src/paperless_ai/vector_store.py b/src/paperless_ai/vector_store.py index dbe84e86b..5a866488f 100644 --- a/src/paperless_ai/vector_store.py +++ b/src/paperless_ai/vector_store.py @@ -40,9 +40,8 @@ _INSERT_CHUNK_INDEX = ( "INSERT INTO document_chunks (chunk_id, document_id) VALUES (?, ?)" ) -# Current schema version. Written to index_meta at table creation and bumped -# whenever a Migration is added to MIGRATIONS. check_and_run_migrations() uses -# this to decide which migrations to run on an existing store. +# Current schema version. Bump when adding a migration -- see +# paperless_ai/migrations/__init__.py for the full procedure. SCHEMA_VERSION = 2 # compact(): rebuild when the cumulative rowid count exceeds this multiple of @@ -522,6 +521,67 @@ class PaperlessSqliteVecVectorStore(BasePydanticVectorStore): result[doc_id] = str(row["modified"] or "") return result + @property + def _db_path(self) -> str: + return str(Path(self._uri) / DB_FILENAME) + + @contextmanager + def _rebuild_file(self) -> Iterator[sqlite3.Connection]: + """Open a fresh temp database file for a file-swap rebuild (compact + or structural migration), yielding its connection for the caller to + populate. + + On success, swaps the temp file in as the live database (closing + this store's current connection first -- see _swap_in_compact()). + On any exception, discards the temp file, including its -wal/-shm, + instead, and this store's own connection is left untouched. + """ + compact_path = self._db_path + ".compact" + new_conn = self._open_connection(compact_path) + try: + yield new_conn + except BaseException: + new_conn.close() + for suffix in ["", "-wal", "-shm"]: + Path(compact_path + suffix).unlink(missing_ok=True) + raise + else: + new_conn.close() + self._swap_in_compact(compact_path, self._db_path) + + @staticmethod + def _rebuild_into( + src_conn: sqlite3.Connection, + dst_conn: sqlite3.Connection, + dim: int, + meta_keys: tuple[str, ...] = ("dim", "embed_model", "schema_version"), + ) -> int: + """Create the vec0 table in ``dst_conn``, copy ``meta_keys`` from + ``src_conn``'s index_meta, and stream every live row across + (populating document_chunks as it goes -- see _copy_rows()). + Returns the number of rows copied. + + Used by both compact() (default meta_keys: the schema is unchanged) + and structural migrations (meta_keys minus "schema_version", which + the migration sets to its own target version instead of preserving + the source's). + """ + PaperlessSqliteVecVectorStore._create_vec_table(dst_conn, dim) + for key in meta_keys: + value = PaperlessSqliteVecVectorStore._meta_get_on(src_conn, key) + if value is not None: + PaperlessSqliteVecVectorStore._meta_set_on(dst_conn, key, value) + dst_conn.execute("BEGIN IMMEDIATE") + copied = _copy_rows(src_conn, dst_conn) + # Reset the cumulative counter: after a rebuild, total_inserts == live. + PaperlessSqliteVecVectorStore._meta_set_on( + dst_conn, + "total_inserts", + str(copied), + ) + dst_conn.execute("COMMIT") + return copied + def compact(self, *, force: bool = False) -> None: """Rebuild the database file to reclaim space left behind by DELETEs. @@ -554,30 +614,8 @@ class PaperlessSqliteVecVectorStore(BasePydanticVectorStore): live, total, ) - db_path = str(Path(self._uri) / DB_FILENAME) - compact_path = db_path + ".compact" - - # Copy all live rows into a fresh database file. - new_conn = self._open_connection(compact_path) - try: - self._create_vec_table(new_conn, dim) - self._meta_set_on(new_conn, "dim", str(dim)) - for key in ("embed_model", "schema_version"): - value = self._meta_get(key) - if value is not None: - self._meta_set_on(new_conn, key, value) - new_conn.execute("BEGIN IMMEDIATE") - _copy_rows(self._conn, new_conn) - # Reset the cumulative counter: after compact, total_inserts == live. - self._meta_set_on(new_conn, "total_inserts", str(live)) - new_conn.execute("COMMIT") - except BaseException: - new_conn.close() - for p in [compact_path, compact_path + "-wal", compact_path + "-shm"]: - Path(p).unlink(missing_ok=True) - raise - new_conn.close() - self._swap_in_compact(compact_path, db_path) + with self._rebuild_file() as new_conn: + self._rebuild_into(self._conn, new_conn, dim) def _swap_in_compact(self, compact_path: str, db_path: str) -> None: """Atomically replace the live database with the compacted copy.""" @@ -589,6 +627,17 @@ class PaperlessSqliteVecVectorStore(BasePydanticVectorStore): Path(compact_path).replace(db_path) self._conn = self._open_connection(db_path) + def _stored_schema_version(self) -> int | None: + """The schema_version recorded in index_meta, or None if no table + exists. A missing key (a store predating version tracking) is + treated as SCHEMA_VERSION -- i.e. already current -- since no + migration in MIGRATIONS targets a version before tracking began. + """ + if not self.table_exists(): + return None + raw = self._meta_get("schema_version") + return int(raw) if raw is not None else SCHEMA_VERSION + def has_pending_migration(self) -> bool: """Cheaply check whether a migration is pending, with no exclusive access needed -- just a metadata read under the connection callers @@ -600,11 +649,8 @@ class PaperlessSqliteVecVectorStore(BasePydanticVectorStore): case -- already at SCHEMA_VERSION -- never contends with readers or a concurrent compaction. """ - if not self.table_exists(): - return False - raw = self._meta_get("schema_version") - current = int(raw) if raw is not None else SCHEMA_VERSION - return current < SCHEMA_VERSION + current = self._stored_schema_version() + return current is not None and current < SCHEMA_VERSION def check_and_run_migrations(self) -> bool: """Apply any pending schema migrations to the store. @@ -619,12 +665,8 @@ class PaperlessSqliteVecVectorStore(BasePydanticVectorStore): that exclusion in the common case). No-op when the table does not exist or is already at SCHEMA_VERSION. """ - if not self.table_exists(): - return False - - raw = self._meta_get("schema_version") - current = int(raw) if raw is not None else SCHEMA_VERSION - if current >= SCHEMA_VERSION: + current = self._stored_schema_version() + if current is None or current >= SCHEMA_VERSION: return False pending = sorted( @@ -658,24 +700,12 @@ class PaperlessSqliteVecVectorStore(BasePydanticVectorStore): dim = self.vector_dim() if dim is None: # pragma: no cover raise RuntimeError("Cannot migrate: no stored vector dimension") - db_path = str(Path(self._uri) / DB_FILENAME) - compact_path = db_path + ".compact" - new_conn = self._open_connection(compact_path) - try: + with self._rebuild_file() as new_conn: migration.apply(self._conn, new_conn, dim) self._meta_set_on(new_conn, "schema_version", str(migration.to_version)) - except BaseException: # pragma: no cover - new_conn.close() - for p in [compact_path, compact_path + "-wal", compact_path + "-shm"]: - Path(p).unlink(missing_ok=True) - raise - new_conn.close() - self._swap_in_compact(compact_path, db_path) -# Import migration modules for their registration side effect (each appends -# itself to MIGRATIONS on import). Must be at the bottom of this file: those -# modules import PaperlessSqliteVecVectorStore and _copy_rows from here, which -# must already be fully defined by the time they run. Add a new migration -# module's import here (and bump SCHEMA_VERSION) when the schema changes. +# Registers m0001 into MIGRATIONS; must be at the bottom (needs +# PaperlessSqliteVecVectorStore fully defined) -- see +# paperless_ai/migrations/__init__.py for the full procedure. from paperless_ai.migrations import m0001_add_document_chunks # noqa: E402, F401