diff --git a/src/paperless_ai/indexing.py b/src/paperless_ai/indexing.py index bd29273e0..ef7379462 100644 --- a/src/paperless_ai/indexing.py +++ b/src/paperless_ai/indexing.py @@ -1,5 +1,4 @@ import logging -import shutil from collections.abc import Iterable from contextlib import contextmanager from datetime import timedelta @@ -74,21 +73,6 @@ def get_vector_store() -> "PaperlessSqliteVecVectorStore": ) -def _cleanup_legacy_lance_index() -> bool: - """Delete a LanceDB index left by a pre-sqlite-vec version, if present. - - Beta transition policy: no cross-store conversion; the caller forces a - full rebuild (re-embed) instead. Returns True when leftovers were found. - """ - legacy_table = settings.LLM_INDEX_DIR / f"{LLM_INDEX_TABLE}.lance" - found = legacy_table.exists() - if found: - shutil.rmtree(legacy_table, ignore_errors=True) - # faiss-era metadata file, removed on the same occasion - (settings.LLM_INDEX_DIR / "meta.json").unlink(missing_ok=True) - return found - - @contextmanager def write_store(embed_model_name: str | None = None): """Acquire the write lock and yield the vector store. @@ -243,11 +227,6 @@ def update_llm_index( rebuild=False, ) -> str: """Rebuild or incrementally update the LLM index.""" - if _cleanup_legacy_lance_index(): - logger.warning( - "Found a LanceDB index from a previous version; forcing a full rebuild.", - ) - rebuild = True documents = Document.objects.all() no_documents = not documents.exists() diff --git a/src/paperless_ai/tests/test_ai_indexing.py b/src/paperless_ai/tests/test_ai_indexing.py index d3ff256d1..2628596d4 100644 --- a/src/paperless_ai/tests/test_ai_indexing.py +++ b/src/paperless_ai/tests/test_ai_indexing.py @@ -1,4 +1,3 @@ -import json from pathlib import Path from unittest.mock import MagicMock from unittest.mock import patch @@ -164,27 +163,6 @@ def test_update_llm_index( build_document_node.assert_called_once_with(real_document, chunk_size=512) -@pytest.mark.django_db -def test_update_llm_index_cleans_stale_meta_on_rebuild( - temp_llm_index_dir: Path, - real_document: Document, - mock_embed_model: FakeEmbedding, -) -> None: - # A meta.json left over from the FAISS era (or written by older code) must be - # deleted on rebuild so stale artifacts don't accumulate on disk. - stale_meta = temp_llm_index_dir / "meta.json" - stale_meta.write_text(json.dumps({"embedding_model": "old", "dim": 1})) - - with patch("documents.models.Document.objects.all") as mock_all: - mock_queryset = MagicMock() - mock_queryset.exists.return_value = True - mock_queryset.__iter__.return_value = iter([real_document]) - mock_all.return_value = mock_queryset - indexing.update_llm_index(rebuild=True) - - assert not stale_meta.exists(), ( - "update_llm_index(rebuild=True) must remove stale meta.json" - ) @pytest.mark.django_db diff --git a/src/paperless_ai/tests/test_legacy_lance_cleanup.py b/src/paperless_ai/tests/test_legacy_lance_cleanup.py deleted file mode 100644 index 7f0c64ef0..000000000 --- a/src/paperless_ai/tests/test_legacy_lance_cleanup.py +++ /dev/null @@ -1,100 +0,0 @@ -from pathlib import Path - -import pytest -from django.utils import timezone - -from documents.models import Document -from paperless_ai import indexing -from paperless_ai.tests.conftest import FakeEmbedding - - -@pytest.fixture -def legacy_lance_dir(temp_llm_index_dir: Path) -> Path: - """Simulate leftovers of a pre-sqlite-vec LanceDB index.""" - lance_table = temp_llm_index_dir / "documents.lance" - (lance_table / "data").mkdir(parents=True) - (lance_table / "data" / "0000.lance").write_bytes(b"not a real lance file") - (temp_llm_index_dir / "meta.json").write_text("{}") - return lance_table - - -@pytest.mark.django_db -class TestLegacyLanceCleanup: - def test_update_removes_legacy_dir_and_forces_rebuild( - self, - legacy_lance_dir: Path, - temp_llm_index_dir: Path, - mock_embed_model: FakeEmbedding, - caplog: pytest.LogCaptureFixture, - ) -> None: - """When a LanceDB directory is present, update_llm_index must delete it, - log a rebuild warning, and produce a valid (empty) sqlite-vec store.""" - Document.objects.create( - title="Test Document", - content="Some content for legacy lance cleanup test.", - added=timezone.now(), - ) - - indexing.update_llm_index(rebuild=False) - - assert not legacy_lance_dir.exists() - assert not (temp_llm_index_dir / "meta.json").exists() - assert "forcing a full rebuild" in caplog.text - store = indexing.get_vector_store() - assert store.table_exists() - - def test_update_without_legacy_dir_does_not_force_rebuild( - self, - temp_llm_index_dir: Path, - mock_embed_model: FakeEmbedding, - caplog: pytest.LogCaptureFixture, - ) -> None: - """When no LanceDB leftovers exist, incremental update must not log a - forced-rebuild warning on a second call.""" - Document.objects.create( - title="Test Document", - content="Some content without legacy lance.", - added=timezone.now(), - ) - - # First call builds the index cleanly (no legacy lance present). - indexing.update_llm_index(rebuild=True) - - caplog.clear() - - # Second incremental call must not mention a forced rebuild. - indexing.update_llm_index(rebuild=False) - - assert "forcing a full rebuild" not in caplog.text - - def test_cleanup_helper_reports_absence(self, temp_llm_index_dir: Path) -> None: - """_cleanup_legacy_lance_index must return False when no lance dir exists.""" - assert indexing._cleanup_legacy_lance_index() is False # noqa: SLF001 - - def test_cleanup_helper_reports_presence( - self, - legacy_lance_dir: Path, - temp_llm_index_dir: Path, - ) -> None: - """_cleanup_legacy_lance_index must return True and remove the directory.""" - result = indexing._cleanup_legacy_lance_index() # noqa: SLF001 - - assert result is True - assert not legacy_lance_dir.exists() - assert not (temp_llm_index_dir / "meta.json").exists() - - def test_cleanup_helper_removes_only_lance_not_other_files( - self, - legacy_lance_dir: Path, - temp_llm_index_dir: Path, - ) -> None: - """_cleanup_legacy_lance_index must not touch files other than the lance dir - and meta.json.""" - other_file = temp_llm_index_dir / "index.db" - other_file.write_bytes(b"sqlite data") - - indexing._cleanup_legacy_lance_index() # noqa: SLF001 - - assert other_file.exists(), ( - "unrelated files in LLM_INDEX_DIR must be left intact after cleanup" - )