Chore(beta): drop automatic LanceDB index cleanup on startup

Leave legacy Lance directory removal to the user rather than deleting it
automatically on first run. Beta policy: user is expected to do a clean
re-embed anyway; no need for the system to silently delete their data.

Remove _cleanup_legacy_lance_index(), the forced-rebuild path that called
it, and the associated tests.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

src/paperless_ai/indexing.py

@@ -1,5 +1,4 @@
 import logging
-import shutil
 from collections.abc import Iterable
 from contextlib import contextmanager
 from datetime import timedelta
@@ -74,21 +73,6 @@ def get_vector_store() -> "PaperlessSqliteVecVectorStore":
     )
 
 
-def _cleanup_legacy_lance_index() -> bool:
-    """Delete a LanceDB index left by a pre-sqlite-vec version, if present.
-
-    Beta transition policy: no cross-store conversion; the caller forces a
-    full rebuild (re-embed) instead. Returns True when leftovers were found.
-    """
-    legacy_table = settings.LLM_INDEX_DIR / f"{LLM_INDEX_TABLE}.lance"
-    found = legacy_table.exists()
-    if found:
-        shutil.rmtree(legacy_table, ignore_errors=True)
-    # faiss-era metadata file, removed on the same occasion
-    (settings.LLM_INDEX_DIR / "meta.json").unlink(missing_ok=True)
-    return found
-
-
 @contextmanager
 def write_store(embed_model_name: str | None = None):
     """Acquire the write lock and yield the vector store.
@@ -243,11 +227,6 @@ def update_llm_index(
     rebuild=False,
 ) -> str:
     """Rebuild or incrementally update the LLM index."""
-    if _cleanup_legacy_lance_index():
-        logger.warning(
-            "Found a LanceDB index from a previous version; forcing a full rebuild.",
-        )
-        rebuild = True
     documents = Document.objects.all()
     no_documents = not documents.exists()
 

src/paperless_ai/tests/test_ai_indexing.py

@@ -1,4 +1,3 @@
-import json
 from pathlib import Path
 from unittest.mock import MagicMock
 from unittest.mock import patch
@@ -164,27 +163,6 @@ def test_update_llm_index(
         build_document_node.assert_called_once_with(real_document, chunk_size=512)
 
 
-@pytest.mark.django_db
-def test_update_llm_index_cleans_stale_meta_on_rebuild(
-    temp_llm_index_dir: Path,
-    real_document: Document,
-    mock_embed_model: FakeEmbedding,
-) -> None:
-    # A meta.json left over from the FAISS era (or written by older code) must be
-    # deleted on rebuild so stale artifacts don't accumulate on disk.
-    stale_meta = temp_llm_index_dir / "meta.json"
-    stale_meta.write_text(json.dumps({"embedding_model": "old", "dim": 1}))
-
-    with patch("documents.models.Document.objects.all") as mock_all:
-        mock_queryset = MagicMock()
-        mock_queryset.exists.return_value = True
-        mock_queryset.__iter__.return_value = iter([real_document])
-        mock_all.return_value = mock_queryset
-        indexing.update_llm_index(rebuild=True)
-
-    assert not stale_meta.exists(), (
-        "update_llm_index(rebuild=True) must remove stale meta.json"
-    )
 
 
 @pytest.mark.django_db

src/paperless_ai/tests/test_legacy_lance_cleanup.py

@@ -1,100 +0,0 @@
-from pathlib import Path
-
-import pytest
-from django.utils import timezone
-
-from documents.models import Document
-from paperless_ai import indexing
-from paperless_ai.tests.conftest import FakeEmbedding
-
-
-@pytest.fixture
-def legacy_lance_dir(temp_llm_index_dir: Path) -> Path:
-    """Simulate leftovers of a pre-sqlite-vec LanceDB index."""
-    lance_table = temp_llm_index_dir / "documents.lance"
-    (lance_table / "data").mkdir(parents=True)
-    (lance_table / "data" / "0000.lance").write_bytes(b"not a real lance file")
-    (temp_llm_index_dir / "meta.json").write_text("{}")
-    return lance_table
-
-
-@pytest.mark.django_db
-class TestLegacyLanceCleanup:
-    def test_update_removes_legacy_dir_and_forces_rebuild(
-        self,
-        legacy_lance_dir: Path,
-        temp_llm_index_dir: Path,
-        mock_embed_model: FakeEmbedding,
-        caplog: pytest.LogCaptureFixture,
-    ) -> None:
-        """When a LanceDB directory is present, update_llm_index must delete it,
-        log a rebuild warning, and produce a valid (empty) sqlite-vec store."""
-        Document.objects.create(
-            title="Test Document",
-            content="Some content for legacy lance cleanup test.",
-            added=timezone.now(),
-        )
-
-        indexing.update_llm_index(rebuild=False)
-
-        assert not legacy_lance_dir.exists()
-        assert not (temp_llm_index_dir / "meta.json").exists()
-        assert "forcing a full rebuild" in caplog.text
-        store = indexing.get_vector_store()
-        assert store.table_exists()
-
-    def test_update_without_legacy_dir_does_not_force_rebuild(
-        self,
-        temp_llm_index_dir: Path,
-        mock_embed_model: FakeEmbedding,
-        caplog: pytest.LogCaptureFixture,
-    ) -> None:
-        """When no LanceDB leftovers exist, incremental update must not log a
-        forced-rebuild warning on a second call."""
-        Document.objects.create(
-            title="Test Document",
-            content="Some content without legacy lance.",
-            added=timezone.now(),
-        )
-
-        # First call builds the index cleanly (no legacy lance present).
-        indexing.update_llm_index(rebuild=True)
-
-        caplog.clear()
-
-        # Second incremental call must not mention a forced rebuild.
-        indexing.update_llm_index(rebuild=False)
-
-        assert "forcing a full rebuild" not in caplog.text
-
-    def test_cleanup_helper_reports_absence(self, temp_llm_index_dir: Path) -> None:
-        """_cleanup_legacy_lance_index must return False when no lance dir exists."""
-        assert indexing._cleanup_legacy_lance_index() is False  # noqa: SLF001
-
-    def test_cleanup_helper_reports_presence(
-        self,
-        legacy_lance_dir: Path,
-        temp_llm_index_dir: Path,
-    ) -> None:
-        """_cleanup_legacy_lance_index must return True and remove the directory."""
-        result = indexing._cleanup_legacy_lance_index()  # noqa: SLF001
-
-        assert result is True
-        assert not legacy_lance_dir.exists()
-        assert not (temp_llm_index_dir / "meta.json").exists()
-
-    def test_cleanup_helper_removes_only_lance_not_other_files(
-        self,
-        legacy_lance_dir: Path,
-        temp_llm_index_dir: Path,
-    ) -> None:
-        """_cleanup_legacy_lance_index must not touch files other than the lance dir
-        and meta.json."""
-        other_file = temp_llm_index_dir / "index.db"
-        other_file.write_bytes(b"sqlite data")
-
-        indexing._cleanup_legacy_lance_index()  # noqa: SLF001
-
-        assert other_file.exists(), (
-            "unrelated files in LLM_INDEX_DIR must be left intact after cleanup"
-        )