refactor(ai): build the index from the LanceDB store alone (lazy import)

Replace get_or_create_storage_context with get_vector_store() (lazy import
of paperless_ai.vector_store inside the function), rewrite load_or_build_index
to use VectorStoreIndex.from_vector_store, and rewrite vector_store_file_exists
to use store.table_exists(). Add LLM_INDEX_TABLE constant and TYPE_CHECKING-only
import of PaperlessLanceVectorStore. Delete remove_document_docstore_nodes and
rewire llm_index_add_or_update_document, llm_index_remove_document, and
update_llm_index to use upsert_document/delete/drop_table on the LanceDB store.
Serialize tags list as JSON string to satisfy flat_metadata validation. Add
test_get_vector_store_roundtrip, test_add_then_remove_document,
test_update_shrinks_chunks_without_orphans, and the subprocess lazy-import guard.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
stumpylogandClaude Opus 4.8 committed 2026-06-05 11:43:41 -07:00
1 parent 2f5d199fef
commit f41e32cfcd
3 files changed
+164 -172

No files matched your search

+95 -172
View File
@@ -1,6 +1,5 @@
import json
import logging import logging
import shutil
from collections import defaultdict
from collections.abc import Iterable from collections.abc import Iterable
from datetime import timedelta from datetime import timedelta
from pathlib import Path from pathlib import Path
@@ -16,16 +15,18 @@ from documents.utils import IterWrapper
from documents.utils import identity from documents.utils import identity
from paperless.config import AIConfig from paperless.config import AIConfig
from paperless_ai.embedding import build_llm_index_text from paperless_ai.embedding import build_llm_index_text
from paperless_ai.embedding import get_embedding_dim
from paperless_ai.embedding import get_embedding_model from paperless_ai.embedding import get_embedding_model
if TYPE_CHECKING: if TYPE_CHECKING:
from llama_index.core import VectorStoreIndex
from llama_index.core.schema import BaseNode from llama_index.core.schema import BaseNode
from paperless_ai.vector_store import PaperlessLanceVectorStore
logger = logging.getLogger("paperless_ai.indexing") logger = logging.getLogger("paperless_ai.indexing")
LLM_INDEX_TABLE = "documents"
RAG_NUM_OUTPUT = 512 RAG_NUM_OUTPUT = 512
RAG_CHUNK_OVERLAP = 200 RAG_CHUNK_OVERLAP = 200
@@ -71,43 +72,18 @@ def queue_llm_index_update_if_needed(*, rebuild: bool, reason: str) -> bool:
return True return True
def get_or_create_storage_context(*, rebuild=False): def get_vector_store() -> "PaperlessLanceVectorStore":
"""Open (or lazily create) the LanceDB-backed vector store.
Imports ``vector_store`` lazily so that importing ``indexing`` (which
``documents.tasks`` does at module top) never drags in lancedb/llama_index.
""" """
Loads or creates the StorageContext (vector store, docstore, index store). from paperless_ai.vector_store import PaperlessLanceVectorStore
If rebuild=True, deletes and recreates everything.
"""
if rebuild:
shutil.rmtree(settings.LLM_INDEX_DIR, ignore_errors=True)
settings.LLM_INDEX_DIR.mkdir(parents=True, exist_ok=True)
if rebuild or not settings.LLM_INDEX_DIR.exists(): settings.LLM_INDEX_DIR.mkdir(parents=True, exist_ok=True)
import faiss return PaperlessLanceVectorStore(
from llama_index.core import StorageContext uri=str(settings.LLM_INDEX_DIR),
from llama_index.core.storage.docstore import SimpleDocumentStore table_name=LLM_INDEX_TABLE,
from llama_index.core.storage.index_store import SimpleIndexStore
from llama_index.vector_stores.faiss import FaissVectorStore
settings.LLM_INDEX_DIR.mkdir(parents=True, exist_ok=True)
embedding_dim = get_embedding_dim()
faiss_index = faiss.IndexFlatL2(embedding_dim)
vector_store = FaissVectorStore(faiss_index=faiss_index)
docstore = SimpleDocumentStore()
index_store = SimpleIndexStore()
else:
from llama_index.core import StorageContext
from llama_index.core.storage.docstore import SimpleDocumentStore
from llama_index.core.storage.index_store import SimpleIndexStore
from llama_index.vector_stores.faiss import FaissVectorStore
vector_store = FaissVectorStore.from_persist_dir(settings.LLM_INDEX_DIR)
docstore = SimpleDocumentStore.from_persist_dir(settings.LLM_INDEX_DIR)
index_store = SimpleIndexStore.from_persist_dir(settings.LLM_INDEX_DIR)
return StorageContext.from_defaults(
docstore=docstore,
index_store=index_store,
vector_store=vector_store,
persist_dir=settings.LLM_INDEX_DIR,
) )
@@ -123,7 +99,7 @@ def build_document_node(
metadata = { metadata = {
"document_id": str(document.id), "document_id": str(document.id),
"title": document.title, "title": document.title,
"tags": [t.name for t in document.tags.all()], "tags": json.dumps([t.name for t in document.tags.all()]),
"correspondent": document.correspondent.name "correspondent": document.correspondent.name
if document.correspondent if document.correspondent
else None, else None,
@@ -156,65 +132,27 @@ def build_document_node(
def load_or_build_index(nodes=None): def load_or_build_index(nodes=None):
""" """Load the VectorStoreIndex backed by the LanceDB store.
Load an existing VectorStoreIndex if present,
or build a new one using provided nodes if storage is empty. With ``stores_text=True`` the index runs off the vector store alone — no
docstore or index store. ``nodes`` is accepted for signature compatibility
but unused; the store is the source of truth.
""" """
import llama_index.core.settings as llama_settings import llama_index.core.settings as llama_settings
from llama_index.core import VectorStoreIndex from llama_index.core import VectorStoreIndex
from llama_index.core import load_index_from_storage
embed_model = get_embedding_model() embed_model = get_embedding_model()
llama_settings.Settings.embed_model = embed_model llama_settings.Settings.embed_model = embed_model
storage_context = get_or_create_storage_context() vector_store = get_vector_store()
try: return VectorStoreIndex.from_vector_store(
return load_index_from_storage(storage_context=storage_context) vector_store=vector_store,
except ValueError as e: embed_model=embed_model,
logger.warning("Failed to load index from storage: %s", e) )
if not nodes:
queue_llm_index_update_if_needed(
rebuild=vector_store_file_exists(),
reason="LLM index missing or invalid while loading.",
)
logger.info("No nodes provided for index creation.")
raise
return VectorStoreIndex(
nodes=nodes,
storage_context=storage_context,
embed_model=embed_model,
)
def remove_document_docstore_nodes(document: Document, index: "VectorStoreIndex"): def vector_store_file_exists() -> bool:
""" """True when the LanceDB table exists."""
Removes existing documents from docstore for a given document from the index. return get_vector_store().table_exists()
This is necessary because FAISS IndexFlatL2 is append-only.
"""
all_node_ids = list(index.docstore.docs.keys())
existing_nodes = [
node.node_id
for node in index.docstore.get_nodes(all_node_ids)
if node.metadata.get("document_id") == str(document.id)
]
for node_id in existing_nodes:
# Delete from docstore, FAISS IndexFlatL2 are append-only
index.docstore.delete_document(node_id)
# Also purge the FAISS position -> UUID mapping so subsequent similarity
# queries don't raise KeyError on ghost vector positions.
stale_keys = [
k for k, v in index.index_struct.nodes_dict.items() if v == node_id
]
for key in stale_keys:
del index.index_struct.nodes_dict[key]
# Re-sync the mutated index_struct so persist() writes the updated nodes_dict.
index.storage_context.index_store.add_index_struct(index.index_struct)
def vector_store_file_exists():
"""
Check if the vector store file exists in the LLM index directory.
"""
return Path(settings.LLM_INDEX_DIR / "default__vector_store.json").exists()
def get_rag_chunk_size() -> int: def get_rag_chunk_size() -> int:
@@ -250,17 +188,28 @@ def get_rag_prompt_helper(
) )
def _iter_existing_modified(store: "PaperlessLanceVectorStore") -> list[dict]:
"""One representative row per document_id, for modified-time comparison."""
if LLM_INDEX_TABLE not in store.client.table_names():
return []
seen: dict[str, dict] = {}
for row in store.client.open_table(LLM_INDEX_TABLE).search().to_list():
seen.setdefault(str(row["document_id"]), row)
return list(seen.values())
def get_llm_index_compaction_retention() -> int:
"""Seconds of MVCC version history to keep during compaction."""
return 60 * 60 # 1 hour: safe for in-flight readers, reclaims daily
def update_llm_index( def update_llm_index(
*, *,
iter_wrapper: IterWrapper[Document] = identity, iter_wrapper: IterWrapper[Document] = identity,
rebuild=False, rebuild=False,
) -> str: ) -> str:
""" """Rebuild or incrementally update the LLM index."""
Rebuild or update the LLM index. from llama_index.core.schema import MetadataMode
"""
from llama_index.core import VectorStoreIndex
nodes = []
documents = Document.objects.all() documents = Document.objects.all()
if not documents.exists(): if not documents.exists():
@@ -268,105 +217,79 @@ def update_llm_index(
if not rebuild and not vector_store_file_exists(): if not rebuild and not vector_store_file_exists():
return "No documents found to index." return "No documents found to index."
config = AIConfig() chunk_size = AIConfig().llm_embedding_chunk_size
chunk_size = config.llm_embedding_chunk_size embed_model = get_embedding_model()
with FileLock(_index_lock_path()): with FileLock(_index_lock_path()):
if rebuild or not vector_store_file_exists(): if rebuild or not vector_store_file_exists():
# remove meta.json to force re-detection of embedding dim
(settings.LLM_INDEX_DIR / "meta.json").unlink(missing_ok=True) (settings.LLM_INDEX_DIR / "meta.json").unlink(missing_ok=True)
# Rebuild index from scratch
logger.info("Rebuilding LLM index.") logger.info("Rebuilding LLM index.")
import llama_index.core.settings as llama_settings store = get_vector_store()
store.drop_table()
embed_model = get_embedding_model()
llama_settings.Settings.embed_model = embed_model
storage_context = get_or_create_storage_context(rebuild=True)
for document in iter_wrapper(documents): for document in iter_wrapper(documents):
document_nodes = build_document_node(document, chunk_size=chunk_size) nodes = build_document_node(document, chunk_size=chunk_size)
nodes.extend(document_nodes) for node in nodes:
node.embedding = embed_model.get_text_embedding(
index = VectorStoreIndex( node.get_content(metadata_mode=MetadataMode.EMBED),
nodes=nodes, )
storage_context=storage_context, store.add(nodes)
embed_model=embed_model,
show_progress=False,
)
msg = "LLM index rebuilt successfully." msg = "LLM index rebuilt successfully."
else: else:
# Update existing index store = get_vector_store()
index = load_or_build_index() existing = {
existing_nodes: defaultdict[str, list] = defaultdict(list) str(row["document_id"]): json.loads(row["node_content"])
for node in index.docstore.docs.values(): for row in _iter_existing_modified(store)
doc_id = node.metadata.get("document_id") }
if doc_id is not None: changed = 0
existing_nodes[doc_id].append(node)
for document in iter_wrapper(documents): for document in iter_wrapper(documents):
doc_id = str(document.id) doc_id = str(document.id)
document_modified = document.modified.isoformat() node_meta = existing.get(doc_id)
if node_meta is not None:
if doc_id in existing_nodes: stored_modified = node_meta.get("modified")
doc_nodes = existing_nodes[doc_id] if stored_modified == document.modified.isoformat():
node_modified = doc_nodes[0].metadata.get("modified")
if node_modified == document_modified:
continue continue
nodes = build_document_node(document, chunk_size=chunk_size)
for node in nodes:
node.embedding = embed_model.get_text_embedding(
node.get_content(metadata_mode=MetadataMode.EMBED),
)
store.upsert_document(doc_id, nodes)
changed += 1
msg = (
"LLM index updated successfully."
if changed
else "No changes detected in LLM index."
)
# Delete from docstore, FAISS IndexFlatL2 are append-only store.ensure_document_id_scalar_index()
for node in doc_nodes: store.maybe_create_ann_index()
remove_document_docstore_nodes(document, index) store.compact(retention_seconds=get_llm_index_compaction_retention())
nodes.extend(build_document_node(document, chunk_size=chunk_size))
if nodes:
msg = "LLM index updated successfully."
logger.info(
"Updating %d nodes in LLM index.",
len(nodes),
)
index.insert_nodes(nodes)
else:
msg = "No changes detected in LLM index."
logger.info(msg)
index.storage_context.persist(persist_dir=settings.LLM_INDEX_DIR)
return msg return msg
def llm_index_add_or_update_document(document: Document): def llm_index_add_or_update_document(document: Document):
""" """Add or atomically replace a document's chunks in the LLM index."""
Adds or updates a document in the LLM index. from llama_index.core.schema import MetadataMode
If the document already exists, it will be replaced.
"""
new_nodes = build_document_node(document, chunk_size=get_rag_chunk_size()) new_nodes = build_document_node(document, chunk_size=get_rag_chunk_size())
if not new_nodes:
logger.warning( embed_model = get_embedding_model()
"No indexable content for document %s; skipping LLM index update.", for node in new_nodes:
document.pk, node.embedding = embed_model.get_text_embedding(
node.get_content(metadata_mode=MetadataMode.EMBED),
) )
return
with FileLock(_index_lock_path()): with FileLock(_index_lock_path()):
index = load_or_build_index(nodes=new_nodes) store = get_vector_store()
store.upsert_document(str(document.id), new_nodes)
remove_document_docstore_nodes(document, index) store.ensure_document_id_scalar_index()
index.insert_nodes(new_nodes)
index.storage_context.persist(persist_dir=settings.LLM_INDEX_DIR)
def llm_index_remove_document(document: Document): def llm_index_remove_document(document: Document):
""" """Remove a document's chunks from the LLM index."""
Removes a document from the LLM index.
"""
with FileLock(_index_lock_path()): with FileLock(_index_lock_path()):
index = load_or_build_index() store = get_vector_store()
store.delete(str(document.id))
remove_document_docstore_nodes(document, index)
index.storage_context.persist(persist_dir=settings.LLM_INDEX_DIR)
def truncate_content( def truncate_content(
@@ -908,3 +908,50 @@ class TestLlmIndexLocking:
assert call_order.index("lock_acquired") < call_order.index("index_loaded"), ( assert call_order.index("lock_acquired") < call_order.index("index_loaded"), (
"Lock must be acquired before the index is loaded" "Lock must be acquired before the index is loaded"
) )
@pytest.mark.django_db
def test_get_vector_store_roundtrip(
temp_llm_index_dir,
mock_embed_model,
) -> None:
from paperless_ai.vector_store import PaperlessLanceVectorStore
store = indexing.get_vector_store()
assert isinstance(store, PaperlessLanceVectorStore)
@pytest.mark.django_db
def test_add_then_remove_document(
temp_llm_index_dir,
mock_embed_model,
real_document,
) -> None:
indexing.llm_index_add_or_update_document(real_document)
store = indexing.get_vector_store()
table = store.client.open_table(indexing.LLM_INDEX_TABLE)
assert table.count_rows() >= 1
indexing.llm_index_remove_document(real_document)
assert store.client.open_table(indexing.LLM_INDEX_TABLE).count_rows() == 0
@pytest.mark.django_db
def test_update_shrinks_chunks_without_orphans(
temp_llm_index_dir,
mock_embed_model,
real_document,
) -> None:
real_document.content = "word " * 4000 # many chunks
real_document.save()
indexing.llm_index_add_or_update_document(real_document)
store = indexing.get_vector_store()
big = store.client.open_table(indexing.LLM_INDEX_TABLE).count_rows()
real_document.content = "short" # one chunk
real_document.save()
indexing.llm_index_add_or_update_document(real_document)
rows = store.client.open_table(indexing.LLM_INDEX_TABLE).count_rows()
assert rows < big
assert rows >= 1
@@ -0,0 +1,22 @@
import subprocess
import sys
class TestLazyAiImports:
def test_importing_tasks_does_not_load_ai_libraries(self) -> None:
code = (
"import os, django, sys\n"
"os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'paperless.settings')\n"
"django.setup()\n"
"import documents.tasks # noqa: F401\n"
"leaked = [m for m in ('lancedb', 'pyarrow', 'llama_index') "
"if m in sys.modules]\n"
"assert not leaked, f'AI libraries leaked into the light path: {leaked}'\n"
)
result = subprocess.run(
[sys.executable, "-c", code],
capture_output=True,
text=True,
cwd="src",
)
assert result.returncode == 0, result.stdout + result.stderr