Enhancement (beta): Switch the AI vector store to LanceDB (#12944)

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: shamoon <shamoon@users.noreply.github.com>

src/paperless_ai/embedding.py

@@ -1,12 +1,9 @@
-import json
 import re
 from typing import TYPE_CHECKING
 
 from django.conf import settings
 
 if TYPE_CHECKING:
-    from pathlib import Path
-
     from llama_index.core.base.embeddings.base import BaseEmbedding
 
 from documents.models import Document
@@ -23,9 +20,7 @@ OCR_LEADER_REGEX = re.compile(r"[._\-\u00b7]{4,}")
 HORIZONTAL_WHITESPACE_REGEX = re.compile(r"[ \t\u00a0]+")
 
 
-def get_embedding_model() -> "BaseEmbedding":
-    config = AIConfig()
-
+def get_embedding_model(config: AIConfig) -> "BaseEmbedding":
     match config.llm_embedding_backend:
         case LLMEmbeddingBackend.OPENAI_LIKE:
             from llama_index.embeddings.openai_like import OpenAILikeEmbedding
@@ -95,41 +90,20 @@ def get_embedding_model() -> "BaseEmbedding":
             )
 
 
-def get_embedding_dim() -> int:
-    """
-    Loads embedding dimension from meta.json if available, otherwise infers it
-    from a dummy embedding and stores it for future use.
-    """
-    config = AIConfig()
-    default_model = {
-        LLMEmbeddingBackend.OPENAI_LIKE: "text-embedding-3-small",
-        LLMEmbeddingBackend.HUGGINGFACE: "sentence-transformers/all-MiniLM-L6-v2",
-        LLMEmbeddingBackend.OLLAMA: "embeddinggemma",
-    }.get(
+_DEFAULT_MODEL_NAMES = {
+    LLMEmbeddingBackend.OPENAI_LIKE: "text-embedding-3-small",
+    LLMEmbeddingBackend.HUGGINGFACE: "sentence-transformers/all-MiniLM-L6-v2",
+    LLMEmbeddingBackend.OLLAMA: "embeddinggemma",
+}
+
+
+def get_configured_model_name(config: AIConfig) -> str:
+    """Return the canonical name of the currently configured embedding model."""
+    default = _DEFAULT_MODEL_NAMES.get(
         config.llm_embedding_backend,
         "sentence-transformers/all-MiniLM-L6-v2",
     )
-    model = config.llm_embedding_model or default_model
-
-    meta_path: Path = settings.LLM_INDEX_DIR / "meta.json"
-    if meta_path.exists():
-        with meta_path.open() as f:
-            meta = json.load(f)
-        if meta.get("embedding_model") != model:
-            raise RuntimeError(
-                f"Embedding model changed from {meta.get('embedding_model')} to {model}. "
-                "You must rebuild the index.",
-            )
-        return meta["dim"]
-
-    embedding_model = get_embedding_model()
-    test_embed = embedding_model.get_text_embedding("test")
-    dim = len(test_embed)
-
-    with meta_path.open("w") as f:
-        json.dump({"embedding_model": model, "dim": dim}, f)
-
-    return dim
+    return config.llm_embedding_model or default
 
 
 def _normalize_llm_index_text(text: str) -> str:
@@ -138,15 +112,13 @@ def _normalize_llm_index_text(text: str) -> str:
 
 
 def build_llm_index_text(doc: Document) -> str:
+    # TODO: Filename, Storage Path, and Archive Serial Number are short structured
+    # values that could move to node.metadata (excluded from embeddings, visible to
+    # LLM via metadata prepend) — same pattern as title/tags/correspondent. Notes
+    # and Custom Fields should stay here: Notes can be long free text, Custom Fields
+    # are dynamic in count and best kept in the embedding.
     lines = [
-        f"Title: {doc.title}",
         f"Filename: {doc.filename}",
-        f"Created: {doc.created}",
-        f"Added: {doc.added}",
-        f"Modified: {doc.modified}",
-        f"Tags: {', '.join(tag.name for tag in doc.tags.all())}",
-        f"Document Type: {doc.document_type.name if doc.document_type else ''}",
-        f"Correspondent: {doc.correspondent.name if doc.correspondent else ''}",
         f"Storage Path: {doc.storage_path.name if doc.storage_path else ''}",
         f"Archive Serial Number: {doc.archive_serial_number or ''}",
         f"Notes: {','.join([str(c.note) for c in Note.objects.filter(document=doc)])}",