mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-11-21 13:02:40 +00:00
Enhancement (beta): Switch the AI vector store to LanceDB (#12944)
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: shamoon <shamoon@users.noreply.github.com>
This commit is contained in:
19 files changed
+1606
-1060
No files matched your search
@@ -1,4 +1,3 @@
|
||||
import json
|
||||
from unittest.mock import ANY
|
||||
from unittest.mock import MagicMock
|
||||
from unittest.mock import patch
|
||||
@@ -10,7 +9,7 @@ from documents.models import Document
|
||||
from paperless.models import LLMEmbeddingBackend
|
||||
from paperless_ai.embedding import _normalize_llm_index_text
|
||||
from paperless_ai.embedding import build_llm_index_text
|
||||
from paperless_ai.embedding import get_embedding_dim
|
||||
from paperless_ai.embedding import get_configured_model_name
|
||||
from paperless_ai.embedding import get_embedding_model
|
||||
|
||||
|
||||
@@ -67,7 +66,7 @@ def test_get_embedding_model_openai(mock_ai_config):
|
||||
with patch(
|
||||
"llama_index.embeddings.openai_like.OpenAILikeEmbedding",
|
||||
) as MockOpenAIEmbedding:
|
||||
model = get_embedding_model()
|
||||
model = get_embedding_model(mock_ai_config.return_value)
|
||||
MockOpenAIEmbedding.assert_called_once_with(
|
||||
model_name="text-embedding-3-small",
|
||||
api_key="test_api_key",
|
||||
@@ -88,7 +87,7 @@ def test_get_embedding_model_openai_prefers_embedding_endpoint(mock_ai_config):
|
||||
with patch(
|
||||
"llama_index.embeddings.openai_like.OpenAILikeEmbedding",
|
||||
) as MockOpenAIEmbedding:
|
||||
model = get_embedding_model()
|
||||
model = get_embedding_model(mock_ai_config.return_value)
|
||||
MockOpenAIEmbedding.assert_called_once_with(
|
||||
model_name="text-embedding-3-small",
|
||||
api_key="test_api_key",
|
||||
@@ -109,7 +108,7 @@ def test_get_embedding_model_openai_blocks_internal_endpoint_when_disallowed(
|
||||
mock_ai_config.return_value.llm_allow_internal_endpoints = False
|
||||
|
||||
with pytest.raises(ValueError, match="non-public address"):
|
||||
get_embedding_model()
|
||||
get_embedding_model(mock_ai_config.return_value)
|
||||
|
||||
|
||||
def test_get_embedding_model_huggingface(mock_ai_config):
|
||||
@@ -121,7 +120,7 @@ def test_get_embedding_model_huggingface(mock_ai_config):
|
||||
with patch(
|
||||
"llama_index.embeddings.huggingface.HuggingFaceEmbedding",
|
||||
) as MockHuggingFaceEmbedding:
|
||||
model = get_embedding_model()
|
||||
model = get_embedding_model(mock_ai_config.return_value)
|
||||
MockHuggingFaceEmbedding.assert_called_once_with(
|
||||
model_name="sentence-transformers/all-MiniLM-L6-v2",
|
||||
cache_folder=str(settings.DATA_DIR / "hf_cache"),
|
||||
@@ -137,7 +136,7 @@ def test_get_embedding_model_ollama(mock_ai_config):
|
||||
with patch(
|
||||
"llama_index.embeddings.ollama.OllamaEmbedding",
|
||||
) as MockOllamaEmbedding:
|
||||
model = get_embedding_model()
|
||||
model = get_embedding_model(mock_ai_config.return_value)
|
||||
MockOllamaEmbedding.assert_called_once_with(
|
||||
model_name="embeddinggemma",
|
||||
base_url="http://test-url",
|
||||
@@ -155,7 +154,7 @@ def test_get_embedding_model_ollama_prefers_embedding_endpoint(mock_ai_config):
|
||||
with patch(
|
||||
"llama_index.embeddings.ollama.OllamaEmbedding",
|
||||
) as MockOllamaEmbedding:
|
||||
model = get_embedding_model()
|
||||
model = get_embedding_model(mock_ai_config.return_value)
|
||||
MockOllamaEmbedding.assert_called_once_with(
|
||||
model_name="embeddinggemma",
|
||||
base_url="http://embedding-url",
|
||||
@@ -173,7 +172,7 @@ def test_get_embedding_model_ollama_blocks_internal_endpoint_when_disallowed(
|
||||
mock_ai_config.return_value.llm_allow_internal_endpoints = False
|
||||
|
||||
with pytest.raises(ValueError, match="non-public address"):
|
||||
get_embedding_model()
|
||||
get_embedding_model(mock_ai_config.return_value)
|
||||
|
||||
|
||||
def test_get_embedding_model_invalid_backend(mock_ai_config):
|
||||
@@ -183,55 +182,37 @@ def test_get_embedding_model_invalid_backend(mock_ai_config):
|
||||
ValueError,
|
||||
match="Unsupported embedding backend: INVALID_BACKEND",
|
||||
):
|
||||
get_embedding_model()
|
||||
get_embedding_model(mock_ai_config.return_value)
|
||||
|
||||
|
||||
def test_get_embedding_dim_infers_and_saves(temp_llm_index_dir, mock_ai_config):
|
||||
mock_ai_config.return_value.llm_embedding_backend = "openai-like"
|
||||
mock_ai_config.return_value.llm_embedding_model = None
|
||||
|
||||
class DummyEmbedding:
|
||||
def get_text_embedding(self, text):
|
||||
return [0.0] * 7
|
||||
|
||||
with patch(
|
||||
"paperless_ai.embedding.get_embedding_model",
|
||||
return_value=DummyEmbedding(),
|
||||
) as mock_get:
|
||||
dim = get_embedding_dim()
|
||||
mock_get.assert_called_once()
|
||||
|
||||
assert dim == 7
|
||||
meta = json.loads((temp_llm_index_dir / "meta.json").read_text())
|
||||
assert meta == {"embedding_model": "text-embedding-3-small", "dim": 7}
|
||||
@pytest.mark.parametrize(
|
||||
("backend", "expected_default"),
|
||||
[
|
||||
(LLMEmbeddingBackend.OPENAI_LIKE, "text-embedding-3-small"),
|
||||
(LLMEmbeddingBackend.HUGGINGFACE, "sentence-transformers/all-MiniLM-L6-v2"),
|
||||
(LLMEmbeddingBackend.OLLAMA, "embeddinggemma"),
|
||||
],
|
||||
)
|
||||
def test_get_configured_model_name_falls_back_to_backend_default(
|
||||
mock_ai_config,
|
||||
backend,
|
||||
expected_default,
|
||||
):
|
||||
"""When no model is explicitly configured, each backend has a distinct default."""
|
||||
config = mock_ai_config.return_value
|
||||
config.llm_embedding_backend = backend
|
||||
config.llm_embedding_model = None
|
||||
assert get_configured_model_name(config) == expected_default
|
||||
|
||||
|
||||
def test_get_embedding_dim_reads_existing_meta(temp_llm_index_dir, mock_ai_config):
|
||||
mock_ai_config.return_value.llm_embedding_backend = "openai-like"
|
||||
mock_ai_config.return_value.llm_embedding_model = None
|
||||
|
||||
(temp_llm_index_dir / "meta.json").write_text(
|
||||
json.dumps({"embedding_model": "text-embedding-3-small", "dim": 11}),
|
||||
)
|
||||
|
||||
with patch("paperless_ai.embedding.get_embedding_model") as mock_get:
|
||||
assert get_embedding_dim() == 11
|
||||
mock_get.assert_not_called()
|
||||
|
||||
|
||||
def test_get_embedding_dim_raises_on_model_change(temp_llm_index_dir, mock_ai_config):
|
||||
mock_ai_config.return_value.llm_embedding_backend = "openai-like"
|
||||
mock_ai_config.return_value.llm_embedding_model = None
|
||||
|
||||
(temp_llm_index_dir / "meta.json").write_text(
|
||||
json.dumps({"embedding_model": "old", "dim": 11}),
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
RuntimeError,
|
||||
match="Embedding model changed from old to text-embedding-3-small",
|
||||
):
|
||||
get_embedding_dim()
|
||||
def test_get_configured_model_name_explicit_overrides_default(mock_ai_config):
|
||||
"""An explicit model name overrides the backend default for all backends."""
|
||||
config = mock_ai_config.return_value
|
||||
config.llm_embedding_backend = LLMEmbeddingBackend.OPENAI_LIKE
|
||||
config.llm_embedding_model = "my-custom-model"
|
||||
# The backend default for OPENAI_LIKE is "text-embedding-3-small", so if
|
||||
# the explicit name was ignored we'd get the wrong result.
|
||||
assert get_configured_model_name(config) == "my-custom-model"
|
||||
|
||||
|
||||
def test_build_llm_index_text(mock_document):
|
||||
@@ -243,12 +224,15 @@ def test_build_llm_index_text(mock_document):
|
||||
|
||||
result = build_llm_index_text(mock_document)
|
||||
|
||||
assert "Title: Test Title" in result
|
||||
# Structured fields live in node.metadata for LLM context — not body text
|
||||
assert "Title: Test Title" not in result
|
||||
assert "Created: 2023-01-01" not in result
|
||||
assert "Tags: Tag1, Tag2" not in result
|
||||
assert "Document Type: Invoice" not in result
|
||||
assert "Correspondent: Test Correspondent" not in result
|
||||
|
||||
# Fields without a metadata equivalent stay in body text
|
||||
assert "Filename: test_file.pdf" in result
|
||||
assert "Created: 2023-01-01" in result
|
||||
assert "Tags: Tag1, Tag2" in result
|
||||
assert "Document Type: Invoice" in result
|
||||
assert "Correspondent: Test Correspondent" in result
|
||||
assert "Notes: Note1,Note2" in result
|
||||
assert "Content:\n\nThis is the document content." in result
|
||||
assert "Custom Field - Field1: Value1\nCustom Field - Field2: Value2" in result
|
||||
|
||||
Reference in new issue
Block a user