mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-08-11 05:13:18 +00:00
2c58d86380
* Fix: Remove all nodes for multi-chunk documents in update_llm_index incremental path The existing_nodes dict comprehension keyed on document_id silently dropped all but the last node per document, so only that one node was deleted when a modified document was re-indexed, leaving all other chunks as ghost vectors in the FAISS index. Switch to a defaultdict(list) that collects every node per document_id, then iterate and delete all of them before inserting fresh nodes. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> * Fix: Wire document_updated signal to LLM index update handler Connect document_updated to add_or_update_document_in_llm_index in DocumentsConfig.ready() so REST API edits (PATCH /api/documents/{id}/) enqueue an LLM vector store update, matching the existing document_consumption_finished behavior. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> * Fix: Add file lock around FAISS index mutations to prevent concurrent write corruption Two concurrent Celery workers calling llm_index_add_or_update_document or llm_index_remove_document each loaded the same on-disk index independently, made their own change, and the last writer silently overwrote the first's update. Wrap both functions and the rebuild/persist body of update_llm_index in a filelock.FileLock keyed on LLM_INDEX_DIR/index.lock. Add a TOCTOU comment on queue_llm_index_update_if_needed explaining the residual risk (duplicate rebuild tasks are wasteful but not corrupting because the lock serialises the actual write). Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> * Fix: Apply _normalize() in extract_unmatched_names to prevent duplicate suggestions extract_unmatched_names was using .lower() while _match_names_to_queryset uses _normalize() (which also strips punctuation). A name like "J. Smith" matched to existing correspondent "J Smith" would still appear in the unmatched list, causing duplicate object creation. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> * Fix: Skip LLM index update gracefully when document has no indexable content Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> * Fix: Persist empty index when all documents are deleted to clear stale FAISS vectors The early-return guard in update_llm_index fired before persist() when no documents existed, leaving a stale on-disk FAISS index that returned phantom hits for deleted document IDs. Now the guard only returns early for the incremental (rebuild=False) path when no index exists on disk; the rebuild path always continues through to persist(), producing an empty clean index. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> * Chore: Simplify incremental index update — use docs.values() and deduplicate node extend --------- Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
102 lines
4.6 KiB
Python
102 lines
4.6 KiB
Python
from unittest.mock import patch
|
|
|
|
import pytest
|
|
from django.test import TestCase
|
|
|
|
from documents.models import Correspondent
|
|
from documents.models import DocumentType
|
|
from documents.models import StoragePath
|
|
from documents.models import Tag
|
|
from paperless_ai.matching import extract_unmatched_names
|
|
from paperless_ai.matching import match_correspondents_by_name
|
|
from paperless_ai.matching import match_document_types_by_name
|
|
from paperless_ai.matching import match_storage_paths_by_name
|
|
from paperless_ai.matching import match_tags_by_name
|
|
|
|
|
|
class TestAIMatching(TestCase):
|
|
def setUp(self) -> None:
|
|
# Create test data for Tag
|
|
self.tag1 = Tag.objects.create(name="Test Tag 1")
|
|
self.tag2 = Tag.objects.create(name="Test Tag 2")
|
|
|
|
# Create test data for Correspondent
|
|
self.correspondent1 = Correspondent.objects.create(name="Test Correspondent 1")
|
|
self.correspondent2 = Correspondent.objects.create(name="Test Correspondent 2")
|
|
|
|
# Create test data for DocumentType
|
|
self.document_type1 = DocumentType.objects.create(name="Test Document Type 1")
|
|
self.document_type2 = DocumentType.objects.create(name="Test Document Type 2")
|
|
|
|
# Create test data for StoragePath
|
|
self.storage_path1 = StoragePath.objects.create(name="Test Storage Path 1")
|
|
self.storage_path2 = StoragePath.objects.create(name="Test Storage Path 2")
|
|
|
|
@patch("paperless_ai.matching.get_objects_for_user_owner_aware")
|
|
def test_match_tags_by_name(self, mock_get_objects) -> None:
|
|
mock_get_objects.return_value = Tag.objects.all()
|
|
names = ["Test Tag 1", "Nonexistent Tag"]
|
|
result = match_tags_by_name(names, user=None)
|
|
self.assertEqual(len(result), 1)
|
|
self.assertEqual(result[0].name, "Test Tag 1")
|
|
|
|
@patch("paperless_ai.matching.get_objects_for_user_owner_aware")
|
|
def test_match_correspondents_by_name(self, mock_get_objects) -> None:
|
|
mock_get_objects.return_value = Correspondent.objects.all()
|
|
names = ["Test Correspondent 1", "Nonexistent Correspondent"]
|
|
result = match_correspondents_by_name(names, user=None)
|
|
self.assertEqual(len(result), 1)
|
|
self.assertEqual(result[0].name, "Test Correspondent 1")
|
|
|
|
@patch("paperless_ai.matching.get_objects_for_user_owner_aware")
|
|
def test_match_document_types_by_name(self, mock_get_objects) -> None:
|
|
mock_get_objects.return_value = DocumentType.objects.all()
|
|
names = ["Test Document Type 1", "Nonexistent Document Type"]
|
|
result = match_document_types_by_name(names, user=None)
|
|
self.assertEqual(len(result), 1)
|
|
self.assertEqual(result[0].name, "Test Document Type 1")
|
|
|
|
@patch("paperless_ai.matching.get_objects_for_user_owner_aware")
|
|
def test_match_storage_paths_by_name(self, mock_get_objects) -> None:
|
|
mock_get_objects.return_value = StoragePath.objects.all()
|
|
names = ["Test Storage Path 1", "Nonexistent Storage Path"]
|
|
result = match_storage_paths_by_name(names, user=None)
|
|
self.assertEqual(len(result), 1)
|
|
self.assertEqual(result[0].name, "Test Storage Path 1")
|
|
|
|
def test_extract_unmatched_names(self) -> None:
|
|
llm_names = ["Test Tag 1", "Nonexistent Tag"]
|
|
matched_objects = [self.tag1]
|
|
unmatched_names = extract_unmatched_names(llm_names, matched_objects)
|
|
self.assertEqual(unmatched_names, ["Nonexistent Tag"])
|
|
|
|
@patch("paperless_ai.matching.get_objects_for_user_owner_aware")
|
|
def test_match_tags_by_name_with_empty_names(self, mock_get_objects) -> None:
|
|
mock_get_objects.return_value = Tag.objects.all()
|
|
names = [None, "", " "]
|
|
result = match_tags_by_name(names, user=None)
|
|
self.assertEqual(result, [])
|
|
|
|
@patch("paperless_ai.matching.get_objects_for_user_owner_aware")
|
|
def test_match_tags_with_fuzzy_matching(self, mock_get_objects) -> None:
|
|
mock_get_objects.return_value = Tag.objects.all()
|
|
names = ["Test Taag 1", "Teest Tag 2"]
|
|
result = match_tags_by_name(names, user=None)
|
|
self.assertEqual(len(result), 2)
|
|
self.assertEqual(result[0].name, "Test Tag 1")
|
|
self.assertEqual(result[1].name, "Test Tag 2")
|
|
|
|
|
|
@pytest.mark.django_db
|
|
class TestExtractUnmatchedNamesNormalization:
|
|
def test_punctuated_name_already_matched_is_not_returned_as_unmatched(
|
|
self,
|
|
) -> None:
|
|
correspondent = Correspondent.objects.create(name="J Smith")
|
|
llm_names = ["J. Smith"]
|
|
matched_objects: list[Correspondent] = [correspondent]
|
|
|
|
unmatched = extract_unmatched_names(llm_names, matched_objects)
|
|
|
|
assert "J. Smith" not in unmatched
|