mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-08-01 08:32:18 +00:00
Fix: don't gate document_ids-scoped LLM index updates on Document.modified
bulk_edit.py's tag/correspondent/document_type/storage_path/custom_field helpers write via queryset.update() or direct M2M/through-model bulk operations, none of which call Document.save() -- so Document.modified's auto_now never fires. Comparing against get_modified_times() for a document_ids-scoped update_llm_index() call was silently skipping reindex of documents whose embedded metadata had just changed via a bulk edit. The modified-time comparison is now only used for the unscoped, full-library incremental scan, where it is still needed. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
df8a6cc715
commit
cd7038a7a4
@@ -419,12 +419,24 @@ def update_llm_index(
|
||||
if document_ids is not None
|
||||
else documents
|
||||
)
|
||||
existing = store.get_modified_times()
|
||||
# When document_ids is given, the caller already knows exactly
|
||||
# which documents to reindex (e.g. a bulk edit) -- trust it and
|
||||
# skip the modified-time comparison entirely. Bulk edits (tags,
|
||||
# correspondent, document type, storage path, custom fields)
|
||||
# write via queryset.update()/M2M bulk operations, which bypass
|
||||
# Document.modified's auto_now, so comparing against
|
||||
# get_modified_times() here would silently skip reindexing
|
||||
# documents whose embedded metadata just changed. The comparison
|
||||
# is only meaningful for the unscoped, full-library scan, where
|
||||
# it avoids re-embedding documents that have not changed.
|
||||
existing = store.get_modified_times() if document_ids is None else None
|
||||
changed = 0
|
||||
for document in iter_wrapper(scoped_documents):
|
||||
doc_id = str(document.id)
|
||||
if existing.get(doc_id) == document.modified.isoformat():
|
||||
continue
|
||||
if existing is not None:
|
||||
stored_modified = existing.get(doc_id)
|
||||
if stored_modified == document.modified.isoformat():
|
||||
continue
|
||||
nodes = build_document_node(document, chunk_size=chunk_size)
|
||||
_embed_nodes(nodes, embed_model)
|
||||
store.upsert_document(doc_id, nodes)
|
||||
|
||||
@@ -21,6 +21,7 @@ from documents.signals import document_consumption_finished
|
||||
from documents.signals import document_updated
|
||||
from documents.tests.factories import DocumentFactory
|
||||
from documents.tests.factories import PaperlessTaskFactory
|
||||
from documents.tests.factories import TagFactory
|
||||
from paperless.models import ApplicationConfiguration
|
||||
from paperless_ai import indexing
|
||||
from paperless_ai.tests.conftest import FakeEmbedding
|
||||
@@ -347,6 +348,62 @@ def test_update_llm_index_partial_update(
|
||||
assert after[str(doc2.pk)] == before[str(doc2.pk)]
|
||||
|
||||
|
||||
class TestUpdateLlmIndexScopedDocumentIds:
|
||||
"""A document_ids-scoped update must trust the caller and reindex every scoped
|
||||
document, never gating on Document.modified.
|
||||
|
||||
bulk_edit.py's add_tag/remove_tag/modify_tags/set_correspondent/set_document_type/
|
||||
set_storage_path/modify_custom_fields all write via queryset.update() or direct
|
||||
M2M/through-model bulk operations -- none of which call Document.save(), so
|
||||
Document.modified's auto_now never fires. Comparing against
|
||||
get_modified_times() for a document_ids-scoped call would therefore skip
|
||||
reindexing documents whose embedded tags/correspondent/etc. just changed.
|
||||
"""
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_scoped_update_reindexes_despite_unchanged_modified(
|
||||
self,
|
||||
temp_llm_index_dir: Path,
|
||||
mock_embed_model: FakeEmbedding,
|
||||
) -> None:
|
||||
"""A document_ids-scoped update must pick up a tag added via the M2M
|
||||
manager's bulk_create path, even though that leaves modified untouched.
|
||||
|
||||
Steps:
|
||||
1. Build an initial index for a document with no tags.
|
||||
2. Add a tag via a direct through-model bulk_create, mirroring
|
||||
bulk_edit.add_tag -- this does not call Document.save().
|
||||
3. Call update_llm_index(rebuild=False, document_ids=[doc.pk]).
|
||||
4. Assert the stored node metadata now includes the new tag.
|
||||
"""
|
||||
# Step 1
|
||||
tag = TagFactory.create(name="Important")
|
||||
doc = DocumentFactory.create(title="Test Document", added=timezone.now())
|
||||
indexing.update_llm_index(rebuild=True)
|
||||
modified_before = doc.modified
|
||||
|
||||
# Step 2: bulk-add the tag the way bulk_edit.add_tag does -- a direct
|
||||
# through-model insert, no Document.save().
|
||||
DocumentTagRelationship = Document.tags.through
|
||||
DocumentTagRelationship.objects.bulk_create(
|
||||
[DocumentTagRelationship(document_id=doc.pk, tag_id=tag.pk)],
|
||||
)
|
||||
doc.refresh_from_db()
|
||||
assert doc.modified == modified_before, (
|
||||
"Precondition failed: expected modified to be unchanged after a "
|
||||
"through-model bulk tag add"
|
||||
)
|
||||
|
||||
# Step 3
|
||||
result = indexing.update_llm_index(rebuild=False, document_ids=[doc.pk])
|
||||
assert result == "LLM index updated successfully."
|
||||
|
||||
# Step 4
|
||||
with indexing.get_vector_store() as store:
|
||||
nodes = store.get_nodes()
|
||||
assert any(tag.name in node.metadata.get("tags", []) for node in nodes)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_add_or_update_document_updates_existing_entry(
|
||||
temp_llm_index_dir: Path,
|
||||
|
||||
Reference in New Issue
Block a user