Fix: don't gate document_ids-scoped LLM index updates on Document.modified

bulk_edit.py's tag/correspondent/document_type/storage_path/custom_field
helpers write via queryset.update() or direct M2M/through-model bulk
operations, none of which call Document.save() -- so Document.modified's
auto_now never fires. Comparing against get_modified_times() for a
document_ids-scoped update_llm_index() call was silently skipping reindex
of documents whose embedded metadata had just changed via a bulk edit.
The modified-time comparison is now only used for the unscoped,
full-library incremental scan, where it is still needed.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
stumpylog
2026-07-28 11:13:05 -07:00
co-authored by Claude Sonnet 5
parent df8a6cc715
commit cd7038a7a4
2 changed files with 72 additions and 3 deletions
+15 -3
View File
@@ -419,12 +419,24 @@ def update_llm_index(
if document_ids is not None
else documents
)
existing = store.get_modified_times()
# When document_ids is given, the caller already knows exactly
# which documents to reindex (e.g. a bulk edit) -- trust it and
# skip the modified-time comparison entirely. Bulk edits (tags,
# correspondent, document type, storage path, custom fields)
# write via queryset.update()/M2M bulk operations, which bypass
# Document.modified's auto_now, so comparing against
# get_modified_times() here would silently skip reindexing
# documents whose embedded metadata just changed. The comparison
# is only meaningful for the unscoped, full-library scan, where
# it avoids re-embedding documents that have not changed.
existing = store.get_modified_times() if document_ids is None else None
changed = 0
for document in iter_wrapper(scoped_documents):
doc_id = str(document.id)
if existing.get(doc_id) == document.modified.isoformat():
continue
if existing is not None:
stored_modified = existing.get(doc_id)
if stored_modified == document.modified.isoformat():
continue
nodes = build_document_node(document, chunk_size=chunk_size)
_embed_nodes(nodes, embed_model)
store.upsert_document(doc_id, nodes)
@@ -21,6 +21,7 @@ from documents.signals import document_consumption_finished
from documents.signals import document_updated
from documents.tests.factories import DocumentFactory
from documents.tests.factories import PaperlessTaskFactory
from documents.tests.factories import TagFactory
from paperless.models import ApplicationConfiguration
from paperless_ai import indexing
from paperless_ai.tests.conftest import FakeEmbedding
@@ -347,6 +348,62 @@ def test_update_llm_index_partial_update(
assert after[str(doc2.pk)] == before[str(doc2.pk)]
class TestUpdateLlmIndexScopedDocumentIds:
"""A document_ids-scoped update must trust the caller and reindex every scoped
document, never gating on Document.modified.
bulk_edit.py's add_tag/remove_tag/modify_tags/set_correspondent/set_document_type/
set_storage_path/modify_custom_fields all write via queryset.update() or direct
M2M/through-model bulk operations -- none of which call Document.save(), so
Document.modified's auto_now never fires. Comparing against
get_modified_times() for a document_ids-scoped call would therefore skip
reindexing documents whose embedded tags/correspondent/etc. just changed.
"""
@pytest.mark.django_db
def test_scoped_update_reindexes_despite_unchanged_modified(
self,
temp_llm_index_dir: Path,
mock_embed_model: FakeEmbedding,
) -> None:
"""A document_ids-scoped update must pick up a tag added via the M2M
manager's bulk_create path, even though that leaves modified untouched.
Steps:
1. Build an initial index for a document with no tags.
2. Add a tag via a direct through-model bulk_create, mirroring
bulk_edit.add_tag -- this does not call Document.save().
3. Call update_llm_index(rebuild=False, document_ids=[doc.pk]).
4. Assert the stored node metadata now includes the new tag.
"""
# Step 1
tag = TagFactory.create(name="Important")
doc = DocumentFactory.create(title="Test Document", added=timezone.now())
indexing.update_llm_index(rebuild=True)
modified_before = doc.modified
# Step 2: bulk-add the tag the way bulk_edit.add_tag does -- a direct
# through-model insert, no Document.save().
DocumentTagRelationship = Document.tags.through
DocumentTagRelationship.objects.bulk_create(
[DocumentTagRelationship(document_id=doc.pk, tag_id=tag.pk)],
)
doc.refresh_from_db()
assert doc.modified == modified_before, (
"Precondition failed: expected modified to be unchanged after a "
"through-model bulk tag add"
)
# Step 3
result = indexing.update_llm_index(rebuild=False, document_ids=[doc.pk])
assert result == "LLM index updated successfully."
# Step 4
with indexing.get_vector_store() as store:
nodes = store.get_nodes()
assert any(tag.name in node.metadata.get("tags", []) for node in nodes)
@pytest.mark.django_db
def test_add_or_update_document_updates_existing_entry(
temp_llm_index_dir: Path,