From cd7038a7a4b1c44b292e33f81f09706e17b11a2e Mon Sep 17 00:00:00 2001 From: stumpylog <797416+stumpylog@users.noreply.github.com> Date: Tue, 28 Jul 2026 11:13:05 -0700 Subject: [PATCH] Fix: don't gate document_ids-scoped LLM index updates on Document.modified bulk_edit.py's tag/correspondent/document_type/storage_path/custom_field helpers write via queryset.update() or direct M2M/through-model bulk operations, none of which call Document.save() -- so Document.modified's auto_now never fires. Comparing against get_modified_times() for a document_ids-scoped update_llm_index() call was silently skipping reindex of documents whose embedded metadata had just changed via a bulk edit. The modified-time comparison is now only used for the unscoped, full-library incremental scan, where it is still needed. Co-Authored-By: Claude Sonnet 5 --- src/paperless_ai/indexing.py | 18 +++++-- src/paperless_ai/tests/test_ai_indexing.py | 57 ++++++++++++++++++++++ 2 files changed, 72 insertions(+), 3 deletions(-) diff --git a/src/paperless_ai/indexing.py b/src/paperless_ai/indexing.py index c6dcdf410..2d8d8d5b5 100644 --- a/src/paperless_ai/indexing.py +++ b/src/paperless_ai/indexing.py @@ -419,12 +419,24 @@ def update_llm_index( if document_ids is not None else documents ) - existing = store.get_modified_times() + # When document_ids is given, the caller already knows exactly + # which documents to reindex (e.g. a bulk edit) -- trust it and + # skip the modified-time comparison entirely. Bulk edits (tags, + # correspondent, document type, storage path, custom fields) + # write via queryset.update()/M2M bulk operations, which bypass + # Document.modified's auto_now, so comparing against + # get_modified_times() here would silently skip reindexing + # documents whose embedded metadata just changed. The comparison + # is only meaningful for the unscoped, full-library scan, where + # it avoids re-embedding documents that have not changed. + existing = store.get_modified_times() if document_ids is None else None changed = 0 for document in iter_wrapper(scoped_documents): doc_id = str(document.id) - if existing.get(doc_id) == document.modified.isoformat(): - continue + if existing is not None: + stored_modified = existing.get(doc_id) + if stored_modified == document.modified.isoformat(): + continue nodes = build_document_node(document, chunk_size=chunk_size) _embed_nodes(nodes, embed_model) store.upsert_document(doc_id, nodes) diff --git a/src/paperless_ai/tests/test_ai_indexing.py b/src/paperless_ai/tests/test_ai_indexing.py index 73389e142..895d5113a 100644 --- a/src/paperless_ai/tests/test_ai_indexing.py +++ b/src/paperless_ai/tests/test_ai_indexing.py @@ -21,6 +21,7 @@ from documents.signals import document_consumption_finished from documents.signals import document_updated from documents.tests.factories import DocumentFactory from documents.tests.factories import PaperlessTaskFactory +from documents.tests.factories import TagFactory from paperless.models import ApplicationConfiguration from paperless_ai import indexing from paperless_ai.tests.conftest import FakeEmbedding @@ -347,6 +348,62 @@ def test_update_llm_index_partial_update( assert after[str(doc2.pk)] == before[str(doc2.pk)] +class TestUpdateLlmIndexScopedDocumentIds: + """A document_ids-scoped update must trust the caller and reindex every scoped + document, never gating on Document.modified. + + bulk_edit.py's add_tag/remove_tag/modify_tags/set_correspondent/set_document_type/ + set_storage_path/modify_custom_fields all write via queryset.update() or direct + M2M/through-model bulk operations -- none of which call Document.save(), so + Document.modified's auto_now never fires. Comparing against + get_modified_times() for a document_ids-scoped call would therefore skip + reindexing documents whose embedded tags/correspondent/etc. just changed. + """ + + @pytest.mark.django_db + def test_scoped_update_reindexes_despite_unchanged_modified( + self, + temp_llm_index_dir: Path, + mock_embed_model: FakeEmbedding, + ) -> None: + """A document_ids-scoped update must pick up a tag added via the M2M + manager's bulk_create path, even though that leaves modified untouched. + + Steps: + 1. Build an initial index for a document with no tags. + 2. Add a tag via a direct through-model bulk_create, mirroring + bulk_edit.add_tag -- this does not call Document.save(). + 3. Call update_llm_index(rebuild=False, document_ids=[doc.pk]). + 4. Assert the stored node metadata now includes the new tag. + """ + # Step 1 + tag = TagFactory.create(name="Important") + doc = DocumentFactory.create(title="Test Document", added=timezone.now()) + indexing.update_llm_index(rebuild=True) + modified_before = doc.modified + + # Step 2: bulk-add the tag the way bulk_edit.add_tag does -- a direct + # through-model insert, no Document.save(). + DocumentTagRelationship = Document.tags.through + DocumentTagRelationship.objects.bulk_create( + [DocumentTagRelationship(document_id=doc.pk, tag_id=tag.pk)], + ) + doc.refresh_from_db() + assert doc.modified == modified_before, ( + "Precondition failed: expected modified to be unchanged after a " + "through-model bulk tag add" + ) + + # Step 3 + result = indexing.update_llm_index(rebuild=False, document_ids=[doc.pk]) + assert result == "LLM index updated successfully." + + # Step 4 + with indexing.get_vector_store() as store: + nodes = store.get_nodes() + assert any(tag.name in node.metadata.get("tags", []) for node in nodes) + + @pytest.mark.django_db def test_add_or_update_document_updates_existing_entry( temp_llm_index_dir: Path,