mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-07-26 22:04:56 +00:00
Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
318520a0f1 | ||
|
|
ea1f3653a9 | ||
|
|
423d4b9f2d |
@@ -25,7 +25,7 @@
|
||||
<input #textFilterInput class="form-control form-control-sm" type="text"
|
||||
[disabled]="textFilterModifierIsNull"
|
||||
[(ngModel)]="textFilter"
|
||||
(keyup)="textFilterKeyup($event)"
|
||||
(keydown)="textFilterKeydown($event)"
|
||||
[ngbTypeahead]="searchAutoComplete"
|
||||
(selectItem)="itemSelected($event)"
|
||||
[readonly]="textFilterTarget === 'fulltext-morelike'">
|
||||
|
||||
+3
-3
@@ -2180,17 +2180,17 @@ describe('FilterEditorComponent', () => {
|
||||
it('should support Enter / Esc key on text field', () => {
|
||||
component.textFilterInput.nativeElement.value = 'foo'
|
||||
component.textFilterInput.nativeElement.dispatchEvent(
|
||||
new KeyboardEvent('keyup', { key: 'Enter' })
|
||||
new KeyboardEvent('keydown', { key: 'Enter' })
|
||||
)
|
||||
expect(component.textFilter).toEqual('foo')
|
||||
component.textFilterInput.nativeElement.value = 'foo bar'
|
||||
component.textFilterInput.nativeElement.dispatchEvent(
|
||||
new KeyboardEvent('keyup', { key: 'Escape' })
|
||||
new KeyboardEvent('keydown', { key: 'Escape' })
|
||||
)
|
||||
expect(component.textFilter).toEqual('')
|
||||
const blurSpy = jest.spyOn(component.textFilterInput.nativeElement, 'blur')
|
||||
component.textFilterInput.nativeElement.dispatchEvent(
|
||||
new KeyboardEvent('keyup', { key: 'Escape' })
|
||||
new KeyboardEvent('keydown', { key: 'Escape' })
|
||||
)
|
||||
expect(blurSpy).toHaveBeenCalled()
|
||||
})
|
||||
|
||||
@@ -1312,7 +1312,7 @@ export class FilterEditorComponent
|
||||
}
|
||||
}
|
||||
|
||||
textFilterKeyup(event: KeyboardEvent) {
|
||||
textFilterKeydown(event: KeyboardEvent) {
|
||||
if (event.key == 'Enter') {
|
||||
const filterString = (
|
||||
this.textFilterInput.nativeElement as HTMLInputElement
|
||||
|
||||
@@ -331,7 +331,6 @@ def bulk_update_documents(document_ids) -> None:
|
||||
if ai_config.llm_index_enabled:
|
||||
update_llm_index(
|
||||
rebuild=False,
|
||||
document_ids=document_ids,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -401,10 +401,6 @@ class TestAIIndex(DirectoriesMixin, TestCase):
|
||||
"documents.tasks.update_llm_index",
|
||||
) as update_llm_index,
|
||||
):
|
||||
doc_ids = [doc.pk for doc in docs]
|
||||
tasks.bulk_update_documents(doc_ids)
|
||||
tasks.bulk_update_documents([doc.pk for doc in docs])
|
||||
self.assertEqual(update_document_in_llm_index.apply_async.call_count, 0)
|
||||
update_llm_index.assert_called_once_with(
|
||||
rebuild=False,
|
||||
document_ids=doc_ids,
|
||||
)
|
||||
update_llm_index.assert_called_once()
|
||||
|
||||
@@ -2,6 +2,8 @@ from pydantic import BaseModel
|
||||
|
||||
|
||||
class DocumentClassifierSchema(BaseModel):
|
||||
"""Schema for document classification suggestions."""
|
||||
|
||||
title: str
|
||||
tags: list[str]
|
||||
correspondents: list[str]
|
||||
|
||||
@@ -5,6 +5,7 @@ from datetime import timedelta
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.exceptions import ObjectDoesNotExist
|
||||
from django.utils import timezone
|
||||
from filelock import FileLock
|
||||
from filelock import ReadWriteLock
|
||||
@@ -167,6 +168,19 @@ def write_store(embed_model_name: str | None = None):
|
||||
yield store
|
||||
|
||||
|
||||
def _safe_related_name(document: Document, field: str) -> str | None:
|
||||
"""
|
||||
Returns the ``name`` of a related object (correspondent, document_type,
|
||||
storage_path), or None if the FK is unset or points at a row that has
|
||||
since been deleted (e.g. concurrently with this call).
|
||||
"""
|
||||
try:
|
||||
related = getattr(document, field)
|
||||
except ObjectDoesNotExist:
|
||||
return None
|
||||
return related.name if related else None
|
||||
|
||||
|
||||
def build_document_node(
|
||||
document: Document,
|
||||
*,
|
||||
@@ -180,14 +194,10 @@ def build_document_node(
|
||||
"document_id": str(document.id),
|
||||
"title": document.title,
|
||||
"tags": [t.name for t in document.tags.all()],
|
||||
"correspondent": document.correspondent.name
|
||||
if document.correspondent
|
||||
else None,
|
||||
"document_type": document.document_type.name
|
||||
if document.document_type
|
||||
else None,
|
||||
"correspondent": _safe_related_name(document, "correspondent"),
|
||||
"document_type": _safe_related_name(document, "document_type"),
|
||||
"filename": document.filename,
|
||||
"storage_path": document.storage_path.name if document.storage_path else None,
|
||||
"storage_path": _safe_related_name(document, "storage_path"),
|
||||
"archive_serial_number": document.archive_serial_number,
|
||||
"created": document.created.isoformat() if document.created else None,
|
||||
"added": document.added.isoformat() if document.added else None,
|
||||
@@ -318,16 +328,8 @@ def update_llm_index(
|
||||
*,
|
||||
iter_wrapper: IterWrapper[Document] = identity,
|
||||
rebuild=False,
|
||||
document_ids: Iterable[int] | None = None,
|
||||
) -> str:
|
||||
"""Rebuild or incrementally update the LLM index.
|
||||
|
||||
``document_ids``, when given, scopes an incremental update to just those
|
||||
documents instead of scanning the whole library -- callers that already
|
||||
know which documents changed (e.g. a bulk edit) should pass this to avoid
|
||||
an O(library size) scan per call. Ignored whenever a rebuild actually
|
||||
happens, since a rebuild always covers the whole library regardless.
|
||||
"""
|
||||
"""Rebuild or incrementally update the LLM index."""
|
||||
with write_store() as store:
|
||||
try:
|
||||
with _exclude_readers():
|
||||
@@ -377,16 +379,9 @@ def update_llm_index(
|
||||
store.add(nodes)
|
||||
msg = "LLM index rebuilt successfully."
|
||||
else:
|
||||
scoped_documents = (
|
||||
Document.objects.filter(id__in=document_ids)
|
||||
.select_related("correspondent", "document_type", "storage_path")
|
||||
.prefetch_related("tags")
|
||||
if document_ids is not None
|
||||
else documents
|
||||
)
|
||||
existing = store.get_modified_times()
|
||||
changed = 0
|
||||
for document in iter_wrapper(scoped_documents):
|
||||
for document in iter_wrapper(documents):
|
||||
doc_id = str(document.id)
|
||||
if existing.get(doc_id) == document.modified.isoformat():
|
||||
continue
|
||||
|
||||
@@ -8,7 +8,9 @@ from django.test import override_settings
|
||||
from django.utils import timezone
|
||||
from llama_index.core.schema import MetadataMode
|
||||
|
||||
from documents.models import Correspondent
|
||||
from documents.models import Document
|
||||
from documents.models import DocumentType
|
||||
from documents.models import PaperlessTask
|
||||
from documents.signals import document_consumption_finished
|
||||
from documents.signals import document_updated
|
||||
@@ -95,6 +97,38 @@ def test_build_document_node_structured_fields_in_metadata(
|
||||
assert "modified" in node.metadata
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_build_document_node_survives_concurrently_deleted_correspondent(
|
||||
real_document: Document,
|
||||
) -> None:
|
||||
"""Regression test for #13314.
|
||||
|
||||
If a document's correspondent (or document type) is deleted after the
|
||||
in-memory Document instance was loaded but before build_document_node
|
||||
resolves the relation, accessing the FK must not raise -- it should
|
||||
behave like an unset FK and produce None in the metadata instead of
|
||||
aborting the whole indexing pass.
|
||||
"""
|
||||
correspondent = Correspondent.objects.create(name="Stale Correspondent")
|
||||
document_type = DocumentType.objects.create(name="Stale Type")
|
||||
real_document.correspondent = correspondent
|
||||
real_document.document_type = document_type
|
||||
real_document.save()
|
||||
|
||||
# Re-fetch to get an instance whose correspondent/document_type relations
|
||||
# are unresolved (not yet cached), mirroring a task that loaded the
|
||||
# document before the concurrent deletion below.
|
||||
stale_document = Document.objects.get(pk=real_document.pk)
|
||||
|
||||
correspondent.delete()
|
||||
document_type.delete()
|
||||
|
||||
nodes = indexing.build_document_node(stale_document)
|
||||
assert len(nodes) > 0
|
||||
assert nodes[0].metadata["correspondent"] is None
|
||||
assert nodes[0].metadata["document_type"] is None
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_build_document_node_excludes_document_id_from_llm_context(
|
||||
real_document: Document,
|
||||
@@ -252,22 +286,6 @@ def test_update_llm_index_partial_update(
|
||||
assert store.table_exists(), (
|
||||
"Expected the vector store table to exist after incremental update"
|
||||
)
|
||||
before = store.get_modified_times()
|
||||
|
||||
# A further edit, scoped via document_ids to just doc3 -- doc2 must be
|
||||
# left exactly as it was, proving document_ids restricts the scan
|
||||
# instead of falling back to the whole library.
|
||||
doc3.modified = timezone.now()
|
||||
doc3.save()
|
||||
|
||||
result = indexing.update_llm_index(rebuild=False, document_ids=[doc3.pk])
|
||||
assert result == "LLM index updated successfully."
|
||||
|
||||
with indexing.get_vector_store() as store:
|
||||
after = store.get_modified_times()
|
||||
|
||||
assert after[str(doc3.pk)] == doc3.modified.isoformat()
|
||||
assert after[str(doc2.pk)] == before[str(doc2.pk)]
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
|
||||
Reference in New Issue
Block a user