mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-11-18 03:41:24 +00:00
Fix: stream AI chat natively async so a disconnect cancels the LLM request
The chat answer was a sync generator wrapped for ASGI, pulling each chunk through sync_to_async(next). A client disconnect cancelled the response task, but the worker thread waiting on the LLM could not be interrupted, so the upstream request kept generating the whole answer for nobody. Make stream_chat_with_documents an async generator. All the blocking setup (ORM queries, the index read lock and its thread-bound sqlite connection, the query embedding, building the LLM client) moves into one sync _prepare_chat called once through sync_to_async, which also hands the DB connection back before the answer streams. The answer then streams through llama-index's asynthesize over the pinned async httpx clients, so a disconnect cancels the LLM request directly. The view returns the async generator as is, replacing the sync-to-async wrapper. The async pinned transport resolved hostnames with a blocking getaddrinfo on the event loop, which would stall the whole worker on slow DNS now that chat uses it in the web server. Resolve in a thread instead, and add the async counterparts of the existing transport tests.
This commit is contained in:
1 parent
a690885a8a
commit
179307d57a
6 files changed
+188
-158
No files matched your search
@@ -9,12 +9,11 @@ from rest_framework import status
|
||||
from rest_framework.test import APIClient
|
||||
from rest_framework.test import APITestCase
|
||||
|
||||
from documents.views import _iterate_in_sync_thread
|
||||
from paperless_testing.factories import UserFactory
|
||||
from paperless_testing.permissions import grant_global
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Generator
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
from django.contrib.auth.models import User
|
||||
from pytest_mock import MockerFixture
|
||||
@@ -59,6 +58,11 @@ class TestChatStreamingViewInputValidation(APITestCase):
|
||||
reaches the Django request the middleware sees.
|
||||
"""
|
||||
chunks = [f"token{i} " for i in range(40)]
|
||||
|
||||
async def answer() -> AsyncIterator[str]:
|
||||
for chunk in chunks:
|
||||
yield chunk
|
||||
|
||||
with (
|
||||
mock.patch(
|
||||
"documents.views.AIConfig",
|
||||
@@ -66,7 +70,7 @@ class TestChatStreamingViewInputValidation(APITestCase):
|
||||
),
|
||||
mock.patch(
|
||||
"paperless_ai.chat.stream_chat_with_documents",
|
||||
return_value=(c for c in chunks),
|
||||
return_value=answer(),
|
||||
),
|
||||
):
|
||||
resp = self.client.post(
|
||||
@@ -163,74 +167,3 @@ class TestChatStreamingViewUnrestrictedFlag:
|
||||
assert (
|
||||
mocked_stream_chat.call_args.kwargs["unrestricted"] is expected_unrestricted
|
||||
)
|
||||
|
||||
|
||||
class TestIterateInSyncThread:
|
||||
@pytest.fixture
|
||||
def events(self) -> list[str]:
|
||||
return []
|
||||
|
||||
@pytest.fixture
|
||||
def chunks(self, events: list[str]) -> Generator[str, None, None]:
|
||||
def generate() -> Generator[str, None, None]:
|
||||
try:
|
||||
for chunk in ("first", "second"):
|
||||
events.append(f"produced {chunk}")
|
||||
yield chunk
|
||||
finally:
|
||||
events.append("closed")
|
||||
|
||||
return generate()
|
||||
|
||||
def test_yields_each_chunk_before_pulling_the_next(
|
||||
self,
|
||||
chunks: Generator[str, None, None],
|
||||
events: list[str],
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A sync generator wrapped for async iteration
|
||||
WHEN:
|
||||
- The wrapper is consumed
|
||||
THEN:
|
||||
- Each chunk is delivered before the next one is generated,
|
||||
rather than the whole generator being drained up front
|
||||
"""
|
||||
|
||||
async def consume() -> None:
|
||||
async for chunk in _iterate_in_sync_thread(chunks):
|
||||
events.append(f"received {chunk}")
|
||||
|
||||
async_to_sync(consume)()
|
||||
|
||||
assert events == [
|
||||
"produced first",
|
||||
"received first",
|
||||
"produced second",
|
||||
"received second",
|
||||
"closed",
|
||||
]
|
||||
|
||||
def test_closing_early_closes_the_source_generator(
|
||||
self,
|
||||
chunks: Generator[str, None, None],
|
||||
events: list[str],
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A sync generator wrapped for async iteration
|
||||
WHEN:
|
||||
- The consumer closes the wrapper after the first chunk, as
|
||||
happens when the client disconnects
|
||||
THEN:
|
||||
- The source generator is closed, so its cleanup runs
|
||||
"""
|
||||
|
||||
async def consume_one() -> None:
|
||||
wrapper = _iterate_in_sync_thread(chunks)
|
||||
await anext(wrapper)
|
||||
await wrapper.aclose()
|
||||
|
||||
async_to_sync(consume_one)()
|
||||
|
||||
assert events == ["produced first", "closed"]
|
||||
+5
-28
@@ -7,9 +7,7 @@ import tempfile
|
||||
import zipfile
|
||||
from collections import Counter
|
||||
from collections import deque
|
||||
from collections.abc import AsyncIterator
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Generator
|
||||
from datetime import datetime
|
||||
from datetime import timedelta
|
||||
from http import HTTPStatus
|
||||
@@ -29,7 +27,6 @@ from urllib.parse import urlparse
|
||||
import httpx
|
||||
import magic
|
||||
import pathvalidate
|
||||
from asgiref.sync import sync_to_async
|
||||
from django.conf import settings
|
||||
from django.contrib.auth.models import Group
|
||||
from django.contrib.auth.models import User
|
||||
@@ -2419,24 +2416,6 @@ class DocumentViewSet(
|
||||
)
|
||||
|
||||
|
||||
async def _iterate_in_sync_thread(
|
||||
chunks: Generator[str, None, None],
|
||||
) -> AsyncIterator[str]:
|
||||
"""Serve a blocking sync generator to an ASGI server chunk by chunk.
|
||||
|
||||
Under ASGI, Django drains a sync iterator with ``sync_to_async(list)``
|
||||
before sending anything. Pull each chunk in the request's sync thread
|
||||
instead, where the generator's ORM access belongs.
|
||||
"""
|
||||
sentinel = object() # StopIteration can't cross an asyncio Future
|
||||
next_chunk = sync_to_async(next)
|
||||
try:
|
||||
while (chunk := await next_chunk(chunks, sentinel)) is not sentinel:
|
||||
yield chunk
|
||||
finally:
|
||||
await sync_to_async(chunks.close)()
|
||||
|
||||
|
||||
class ChatStreamingSerializer(serializers.Serializer[dict[str, Any]]):
|
||||
q = serializers.CharField(required=True, max_length=4000)
|
||||
document_id = serializers.IntegerField(required=False, allow_null=True)
|
||||
@@ -2496,13 +2475,11 @@ class ChatStreamingView(GenericAPIView[Any]):
|
||||
)
|
||||
|
||||
return StreamingHttpResponse(
|
||||
_iterate_in_sync_thread(
|
||||
stream_chat_with_documents(
|
||||
query_str=question,
|
||||
documents=documents,
|
||||
unrestricted=unrestricted,
|
||||
output_language=output_language,
|
||||
),
|
||||
stream_chat_with_documents(
|
||||
query_str=question,
|
||||
documents=documents,
|
||||
unrestricted=unrestricted,
|
||||
output_language=output_language,
|
||||
),
|
||||
content_type="text/event-stream",
|
||||
)
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import asyncio
|
||||
import ipaddress
|
||||
import socket
|
||||
from collections.abc import Collection
|
||||
@@ -175,7 +176,9 @@ class PinnedHostAsyncHTTPTransport(httpx.AsyncHTTPTransport):
|
||||
self.allow_internal = allow_internal
|
||||
|
||||
async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
|
||||
request = _rewrite_request_to_pinned_ip(
|
||||
# Resolving the host blocks, so keep it off the event loop
|
||||
request = await asyncio.to_thread(
|
||||
_rewrite_request_to_pinned_ip,
|
||||
request,
|
||||
allow_internal=self.allow_internal,
|
||||
)
|
||||
|
||||
@@ -1,11 +1,19 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from typing import TYPE_CHECKING
|
||||
from unittest import mock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from paperless.network import PinnedHostAsyncHTTPTransport
|
||||
from paperless.network import PinnedHostHTTPTransport
|
||||
from paperless.network import is_public_ip
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pytest_mock import MockerFixture
|
||||
|
||||
|
||||
def test_pinned_host_transport_blocks_internal_rebinding():
|
||||
transport = PinnedHostHTTPTransport(allow_internal=False)
|
||||
@@ -51,6 +59,58 @@ def test_pinned_host_transport_rewrites_to_vetted_ip():
|
||||
assert response.status_code == 200
|
||||
|
||||
|
||||
class TestPinnedHostAsyncHTTPTransport:
|
||||
def test_blocks_internal_rebinding(self, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A hostname that resolves to a loopback address
|
||||
WHEN:
|
||||
- The async transport sends a request to it
|
||||
THEN:
|
||||
- The request is refused as a non-public address
|
||||
"""
|
||||
mocker.patch(
|
||||
"paperless.network.resolve_hostname_ips",
|
||||
return_value=["127.0.0.1"],
|
||||
)
|
||||
transport = PinnedHostAsyncHTTPTransport(allow_internal=False)
|
||||
request = httpx.Request("GET", "http://example.com/test")
|
||||
|
||||
with pytest.raises(httpx.ConnectError, match="non-public address"):
|
||||
asyncio.run(transport.handle_async_request(request))
|
||||
|
||||
def test_rewrites_to_vetted_ip(self, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A hostname that resolves to a public address
|
||||
WHEN:
|
||||
- The async transport sends a request to it
|
||||
THEN:
|
||||
- The connection goes to that address, keeping the original Host
|
||||
header and TLS SNI hostname
|
||||
"""
|
||||
mocker.patch(
|
||||
"paperless.network.resolve_hostname_ips",
|
||||
return_value=["93.184.216.34"],
|
||||
)
|
||||
sent = mocker.patch.object(
|
||||
httpx.AsyncHTTPTransport,
|
||||
"handle_async_request",
|
||||
autospec=True,
|
||||
side_effect=lambda _self, req: httpx.Response(200, request=req),
|
||||
)
|
||||
transport = PinnedHostAsyncHTTPTransport(allow_internal=False)
|
||||
request = httpx.Request("GET", "https://example.com:8443/test")
|
||||
|
||||
response = asyncio.run(transport.handle_async_request(request))
|
||||
|
||||
assert response.status_code == 200
|
||||
rewritten = sent.call_args.args[1]
|
||||
assert str(rewritten.url) == "https://93.184.216.34:8443/test"
|
||||
assert rewritten.headers["Host"] == "example.com:8443"
|
||||
assert rewritten.extensions["sni_hostname"] == "example.com"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"address",
|
||||
[
|
||||
|
||||
+57
-29
@@ -1,7 +1,10 @@
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from asgiref.sync import sync_to_async
|
||||
from django.db import connections
|
||||
from django.db.models import QuerySet
|
||||
|
||||
from documents.models import Document
|
||||
@@ -17,6 +20,10 @@ from paperless_ai.prompts.context import ChatQaPromptContext
|
||||
from paperless_ai.prompts.context import ChatRefinePromptContext
|
||||
from paperless_ai.prompts.render import render_prompt
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from llama_index.core.response_synthesizers import BaseSynthesizer
|
||||
from llama_index.core.schema import NodeWithScore
|
||||
|
||||
logger = logging.getLogger("paperless_ai.chat")
|
||||
|
||||
CHAT_METADATA_DELIMITER = "\n\n__PAPERLESS_CHAT_METADATA__"
|
||||
@@ -103,35 +110,67 @@ def _format_chat_metadata_trailer(references: list[dict[str, int | str]]) -> str
|
||||
)
|
||||
|
||||
|
||||
def stream_chat_with_documents(
|
||||
async def stream_chat_with_documents(
|
||||
query_str: str,
|
||||
documents: QuerySet[Document],
|
||||
*,
|
||||
unrestricted: bool = False,
|
||||
output_language: str | None = None,
|
||||
):
|
||||
) -> AsyncIterator[str]:
|
||||
try:
|
||||
yield from _stream_chat_with_documents(
|
||||
async for chunk in _stream_chat_with_documents(
|
||||
query_str,
|
||||
documents,
|
||||
unrestricted=unrestricted,
|
||||
output_language=output_language,
|
||||
)
|
||||
):
|
||||
yield chunk
|
||||
except Exception as e:
|
||||
logger.exception("Failed to stream document chat response: %s", e)
|
||||
yield CHAT_ERROR_MESSAGE
|
||||
|
||||
|
||||
def _stream_chat_with_documents(
|
||||
async def _stream_chat_with_documents(
|
||||
query_str: str,
|
||||
documents: QuerySet[Document],
|
||||
*,
|
||||
unrestricted: bool = False,
|
||||
output_language: str | None = None,
|
||||
):
|
||||
if not documents.exists():
|
||||
) -> AsyncIterator[str]:
|
||||
prepared = await sync_to_async(_prepare_chat)(
|
||||
query_str,
|
||||
documents,
|
||||
unrestricted=unrestricted,
|
||||
output_language=output_language,
|
||||
)
|
||||
if prepared is None:
|
||||
yield CHAT_NO_CONTENT_MESSAGE
|
||||
return
|
||||
response_synthesizer, top_nodes, references = prepared
|
||||
|
||||
logger.debug("Document chat query: %s", query_str)
|
||||
response_stream = await response_synthesizer.asynthesize(query_str, top_nodes)
|
||||
async for chunk in response_stream.async_response_gen():
|
||||
yield chunk
|
||||
|
||||
if references:
|
||||
yield _format_chat_metadata_trailer(references)
|
||||
|
||||
|
||||
def _prepare_chat(
|
||||
query_str: str,
|
||||
documents: QuerySet[Document],
|
||||
*,
|
||||
unrestricted: bool,
|
||||
output_language: str | None,
|
||||
) -> tuple["BaseSynthesizer", list["NodeWithScore"], list[dict[str, int | str]]] | None:
|
||||
"""Everything before the answer streams, all of it blocking: ORM queries,
|
||||
the index read lock and its thread-bound sqlite connection, the query
|
||||
embedding, and building the LLM client. Returns None when there is no
|
||||
content to answer from.
|
||||
"""
|
||||
if not documents.exists():
|
||||
return None
|
||||
|
||||
from llama_index.core.prompts import PromptTemplate
|
||||
from llama_index.core.response_synthesizers import get_response_synthesizer
|
||||
@@ -150,8 +189,6 @@ def _stream_chat_with_documents(
|
||||
|
||||
# Hold the shared read lock only while reading the index, so the compaction
|
||||
# swap cannot replace the database file while this connection is open.
|
||||
# Everything after retrieval works on the nodes already in memory, so a slow
|
||||
# LLM stream never holds the lock.
|
||||
with read_store() as store:
|
||||
if permitted_ids is not None:
|
||||
filters = _permitted_documents_filter(store, permitted_ids)
|
||||
@@ -179,35 +216,26 @@ def _stream_chat_with_documents(
|
||||
|
||||
if not top_nodes:
|
||||
logger.warning("No nodes found for the given documents.")
|
||||
yield CHAT_NO_CONTENT_MESSAGE
|
||||
return
|
||||
|
||||
client = AIClient()
|
||||
return None
|
||||
|
||||
references = _get_document_references(documents, top_nodes)
|
||||
|
||||
prompt_template = PromptTemplate(template=_build_chat_prompt(output_language))
|
||||
refine_template = PromptTemplate(template=_build_refine_prompt(output_language))
|
||||
client = AIClient()
|
||||
response_synthesizer = get_response_synthesizer(
|
||||
llm=client.llm,
|
||||
prompt_helper=get_rag_prompt_helper(
|
||||
chunk_size=config.llm_embedding_chunk_size,
|
||||
context_size=config.llm_context_size,
|
||||
),
|
||||
text_qa_template=prompt_template,
|
||||
refine_template=refine_template,
|
||||
text_qa_template=PromptTemplate(template=_build_chat_prompt(output_language)),
|
||||
refine_template=PromptTemplate(
|
||||
template=_build_refine_prompt(output_language),
|
||||
),
|
||||
streaming=True,
|
||||
)
|
||||
|
||||
logger.debug("Document chat query: %s", query_str)
|
||||
# Release the pooled DB connection for the slow streaming LLM response
|
||||
# so it is not pinned for the whole stream; see paperless_ai.db and
|
||||
# #12976.
|
||||
with db_connection_released():
|
||||
response_stream = response_synthesizer.synthesize(query_str, top_nodes)
|
||||
for chunk in response_stream.response_gen:
|
||||
yield chunk
|
||||
sys.stdout.flush()
|
||||
# No ORM access happens while the answer streams, so hand the pooled DB
|
||||
# connection back now rather than pin it for the whole stream. See #12976.
|
||||
connections.close_all()
|
||||
|
||||
if references:
|
||||
yield _format_chat_metadata_trailer(references)
|
||||
return response_synthesizer, top_nodes, references
|
||||
@@ -4,10 +4,12 @@ import json
|
||||
from contextlib import contextmanager
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock
|
||||
from unittest.mock import MagicMock
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from asgiref.sync import async_to_sync
|
||||
from django.db.models.signals import post_init
|
||||
from django.utils import timezone
|
||||
from llama_index.core import settings as llama_settings
|
||||
@@ -26,6 +28,7 @@ from paperless_ai.chat import stream_chat_with_documents
|
||||
from paperless_testing.factories import DocumentFactory
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import AsyncIterator
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
|
||||
@@ -66,6 +69,26 @@ def assert_chat_output(
|
||||
}
|
||||
|
||||
|
||||
def collect(chunks: AsyncIterator[str]) -> list[str]:
|
||||
"""Drain the chat stream from sync test code. Its blocking setup then runs
|
||||
on the test's own thread, inside the test's DB transaction."""
|
||||
|
||||
async def drain() -> list[str]:
|
||||
return [chunk async for chunk in chunks]
|
||||
|
||||
return async_to_sync(drain)()
|
||||
|
||||
|
||||
def fake_response_stream(chunks: list[str]) -> MagicMock:
|
||||
"""An AsyncStreamingResponse stand-in yielding ``chunks``."""
|
||||
|
||||
async def response_gen() -> AsyncIterator[str]:
|
||||
for chunk in chunks:
|
||||
yield chunk
|
||||
|
||||
return MagicMock(async_response_gen=response_gen)
|
||||
|
||||
|
||||
def _fake_documents_queryset(pks: list[int]) -> MagicMock:
|
||||
qs = MagicMock()
|
||||
qs.exists.return_value = bool(pks)
|
||||
@@ -170,19 +193,19 @@ def test_stream_chat_with_one_document_retrieval(
|
||||
),
|
||||
]
|
||||
|
||||
mock_response_stream = MagicMock()
|
||||
mock_response_stream.response_gen = iter(["chunk1", "chunk2"])
|
||||
mock_synthesizer = mock_get_response_synthesizer.return_value
|
||||
mock_synthesizer.synthesize.return_value = mock_response_stream
|
||||
mock_synthesizer.asynthesize = AsyncMock(
|
||||
return_value=fake_response_stream(["chunk1", "chunk2"]),
|
||||
)
|
||||
|
||||
with patch(
|
||||
"llama_index.core.retrievers.VectorIndexRetriever",
|
||||
return_value=mock_retriever_instance,
|
||||
):
|
||||
output = list(stream_chat_with_documents("What is this?", documents))
|
||||
output = collect(stream_chat_with_documents("What is this?", documents))
|
||||
|
||||
mock_retriever_instance.retrieve.assert_called_once_with("What is this?")
|
||||
mock_synthesizer.synthesize.assert_called_once_with(
|
||||
mock_synthesizer.asynthesize.assert_awaited_once_with(
|
||||
"What is this?",
|
||||
mock_retriever_instance.retrieve.return_value,
|
||||
)
|
||||
@@ -236,20 +259,19 @@ def test_stream_chat_with_multiple_documents_retrieval(patch_embed_nodes) -> Non
|
||||
MagicMock(metadata={"document_id": str(doc2.pk), "title": "Document 2"}),
|
||||
]
|
||||
|
||||
mock_response_stream = MagicMock()
|
||||
mock_response_stream.response_gen = iter(["chunk1", "chunk2"])
|
||||
|
||||
mock_synthesizer = mock_get_response_synthesizer.return_value
|
||||
mock_synthesizer.synthesize.return_value = mock_response_stream
|
||||
mock_synthesizer.asynthesize = AsyncMock(
|
||||
return_value=fake_response_stream(["chunk1", "chunk2"]),
|
||||
)
|
||||
|
||||
with patch(
|
||||
"llama_index.core.retrievers.VectorIndexRetriever",
|
||||
return_value=mock_retriever_instance,
|
||||
):
|
||||
output = list(stream_chat_with_documents("What's up?", documents))
|
||||
output = collect(stream_chat_with_documents("What's up?", documents))
|
||||
|
||||
mock_retriever_instance.retrieve.assert_called_once_with("What's up?")
|
||||
mock_synthesizer.synthesize.assert_called_once_with(
|
||||
mock_synthesizer.asynthesize.assert_awaited_once_with(
|
||||
"What's up?",
|
||||
mock_retriever_instance.retrieve.return_value,
|
||||
)
|
||||
@@ -266,7 +288,9 @@ def test_stream_chat_with_multiple_documents_retrieval(patch_embed_nodes) -> Non
|
||||
|
||||
def test_stream_chat_empty_document_list() -> None:
|
||||
with patch("paperless_ai.chat.load_or_build_index") as mock_load_index:
|
||||
output = list(stream_chat_with_documents("Any info?", Document.objects.none()))
|
||||
output = collect(
|
||||
stream_chat_with_documents("Any info?", Document.objects.none()),
|
||||
)
|
||||
mock_load_index.assert_not_called()
|
||||
assert output == ["Sorry, I couldn't find any content to answer your question."]
|
||||
|
||||
@@ -286,7 +310,7 @@ def test_stream_chat_no_matching_nodes() -> None:
|
||||
mock_index.vector_store.get_nodes.return_value = []
|
||||
mock_load_index.return_value = mock_index
|
||||
|
||||
output = list(
|
||||
output = collect(
|
||||
stream_chat_with_documents("Any info?", _fake_documents_queryset([1])),
|
||||
)
|
||||
|
||||
@@ -317,7 +341,7 @@ def test_stream_chat_unexpected_failure_returns_generic_error(caplog) -> None:
|
||||
)
|
||||
mock_retriever_cls.return_value = mock_retriever
|
||||
|
||||
output = list(
|
||||
output = collect(
|
||||
stream_chat_with_documents("Any info?", _fake_documents_queryset([1])),
|
||||
)
|
||||
|
||||
@@ -367,7 +391,7 @@ class TestStreamChatRetrieval:
|
||||
) -> None:
|
||||
doc = DocumentFactory.create(content="hello world")
|
||||
# Nothing indexed for this document yet.
|
||||
out = list(
|
||||
out = collect(
|
||||
chat.stream_chat_with_documents(
|
||||
"question?",
|
||||
Document.objects.filter(pk=doc.pk),
|
||||
@@ -390,7 +414,7 @@ class TestStreamChatRetrieval:
|
||||
indexing.llm_index_add_or_update_document(included)
|
||||
indexing.llm_index_add_or_update_document(excluded)
|
||||
|
||||
list(
|
||||
collect(
|
||||
chat.stream_chat_with_documents(
|
||||
"question?",
|
||||
Document.objects.filter(pk=included.pk),
|
||||
@@ -419,7 +443,7 @@ class TestStreamChatRetrieval:
|
||||
document = DocumentFactory.create(content="indexed document content")
|
||||
indexing.llm_index_add_or_update_document(document)
|
||||
|
||||
list(
|
||||
collect(
|
||||
chat.stream_chat_with_documents(
|
||||
"question?",
|
||||
Document.objects.filter(pk=document.pk),
|
||||
@@ -455,7 +479,7 @@ class TestStreamChatRetrieval:
|
||||
deleted_at=timezone.now(),
|
||||
)
|
||||
|
||||
list(
|
||||
collect(
|
||||
chat.stream_chat_with_documents(
|
||||
"question?",
|
||||
Document.objects.filter(pk=kept.pk),
|
||||
@@ -487,7 +511,7 @@ class TestStreamChatRetrieval:
|
||||
indexing.llm_index_add_or_update_document(doc)
|
||||
permitted = Document.objects.filter(pk__in=[docs[0].pk, docs[1].pk])
|
||||
|
||||
list(chat.stream_chat_with_documents("question?", permitted))
|
||||
collect(chat.stream_chat_with_documents("question?", permitted))
|
||||
|
||||
metadata_filter = captured_filters[0].filters[0]
|
||||
assert metadata_filter.operator == FilterOperator.NIN
|
||||
@@ -520,16 +544,21 @@ class TestStreamChatRetrieval:
|
||||
synthesizer = mocker.patch(
|
||||
"llama_index.core.response_synthesizers.get_response_synthesizer",
|
||||
).return_value
|
||||
synthesizer.synthesize.return_value.response_gen = iter(["chunk"])
|
||||
synthesizer.asynthesize = AsyncMock(
|
||||
return_value=fake_response_stream(["chunk"]),
|
||||
)
|
||||
|
||||
list(
|
||||
collect(
|
||||
chat.stream_chat_with_documents(
|
||||
"question?",
|
||||
Document.objects.filter(pk=permitted_doc.pk),
|
||||
),
|
||||
)
|
||||
|
||||
synthesizer.synthesize.assert_called_once_with("question?", [permitted_node])
|
||||
synthesizer.asynthesize.assert_awaited_once_with(
|
||||
"question?",
|
||||
[permitted_node],
|
||||
)
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_get_document_references_only_queries_referenced_documents(
|
||||
@@ -612,9 +641,9 @@ class TestChatReadLock:
|
||||
]
|
||||
)
|
||||
|
||||
def synthesize(*_args: object) -> object:
|
||||
def asynthesize(*_args: object) -> MagicMock:
|
||||
events.append("synthesize")
|
||||
return mocker.MagicMock(response_gen=iter(["chunk"]))
|
||||
return fake_response_stream(["chunk"])
|
||||
|
||||
mocker.patch("paperless_ai.chat.read_store", fake_read_store)
|
||||
mocker.patch("paperless_ai.chat.AIClient")
|
||||
@@ -625,9 +654,9 @@ class TestChatReadLock:
|
||||
)
|
||||
mocker.patch(
|
||||
"llama_index.core.response_synthesizers.get_response_synthesizer",
|
||||
).return_value.synthesize.side_effect = synthesize
|
||||
).return_value.asynthesize = AsyncMock(side_effect=asynthesize)
|
||||
|
||||
list(
|
||||
collect(
|
||||
stream_chat_with_documents(
|
||||
"What is this?",
|
||||
Document.objects.filter(pk=document.pk),
|
||||
|
||||
Reference in new issue
Block a user