mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-11-18 03:41:24 +00:00
Fix: stream AI chat natively async so a disconnect cancels the LLM request
The chat answer was a sync generator wrapped for ASGI, pulling each chunk through sync_to_async(next). A client disconnect cancelled the response task, but the worker thread waiting on the LLM could not be interrupted, so the upstream request kept generating the whole answer for nobody. Make stream_chat_with_documents an async generator. All the blocking setup (ORM queries, the index read lock and its thread-bound sqlite connection, the query embedding, building the LLM client) moves into one sync _prepare_chat called once through sync_to_async, which also hands the DB connection back before the answer streams. The answer then streams through llama-index's asynthesize over the pinned async httpx clients, so a disconnect cancels the LLM request directly. The view returns the async generator as is, replacing the sync-to-async wrapper. The async pinned transport resolved hostnames with a blocking getaddrinfo on the event loop, which would stall the whole worker on slow DNS now that chat uses it in the web server. Resolve in a thread instead, and add the async counterparts of the existing transport tests.
This commit is contained in:
1 parent
a690885a8a
commit
179307d57a
6 files changed
+188
-158
No files matched your search
@@ -9,12 +9,11 @@ from rest_framework import status
|
||||
from rest_framework.test import APIClient
|
||||
from rest_framework.test import APITestCase
|
||||
|
||||
from documents.views import _iterate_in_sync_thread
|
||||
from paperless_testing.factories import UserFactory
|
||||
from paperless_testing.permissions import grant_global
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Generator
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
from django.contrib.auth.models import User
|
||||
from pytest_mock import MockerFixture
|
||||
@@ -59,6 +58,11 @@ class TestChatStreamingViewInputValidation(APITestCase):
|
||||
reaches the Django request the middleware sees.
|
||||
"""
|
||||
chunks = [f"token{i} " for i in range(40)]
|
||||
|
||||
async def answer() -> AsyncIterator[str]:
|
||||
for chunk in chunks:
|
||||
yield chunk
|
||||
|
||||
with (
|
||||
mock.patch(
|
||||
"documents.views.AIConfig",
|
||||
@@ -66,7 +70,7 @@ class TestChatStreamingViewInputValidation(APITestCase):
|
||||
),
|
||||
mock.patch(
|
||||
"paperless_ai.chat.stream_chat_with_documents",
|
||||
return_value=(c for c in chunks),
|
||||
return_value=answer(),
|
||||
),
|
||||
):
|
||||
resp = self.client.post(
|
||||
@@ -163,74 +167,3 @@ class TestChatStreamingViewUnrestrictedFlag:
|
||||
assert (
|
||||
mocked_stream_chat.call_args.kwargs["unrestricted"] is expected_unrestricted
|
||||
)
|
||||
|
||||
|
||||
class TestIterateInSyncThread:
|
||||
@pytest.fixture
|
||||
def events(self) -> list[str]:
|
||||
return []
|
||||
|
||||
@pytest.fixture
|
||||
def chunks(self, events: list[str]) -> Generator[str, None, None]:
|
||||
def generate() -> Generator[str, None, None]:
|
||||
try:
|
||||
for chunk in ("first", "second"):
|
||||
events.append(f"produced {chunk}")
|
||||
yield chunk
|
||||
finally:
|
||||
events.append("closed")
|
||||
|
||||
return generate()
|
||||
|
||||
def test_yields_each_chunk_before_pulling_the_next(
|
||||
self,
|
||||
chunks: Generator[str, None, None],
|
||||
events: list[str],
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A sync generator wrapped for async iteration
|
||||
WHEN:
|
||||
- The wrapper is consumed
|
||||
THEN:
|
||||
- Each chunk is delivered before the next one is generated,
|
||||
rather than the whole generator being drained up front
|
||||
"""
|
||||
|
||||
async def consume() -> None:
|
||||
async for chunk in _iterate_in_sync_thread(chunks):
|
||||
events.append(f"received {chunk}")
|
||||
|
||||
async_to_sync(consume)()
|
||||
|
||||
assert events == [
|
||||
"produced first",
|
||||
"received first",
|
||||
"produced second",
|
||||
"received second",
|
||||
"closed",
|
||||
]
|
||||
|
||||
def test_closing_early_closes_the_source_generator(
|
||||
self,
|
||||
chunks: Generator[str, None, None],
|
||||
events: list[str],
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A sync generator wrapped for async iteration
|
||||
WHEN:
|
||||
- The consumer closes the wrapper after the first chunk, as
|
||||
happens when the client disconnects
|
||||
THEN:
|
||||
- The source generator is closed, so its cleanup runs
|
||||
"""
|
||||
|
||||
async def consume_one() -> None:
|
||||
wrapper = _iterate_in_sync_thread(chunks)
|
||||
await anext(wrapper)
|
||||
await wrapper.aclose()
|
||||
|
||||
async_to_sync(consume_one)()
|
||||
|
||||
assert events == ["produced first", "closed"]
|
||||
Reference in new issue
Block a user