mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-11-18 03:41:24 +00:00
Fix: stream AI chat natively async so a disconnect cancels the LLM request
The chat answer was a sync generator wrapped for ASGI, pulling each chunk through sync_to_async(next). A client disconnect cancelled the response task, but the worker thread waiting on the LLM could not be interrupted, so the upstream request kept generating the whole answer for nobody. Make stream_chat_with_documents an async generator. All the blocking setup (ORM queries, the index read lock and its thread-bound sqlite connection, the query embedding, building the LLM client) moves into one sync _prepare_chat called once through sync_to_async, which also hands the DB connection back before the answer streams. The answer then streams through llama-index's asynthesize over the pinned async httpx clients, so a disconnect cancels the LLM request directly. The view returns the async generator as is, replacing the sync-to-async wrapper. The async pinned transport resolved hostnames with a blocking getaddrinfo on the event loop, which would stall the whole worker on slow DNS now that chat uses it in the web server. Resolve in a thread instead, and add the async counterparts of the existing transport tests.
This commit is contained in:
1 parent
a690885a8a
commit
179307d57a
6 files changed
+188
-158
No files matched your search
+5
-28
@@ -7,9 +7,7 @@ import tempfile
|
||||
import zipfile
|
||||
from collections import Counter
|
||||
from collections import deque
|
||||
from collections.abc import AsyncIterator
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Generator
|
||||
from datetime import datetime
|
||||
from datetime import timedelta
|
||||
from http import HTTPStatus
|
||||
@@ -29,7 +27,6 @@ from urllib.parse import urlparse
|
||||
import httpx
|
||||
import magic
|
||||
import pathvalidate
|
||||
from asgiref.sync import sync_to_async
|
||||
from django.conf import settings
|
||||
from django.contrib.auth.models import Group
|
||||
from django.contrib.auth.models import User
|
||||
@@ -2419,24 +2416,6 @@ class DocumentViewSet(
|
||||
)
|
||||
|
||||
|
||||
async def _iterate_in_sync_thread(
|
||||
chunks: Generator[str, None, None],
|
||||
) -> AsyncIterator[str]:
|
||||
"""Serve a blocking sync generator to an ASGI server chunk by chunk.
|
||||
|
||||
Under ASGI, Django drains a sync iterator with ``sync_to_async(list)``
|
||||
before sending anything. Pull each chunk in the request's sync thread
|
||||
instead, where the generator's ORM access belongs.
|
||||
"""
|
||||
sentinel = object() # StopIteration can't cross an asyncio Future
|
||||
next_chunk = sync_to_async(next)
|
||||
try:
|
||||
while (chunk := await next_chunk(chunks, sentinel)) is not sentinel:
|
||||
yield chunk
|
||||
finally:
|
||||
await sync_to_async(chunks.close)()
|
||||
|
||||
|
||||
class ChatStreamingSerializer(serializers.Serializer[dict[str, Any]]):
|
||||
q = serializers.CharField(required=True, max_length=4000)
|
||||
document_id = serializers.IntegerField(required=False, allow_null=True)
|
||||
@@ -2496,13 +2475,11 @@ class ChatStreamingView(GenericAPIView[Any]):
|
||||
)
|
||||
|
||||
return StreamingHttpResponse(
|
||||
_iterate_in_sync_thread(
|
||||
stream_chat_with_documents(
|
||||
query_str=question,
|
||||
documents=documents,
|
||||
unrestricted=unrestricted,
|
||||
output_language=output_language,
|
||||
),
|
||||
stream_chat_with_documents(
|
||||
query_str=question,
|
||||
documents=documents,
|
||||
unrestricted=unrestricted,
|
||||
output_language=output_language,
|
||||
),
|
||||
content_type="text/event-stream",
|
||||
)
|
||||
|
||||
Reference in new issue
Block a user