Compare commits

..
Author SHA1 Message Date
shamoon abcdb68852 Tweak: adjust chat timing for streaming 2026-11-17 08:14:09 -08:00
stumpylog a690885a8a Fix: stream AI chat answers under ASGI 2026-11-17 07:42:51 -08:00
4 changed files with 197 additions and 20 deletions

No files matched your search

@@ -188,6 +188,62 @@ describe('ChatComponent', () => {
expect(component.loading()).toBe(false)
})
it('should reveal a complete answer within one second', () => {
component.input.set('Hello')
component.sendMessage()
const answer = 'A'.repeat(660)
mockStream$.next(answer)
mockStream$.complete()
expect(component.messages()[1].content.length).toBeGreaterThan(0)
expect(component.messages()[1].content.length).toBeLessThan(answer.length)
expect(component.loading()).toBe(true)
jest.advanceTimersByTime(800)
expect(component.messages()[1].content).toBe(answer)
expect(component.loading()).toBe(false)
})
it('should give a long answer more time to reveal', () => {
component.input.set('Hello')
component.sendMessage()
const answer = 'A'.repeat(3000)
mockStream$.next(answer)
mockStream$.complete()
jest.advanceTimersByTime(800)
expect(component.messages()[1].content.length).toBeLessThan(answer.length)
expect(component.loading()).toBe(true)
jest.advanceTimersByTime(2500)
expect(component.messages()[1].content).toBe(answer)
expect(component.loading()).toBe(false)
})
it('should reveal fast incoming chunks without making them pop in', () => {
component.input.set('Hello')
component.sendMessage()
let answer = ''
for (let i = 0; i < 20; i++) {
answer += 'A'.repeat(100)
const visibleBefore = component.messages()[1].content.length
mockStream$.next(answer)
jest.advanceTimersByTime(10)
expect(
component.messages()[1].content.length - visibleBefore
).toBeLessThan(100)
}
expect(component.messages()[1].content.length).toBeLessThan(answer.length)
mockStream$.complete()
jest.advanceTimersByTime(5200)
expect(component.messages()[1].content).toBe(answer)
expect(component.loading()).toBe(false)
})
it('should show a slow response message while waiting', () => {
component.input.set('Hello')
component.sendMessage()
@@ -21,6 +21,9 @@ import {
import { pngxPopperOptions } from 'src/app/utils/popper-options'
const SLOW_RESPONSE_DELAY_MS = 8000
const TYPEWRITER_MIN_DURATION_MS = 750
const TYPEWRITER_MAX_DURATION_MS = 3000
const TYPEWRITER_INTERVAL_MS = 16
@Component({
selector: 'pngx-chat',
@@ -73,6 +76,7 @@ export class ChatComponent implements OnInit {
private typewriterBuffer: string[] = []
private typewriterActive = false
private typewriterTimeout: ReturnType<typeof setTimeout>
private typewriterDeadline = 0
private streamSubscription: Subscription
private streamingMessage: ChatMessage
private streamFinished = false
@@ -134,7 +138,7 @@ export class ChatComponent implements OnInit {
if (nextResponse.content.length < lastVisibleContent.length) {
this.resetTypewriter(assistantMessage, nextResponse.content)
lastVisibleContent = nextResponse.content
} else {
} else if (nextResponse.content.length > lastVisibleContent.length) {
const visibleDelta = nextResponse.content.substring(
lastVisibleContent.length
)
@@ -214,6 +218,12 @@ export class ChatComponent implements OnInit {
if (!chunk) return
this.typewriterBuffer.push(...chunk.split(''))
// Longer bursts need more time to remain readable as the chat scrolls.
const revealDuration = Math.min(
TYPEWRITER_MAX_DURATION_MS,
Math.max(TYPEWRITER_MIN_DURATION_MS, this.typewriterBuffer.length)
)
this.typewriterDeadline = Date.now() + revealDuration
if (!this.typewriterActive) {
this.typewriterActive = true
@@ -222,19 +232,29 @@ export class ChatComponent implements OnInit {
}
playTypewriter(message: ChatMessage): void {
if (this.typewriterBuffer.length === 0) {
this.typewriterActive = false
if (this.streamFinished) this.finishStreaming(message)
return
}
const nextChar = this.typewriterBuffer.shift()
message.content += nextChar
// Spread the queued text across the updates left before the deadline
const remainingUpdates = Math.max(
1,
Math.ceil((this.typewriterDeadline - Date.now()) / TYPEWRITER_INTERVAL_MS)
)
const charactersToShow = Math.ceil(
this.typewriterBuffer.length / remainingUpdates
)
message.content += this.typewriterBuffer
.splice(0, charactersToShow)
.join('')
this.notifyMessagesChanged()
this.scrollToBottom()
// 10ms per character
this.typewriterTimeout = setTimeout(() => this.playTypewriter(message), 10)
if (this.typewriterBuffer.length === 0) {
this.typewriterActive = false
if (this.streamFinished) this.finishStreaming(message)
} else {
this.typewriterTimeout = setTimeout(
() => this.playTypewriter(message),
TYPEWRITER_INTERVAL_MS
)
}
}
private notifyMessagesChanged(): void {
+81 -2
View File
@@ -4,14 +4,18 @@ from typing import TYPE_CHECKING
from unittest import mock
import pytest
from asgiref.sync import async_to_sync
from rest_framework import status
from rest_framework.test import APIClient
from rest_framework.test import APITestCase
from documents.views import _iterate_in_sync_thread
from paperless_testing.factories import UserFactory
from paperless_testing.permissions import grant_global
if TYPE_CHECKING:
from collections.abc import Generator
from django.contrib.auth.models import User
from pytest_mock import MockerFixture
@@ -62,7 +66,7 @@ class TestChatStreamingViewInputValidation(APITestCase):
),
mock.patch(
"paperless_ai.chat.stream_chat_with_documents",
return_value=iter(chunks),
return_value=(c for c in chunks),
),
):
resp = self.client.post(
@@ -74,7 +78,11 @@ class TestChatStreamingViewInputValidation(APITestCase):
assert resp.status_code == status.HTTP_200_OK
assert not resp.has_header("Content-Encoding")
assert list(resp.streaming_content) == [c.encode() for c in chunks]
async def read_parts() -> list[bytes]:
return [part async for part in resp.streaming_content]
assert async_to_sync(read_parts)() == [c.encode() for c in chunks]
def test_missing_question_is_rejected(self) -> None:
with mock.patch(
@@ -155,3 +163,74 @@ class TestChatStreamingViewUnrestrictedFlag:
assert (
mocked_stream_chat.call_args.kwargs["unrestricted"] is expected_unrestricted
)
class TestIterateInSyncThread:
@pytest.fixture
def events(self) -> list[str]:
return []
@pytest.fixture
def chunks(self, events: list[str]) -> Generator[str, None, None]:
def generate() -> Generator[str, None, None]:
try:
for chunk in ("first", "second"):
events.append(f"produced {chunk}")
yield chunk
finally:
events.append("closed")
return generate()
def test_yields_each_chunk_before_pulling_the_next(
self,
chunks: Generator[str, None, None],
events: list[str],
) -> None:
"""
GIVEN:
- A sync generator wrapped for async iteration
WHEN:
- The wrapper is consumed
THEN:
- Each chunk is delivered before the next one is generated,
rather than the whole generator being drained up front
"""
async def consume() -> None:
async for chunk in _iterate_in_sync_thread(chunks):
events.append(f"received {chunk}")
async_to_sync(consume)()
assert events == [
"produced first",
"received first",
"produced second",
"received second",
"closed",
]
def test_closing_early_closes_the_source_generator(
self,
chunks: Generator[str, None, None],
events: list[str],
) -> None:
"""
GIVEN:
- A sync generator wrapped for async iteration
WHEN:
- The consumer closes the wrapper after the first chunk, as
happens when the client disconnects
THEN:
- The source generator is closed, so its cleanup runs
"""
async def consume_one() -> None:
wrapper = _iterate_in_sync_thread(chunks)
await anext(wrapper)
await wrapper.aclose()
async_to_sync(consume_one)()
assert events == ["produced first", "closed"]
+29 -7
View File
@@ -7,7 +7,9 @@ import tempfile
import zipfile
from collections import Counter
from collections import deque
from collections.abc import AsyncIterator
from collections.abc import Callable
from collections.abc import Generator
from datetime import datetime
from datetime import timedelta
from http import HTTPStatus
@@ -27,6 +29,7 @@ from urllib.parse import urlparse
import httpx
import magic
import pathvalidate
from asgiref.sync import sync_to_async
from django.conf import settings
from django.contrib.auth.models import Group
from django.contrib.auth.models import User
@@ -2416,6 +2419,24 @@ class DocumentViewSet(
)
async def _iterate_in_sync_thread(
chunks: Generator[str, None, None],
) -> AsyncIterator[str]:
"""Serve a blocking sync generator to an ASGI server chunk by chunk.
Under ASGI, Django drains a sync iterator with ``sync_to_async(list)``
before sending anything. Pull each chunk in the request's sync thread
instead, where the generator's ORM access belongs.
"""
sentinel = object() # StopIteration can't cross an asyncio Future
next_chunk = sync_to_async(next)
try:
while (chunk := await next_chunk(chunks, sentinel)) is not sentinel:
yield chunk
finally:
await sync_to_async(chunks.close)()
class ChatStreamingSerializer(serializers.Serializer[dict[str, Any]]):
q = serializers.CharField(required=True, max_length=4000)
document_id = serializers.IntegerField(required=False, allow_null=True)
@@ -2474,16 +2495,17 @@ class ChatStreamingView(GenericAPIView[Any]):
user=request.user,
)
response = StreamingHttpResponse(
stream_chat_with_documents(
query_str=question,
documents=documents,
unrestricted=unrestricted,
output_language=output_language,
return StreamingHttpResponse(
_iterate_in_sync_thread(
stream_chat_with_documents(
query_str=question,
documents=documents,
unrestricted=unrestricted,
output_language=output_language,
),
),
content_type="text/event-stream",
)
return response
@extend_schema_view(