Compare commits

..
Author SHA1 Message Date
shamoon 069529203f ollama too 2026-09-07 21:23:32 -07:00
shamoon 0bd02c0b5c Fix: better LLM errors 2026-09-07 21:21:00 -07:00
7 changed files with 124 additions and 27 deletions
+1 -1
View File
@@ -72,7 +72,7 @@ jobs:
'You are welcome to open a new issue that describes the problem you observed in your own words.'
: 'This issue was automatically closed because it was not opened using our bug report form. ' +
'Issues have to be created through the form so that the details we need to investigate are included.\n\n' +
`If the problem is still there, please [open a new issue](${newIssue}) using the form. No other action is needed here.\n\n` +
`If the problem is still there, please [open a new issue](${newIssue}) using the form. No other action is needed here.\n\n' +
'If any part of your report was written by an AI tool or agent, you must say so: undisclosed AI-generated ' +
`contributions are a violation of our [Code of Conduct](${codeOfConduct}).`;
+2 -23
View File
@@ -25,10 +25,6 @@ jobs:
pr-bot:
name: Automated PR Bot
runs-on: ubuntu-latest
# Runs after Anti-slop so the welcome comment can see whether the PR was closed
# instead of racing it. Still runs if that job fails, so labeling is not lost.
needs: Anti-slop
if: ${{ !cancelled() }}
permissions:
contents: read
pull-requests: write
@@ -103,25 +99,8 @@ jobs:
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
script: |
const user = context.payload.pull_request.user.login;
// Re-read the PR: Anti-slop may have closed and labeled it after the webhook
const { data: pr } = await github.rest.pulls.get({
owner: context.repo.owner,
repo: context.repo.repo,
pull_number: context.payload.pull_request.number,
});
if (pr.state === 'closed') {
core.info('Skipping comment: PR is already closed');
return;
}
const labels = pr.labels.map((label) => (typeof label === 'string' ? label : label.name));
if (labels.includes('ai')) {
core.info('Skipping comment: PR is labeled ai');
return;
}
const pr = context.payload.pull_request;
const user = pr.user.login;
const { data: members } = await github.rest.orgs.listMembers({
org: 'paperless-ngx',
+33
View File
@@ -32,6 +32,7 @@ from documents.signals.handlers import update_llm_suggestions_cache
from documents.tests.utils import DirectoriesMixin
from documents.tests.utils import read_streaming_response
from paperless.models import ApplicationConfiguration
from paperless_ai.exceptions import LLMProviderError
from paperless_ai.exceptions import LLMTimeoutError
@@ -737,6 +738,38 @@ class TestAISuggestions(DirectoriesMixin, TestCase):
get_llm_suggestion_cache(self.document.pk, backend="openai-like"),
)
@patch("documents.views.get_ai_document_classification")
@override_settings(
AI_ENABLED=True,
LLM_BACKEND="openai-like",
)
def test_ai_suggestions_with_llm_provider_error(
self,
mock_get_ai_classification,
) -> None:
mock_get_ai_classification.side_effect = LLMProviderError(
"confidential provider response",
)
self.client.force_login(user=self.user)
response = self.client.get(
f"/api/documents/{self.document.pk}/ai_suggestions/",
)
self.assertEqual(response.status_code, status.HTTP_502_BAD_GATEWAY)
self.assertEqual(
response.json(),
{
"ai": [
"AI backend rejected the request. Check logs for details.",
],
},
)
self.assertNotIn("confidential provider response", response.content.decode())
self.assertIsNone(
get_llm_suggestion_cache(self.document.pk, backend="openai-like"),
)
@patch("documents.views.get_ai_document_classification")
@override_settings(
AI_ENABLED=True,
+17
View File
@@ -251,6 +251,7 @@ from paperless.views import StandardPagination
from paperless_ai.ai_classifier import get_ai_document_classification
from paperless_ai.ai_classifier import get_llm_output_language
from paperless_ai.chat import stream_chat_with_documents
from paperless_ai.exceptions import LLMProviderError
from paperless_ai.exceptions import LLMTimeoutError
from paperless_ai.matching import extract_unmatched_names
from paperless_ai.matching import match_correspondents_by_name
@@ -1602,6 +1603,22 @@ class DocumentViewSet(
{"ai": [_("AI backend request timed out.")]},
status=status.HTTP_503_SERVICE_UNAVAILABLE,
)
except LLMProviderError:
logger.exception(
"AI backend rejected the request for document %s",
doc.pk,
)
return Response(
{
"ai": [
_(
"AI backend rejected the request. "
"Check logs for details.",
),
],
},
status=status.HTTP_502_BAD_GATEWAY,
)
set_llm_suggestions_cache(
doc.pk,
llm_suggestions,
+19 -3
View File
@@ -22,6 +22,7 @@ from paperless.network import validate_outbound_http_url
from paperless_ai.base_model import ClassificationSuggestions
from paperless_ai.base_model import DocumentClassifierSchema
from paperless_ai.base_model import model_to_classification_suggestions
from paperless_ai.exceptions import LLMProviderError
from paperless_ai.exceptions import LLMTimeoutError
logger = logging.getLogger("paperless_ai.client")
@@ -132,7 +133,7 @@ class AIClient:
from llama_index.core.llms import ChatMessage
if self.settings.llm_backend == LLMBackend.OLLAMA:
with self._normalize_timeouts():
with self._normalize_errors():
result = self.llm.chat(
[ChatMessage(role="user", content=prompt)],
format=DocumentClassifierSchema.model_json_schema(),
@@ -153,7 +154,7 @@ class AIClient:
content=f"{prompt}\n\n"
f"Answer by calling the {tool.metadata.name} tool. Do not write the answer as text.",
)
with self._normalize_timeouts():
with self._normalize_errors():
result = self.llm.chat_with_tools(
tools=[tool],
user_msg=user_msg,
@@ -173,7 +174,7 @@ class AIClient:
)
@contextmanager
def _normalize_timeouts(self) -> Iterator[None]:
def _normalize_errors(self) -> Iterator[None]:
try:
yield
except httpx.TimeoutException as exc:
@@ -181,8 +182,23 @@ class AIClient:
except Exception as exc:
if self._is_openai_timeout(exc):
raise LLMTimeoutError from exc
if self._is_provider_error(exc):
raise LLMProviderError from exc
raise
def _is_provider_error(self, exc: Exception) -> bool:
if self.settings.llm_backend == LLMBackend.OLLAMA:
from ollama import ResponseError
return isinstance(exc, ResponseError)
if self.settings.llm_backend == LLMBackend.OPENAI_LIKE:
from openai import APIStatusError
return isinstance(exc, APIStatusError)
return False
def _is_openai_timeout(self, exc: Exception) -> bool:
if self.settings.llm_backend != LLMBackend.OPENAI_LIKE:
return False
+4
View File
@@ -1,2 +1,6 @@
class LLMTimeoutError(Exception):
pass
class LLMProviderError(Exception):
"""The LLM backend rejected the request."""
+48
View File
@@ -4,6 +4,7 @@ from unittest.mock import MagicMock
from unittest.mock import patch
import httpx
import ollama
import openai
import pytest
from llama_index.core.llms.llm import ToolSelection
@@ -11,6 +12,7 @@ from llama_index.core.llms.llm import ToolSelection
from paperless_ai.client import LLM_SYSTEM_PROMPT
from paperless_ai.client import PLACEHOLDER_API_KEY
from paperless_ai.client import AIClient
from paperless_ai.exceptions import LLMProviderError
from paperless_ai.exceptions import LLMTimeoutError
@@ -214,6 +216,52 @@ def test_run_llm_query_openai_timeout_raises_local_error(
client.run_llm_query("test_prompt")
def test_run_llm_query_openai_status_error_raises_provider_error(
mock_ai_config,
mock_openai_llm,
):
mock_ai_config.llm_backend = "openai-like"
mock_ai_config.llm_model = "test_model"
mock_ai_config.llm_endpoint = "http://test-url"
request = httpx.Request("POST", "http://test-url/v1/chat/completions")
body = {"error": {"message": "Thinking mode does not support this tool_choice"}}
mock_openai_llm.return_value.chat_with_tools.side_effect = openai.BadRequestError(
"Error code: 400",
response=httpx.Response(400, request=request, json=body),
body=body,
)
client = AIClient()
with pytest.raises(LLMProviderError) as exc_info:
client.run_llm_query("test_prompt")
assert str(exc_info.value) == ""
assert isinstance(exc_info.value.__cause__, openai.BadRequestError)
def test_run_llm_query_ollama_response_error_raises_provider_error(
mock_ai_config,
mock_ollama_llm,
):
mock_ai_config.llm_backend = "ollama"
mock_ai_config.llm_model = "test_model"
mock_ai_config.llm_endpoint = "http://test-url"
response_error = ollama.ResponseError(
"confidential provider response",
status_code=400,
)
mock_ollama_llm.return_value.chat.side_effect = response_error
client = AIClient()
with pytest.raises(LLMProviderError) as exc_info:
client.run_llm_query("test_prompt")
assert str(exc_info.value) == ""
assert exc_info.value.__cause__ is response_error
def test_run_llm_query_httpx_timeout_raises_local_error(
mock_ai_config,
mock_ollama_llm,