mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-09-09 03:07:59 +00:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
069529203f | ||
|
|
0bd02c0b5c |
@@ -2,7 +2,6 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import logging
|
import logging
|
||||||
import tempfile
|
import tempfile
|
||||||
import uuid
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING
|
||||||
from typing import Literal
|
from typing import Literal
|
||||||
@@ -380,7 +379,7 @@ def delete(doc_ids: list[int]) -> Literal["OK"]:
|
|||||||
)
|
)
|
||||||
delete_ids = list({*doc_ids, *version_ids})
|
delete_ids = list({*doc_ids, *version_ids})
|
||||||
|
|
||||||
Document.objects.filter(id__in=delete_ids).delete(transaction_id=uuid.uuid4())
|
Document.objects.filter(id__in=delete_ids).delete()
|
||||||
|
|
||||||
from documents.search import get_backend
|
from documents.search import get_backend
|
||||||
|
|
||||||
|
|||||||
+2
-10
@@ -1,5 +1,4 @@
|
|||||||
import datetime
|
import datetime
|
||||||
import uuid
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Final
|
from typing import Final
|
||||||
|
|
||||||
@@ -515,20 +514,13 @@ class Document(SoftDeleteModel, ModelWithOwner): # type: ignore[django-manager-
|
|||||||
def delete(
|
def delete(
|
||||||
self,
|
self,
|
||||||
*args,
|
*args,
|
||||||
transaction_id=None,
|
|
||||||
**kwargs,
|
**kwargs,
|
||||||
):
|
):
|
||||||
# Versions must share the root's transaction ID so they are restored
|
# If deleting a root document, move all its versions to trash as well.
|
||||||
# together by django-softdelete.
|
|
||||||
if transaction_id is None:
|
|
||||||
transaction_id = uuid.uuid4()
|
|
||||||
if self.root_document_id is None:
|
if self.root_document_id is None:
|
||||||
Document.objects.filter(root_document=self).delete(
|
Document.objects.filter(root_document=self).delete()
|
||||||
transaction_id=transaction_id,
|
|
||||||
)
|
|
||||||
return super().delete(
|
return super().delete(
|
||||||
*args,
|
*args,
|
||||||
transaction_id=transaction_id,
|
|
||||||
**kwargs,
|
**kwargs,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -207,65 +207,3 @@ class TestTrashAPI(DirectoriesMixin, APITestCase):
|
|||||||
)
|
)
|
||||||
self.assertEqual(resp.status_code, status.HTTP_400_BAD_REQUEST)
|
self.assertEqual(resp.status_code, status.HTTP_400_BAD_REQUEST)
|
||||||
self.assertIn("have not yet been deleted", resp.data["documents"][0])
|
self.assertIn("have not yet been deleted", resp.data["documents"][0])
|
||||||
|
|
||||||
def _make_versioned_document(self) -> tuple[Document, list[Document]]:
|
|
||||||
root = Document.objects.create(
|
|
||||||
title="root",
|
|
||||||
content="root-content",
|
|
||||||
checksum="root",
|
|
||||||
mime_type="application/pdf",
|
|
||||||
)
|
|
||||||
versions = [
|
|
||||||
Document.objects.create(
|
|
||||||
title=f"v{index}",
|
|
||||||
content=f"v{index}-content",
|
|
||||||
checksum=f"v{index}",
|
|
||||||
mime_type="application/pdf",
|
|
||||||
root_document=root,
|
|
||||||
version_index=index,
|
|
||||||
)
|
|
||||||
for index in range(1, 3)
|
|
||||||
]
|
|
||||||
return root, versions
|
|
||||||
|
|
||||||
def test_api_trash_restore_document_restores_its_versions(self) -> None:
|
|
||||||
"""
|
|
||||||
GIVEN:
|
|
||||||
- Existing document with two versions
|
|
||||||
WHEN:
|
|
||||||
- API request to delete the document
|
|
||||||
- API request to restore it from the trash
|
|
||||||
THEN:
|
|
||||||
- Only the document itself is listed in the trash
|
|
||||||
- A version cannot be restored without its root
|
|
||||||
- The document is restored together with all of its versions
|
|
||||||
"""
|
|
||||||
root, versions = self._make_versioned_document()
|
|
||||||
|
|
||||||
self.client.force_login(user=self.user)
|
|
||||||
self.client.delete(f"/api/documents/{root.pk}/")
|
|
||||||
self.assertEqual(Document.deleted_objects.count(), 3)
|
|
||||||
|
|
||||||
resp = self.client.get("/api/trash/")
|
|
||||||
self.assertEqual(resp.status_code, status.HTTP_200_OK)
|
|
||||||
self.assertEqual(resp.data["count"], 1)
|
|
||||||
self.assertEqual(resp.data["results"][0]["id"], root.pk)
|
|
||||||
|
|
||||||
# A version cannot be restored while its root remains in the trash.
|
|
||||||
resp = self.client.post(
|
|
||||||
"/api/trash/",
|
|
||||||
{"action": "restore", "documents": [versions[0].pk]},
|
|
||||||
)
|
|
||||||
self.assertEqual(resp.status_code, status.HTTP_400_BAD_REQUEST)
|
|
||||||
self.assertIn("Restore the root document", resp.data["documents"][0])
|
|
||||||
|
|
||||||
resp = self.client.post(
|
|
||||||
"/api/trash/",
|
|
||||||
{"action": "restore", "documents": [root.pk]},
|
|
||||||
)
|
|
||||||
self.assertEqual(resp.status_code, status.HTTP_200_OK)
|
|
||||||
self.assertEqual(Document.deleted_objects.count(), 0)
|
|
||||||
self.assertCountEqual(
|
|
||||||
Document.objects.filter(root_document=root).values_list("id", flat=True),
|
|
||||||
[version.pk for version in versions],
|
|
||||||
)
|
|
||||||
|
|||||||
@@ -392,11 +392,6 @@ class TestBulkEdit(DirectoriesMixin, TestCase):
|
|||||||
self.assertFalse(Document.objects.filter(id=self.doc1.id).exists())
|
self.assertFalse(Document.objects.filter(id=self.doc1.id).exists())
|
||||||
self.assertFalse(Document.objects.filter(id=version.id).exists())
|
self.assertFalse(Document.objects.filter(id=version.id).exists())
|
||||||
|
|
||||||
Document.deleted_objects.get(id=self.doc1.id).restore(strict=False)
|
|
||||||
|
|
||||||
self.assertTrue(Document.objects.filter(id=self.doc1.id).exists())
|
|
||||||
self.assertTrue(Document.objects.filter(id=version.id).exists())
|
|
||||||
|
|
||||||
def test_delete_version_document_keeps_root(self) -> None:
|
def test_delete_version_document_keeps_root(self) -> None:
|
||||||
version = Document.objects.create(
|
version = Document.objects.create(
|
||||||
checksum="A-v1",
|
checksum="A-v1",
|
||||||
|
|||||||
@@ -110,7 +110,7 @@ class TestDocument(TestCase):
|
|||||||
checksum="checksum",
|
checksum="checksum",
|
||||||
mime_type="application/pdf",
|
mime_type="application/pdf",
|
||||||
)
|
)
|
||||||
version = Document.objects.create(
|
Document.objects.create(
|
||||||
root_document=root,
|
root_document=root,
|
||||||
correspondent=root.correspondent,
|
correspondent=root.correspondent,
|
||||||
title="Version",
|
title="Version",
|
||||||
@@ -124,10 +124,6 @@ class TestDocument(TestCase):
|
|||||||
self.assertEqual(Document.objects.count(), 0)
|
self.assertEqual(Document.objects.count(), 0)
|
||||||
self.assertEqual(Document.deleted_objects.count(), 2)
|
self.assertEqual(Document.deleted_objects.count(), 2)
|
||||||
|
|
||||||
root.restore(strict=False)
|
|
||||||
|
|
||||||
self.assertTrue(Document.objects.filter(pk=version.pk).exists())
|
|
||||||
|
|
||||||
def test_file_name(self) -> None:
|
def test_file_name(self) -> None:
|
||||||
doc = Document(
|
doc = Document(
|
||||||
mime_type="application/pdf",
|
mime_type="application/pdf",
|
||||||
|
|||||||
@@ -32,6 +32,7 @@ from documents.signals.handlers import update_llm_suggestions_cache
|
|||||||
from documents.tests.utils import DirectoriesMixin
|
from documents.tests.utils import DirectoriesMixin
|
||||||
from documents.tests.utils import read_streaming_response
|
from documents.tests.utils import read_streaming_response
|
||||||
from paperless.models import ApplicationConfiguration
|
from paperless.models import ApplicationConfiguration
|
||||||
|
from paperless_ai.exceptions import LLMProviderError
|
||||||
from paperless_ai.exceptions import LLMTimeoutError
|
from paperless_ai.exceptions import LLMTimeoutError
|
||||||
|
|
||||||
|
|
||||||
@@ -737,6 +738,38 @@ class TestAISuggestions(DirectoriesMixin, TestCase):
|
|||||||
get_llm_suggestion_cache(self.document.pk, backend="openai-like"),
|
get_llm_suggestion_cache(self.document.pk, backend="openai-like"),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@patch("documents.views.get_ai_document_classification")
|
||||||
|
@override_settings(
|
||||||
|
AI_ENABLED=True,
|
||||||
|
LLM_BACKEND="openai-like",
|
||||||
|
)
|
||||||
|
def test_ai_suggestions_with_llm_provider_error(
|
||||||
|
self,
|
||||||
|
mock_get_ai_classification,
|
||||||
|
) -> None:
|
||||||
|
mock_get_ai_classification.side_effect = LLMProviderError(
|
||||||
|
"confidential provider response",
|
||||||
|
)
|
||||||
|
|
||||||
|
self.client.force_login(user=self.user)
|
||||||
|
response = self.client.get(
|
||||||
|
f"/api/documents/{self.document.pk}/ai_suggestions/",
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(response.status_code, status.HTTP_502_BAD_GATEWAY)
|
||||||
|
self.assertEqual(
|
||||||
|
response.json(),
|
||||||
|
{
|
||||||
|
"ai": [
|
||||||
|
"AI backend rejected the request. Check logs for details.",
|
||||||
|
],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
self.assertNotIn("confidential provider response", response.content.decode())
|
||||||
|
self.assertIsNone(
|
||||||
|
get_llm_suggestion_cache(self.document.pk, backend="openai-like"),
|
||||||
|
)
|
||||||
|
|
||||||
@patch("documents.views.get_ai_document_classification")
|
@patch("documents.views.get_ai_document_classification")
|
||||||
@override_settings(
|
@override_settings(
|
||||||
AI_ENABLED=True,
|
AI_ENABLED=True,
|
||||||
|
|||||||
+19
-13
@@ -251,6 +251,7 @@ from paperless.views import StandardPagination
|
|||||||
from paperless_ai.ai_classifier import get_ai_document_classification
|
from paperless_ai.ai_classifier import get_ai_document_classification
|
||||||
from paperless_ai.ai_classifier import get_llm_output_language
|
from paperless_ai.ai_classifier import get_llm_output_language
|
||||||
from paperless_ai.chat import stream_chat_with_documents
|
from paperless_ai.chat import stream_chat_with_documents
|
||||||
|
from paperless_ai.exceptions import LLMProviderError
|
||||||
from paperless_ai.exceptions import LLMTimeoutError
|
from paperless_ai.exceptions import LLMTimeoutError
|
||||||
from paperless_ai.matching import extract_unmatched_names
|
from paperless_ai.matching import extract_unmatched_names
|
||||||
from paperless_ai.matching import match_correspondents_by_name
|
from paperless_ai.matching import match_correspondents_by_name
|
||||||
@@ -1602,6 +1603,22 @@ class DocumentViewSet(
|
|||||||
{"ai": [_("AI backend request timed out.")]},
|
{"ai": [_("AI backend request timed out.")]},
|
||||||
status=status.HTTP_503_SERVICE_UNAVAILABLE,
|
status=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||||
)
|
)
|
||||||
|
except LLMProviderError:
|
||||||
|
logger.exception(
|
||||||
|
"AI backend rejected the request for document %s",
|
||||||
|
doc.pk,
|
||||||
|
)
|
||||||
|
return Response(
|
||||||
|
{
|
||||||
|
"ai": [
|
||||||
|
_(
|
||||||
|
"AI backend rejected the request. "
|
||||||
|
"Check logs for details.",
|
||||||
|
),
|
||||||
|
],
|
||||||
|
},
|
||||||
|
status=status.HTTP_502_BAD_GATEWAY,
|
||||||
|
)
|
||||||
set_llm_suggestions_cache(
|
set_llm_suggestions_cache(
|
||||||
doc.pk,
|
doc.pk,
|
||||||
llm_suggestions,
|
llm_suggestions,
|
||||||
@@ -5431,10 +5448,7 @@ class TrashView(ListModelMixin, PassUserMixin):
|
|||||||
|
|
||||||
model = Document
|
model = Document
|
||||||
|
|
||||||
# A version is listed separately only when its root is not in the trash.
|
queryset = Document.deleted_objects.all()
|
||||||
queryset = Document.deleted_objects.exclude(
|
|
||||||
root_document_id__in=Document.deleted_objects.values("id"),
|
|
||||||
)
|
|
||||||
|
|
||||||
def get(self, request: Request, format: str | None = None) -> Response:
|
def get(self, request: Request, format: str | None = None) -> Response:
|
||||||
self.serializer_class = DocumentSerializer
|
self.serializer_class = DocumentSerializer
|
||||||
@@ -5465,15 +5479,7 @@ class TrashView(ListModelMixin, PassUserMixin):
|
|||||||
return HttpResponseForbidden("Insufficient permissions")
|
return HttpResponseForbidden("Insufficient permissions")
|
||||||
action = serializer.validated_data.get("action")
|
action = serializer.validated_data.get("action")
|
||||||
if action == "restore":
|
if action == "restore":
|
||||||
restored = list(self.get_queryset().filter(id__in=doc_ids))
|
restored = list(Document.deleted_objects.filter(id__in=doc_ids))
|
||||||
if len(restored) != len(doc_ids):
|
|
||||||
raise ValidationError(
|
|
||||||
{
|
|
||||||
"documents": [
|
|
||||||
"Restore the root document instead of one of its versions.",
|
|
||||||
],
|
|
||||||
},
|
|
||||||
)
|
|
||||||
for doc in restored:
|
for doc in restored:
|
||||||
doc.restore(strict=False)
|
doc.restore(strict=False)
|
||||||
if restored:
|
if restored:
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ from paperless.network import validate_outbound_http_url
|
|||||||
from paperless_ai.base_model import ClassificationSuggestions
|
from paperless_ai.base_model import ClassificationSuggestions
|
||||||
from paperless_ai.base_model import DocumentClassifierSchema
|
from paperless_ai.base_model import DocumentClassifierSchema
|
||||||
from paperless_ai.base_model import model_to_classification_suggestions
|
from paperless_ai.base_model import model_to_classification_suggestions
|
||||||
|
from paperless_ai.exceptions import LLMProviderError
|
||||||
from paperless_ai.exceptions import LLMTimeoutError
|
from paperless_ai.exceptions import LLMTimeoutError
|
||||||
|
|
||||||
logger = logging.getLogger("paperless_ai.client")
|
logger = logging.getLogger("paperless_ai.client")
|
||||||
@@ -132,7 +133,7 @@ class AIClient:
|
|||||||
from llama_index.core.llms import ChatMessage
|
from llama_index.core.llms import ChatMessage
|
||||||
|
|
||||||
if self.settings.llm_backend == LLMBackend.OLLAMA:
|
if self.settings.llm_backend == LLMBackend.OLLAMA:
|
||||||
with self._normalize_timeouts():
|
with self._normalize_errors():
|
||||||
result = self.llm.chat(
|
result = self.llm.chat(
|
||||||
[ChatMessage(role="user", content=prompt)],
|
[ChatMessage(role="user", content=prompt)],
|
||||||
format=DocumentClassifierSchema.model_json_schema(),
|
format=DocumentClassifierSchema.model_json_schema(),
|
||||||
@@ -153,7 +154,7 @@ class AIClient:
|
|||||||
content=f"{prompt}\n\n"
|
content=f"{prompt}\n\n"
|
||||||
f"Answer by calling the {tool.metadata.name} tool. Do not write the answer as text.",
|
f"Answer by calling the {tool.metadata.name} tool. Do not write the answer as text.",
|
||||||
)
|
)
|
||||||
with self._normalize_timeouts():
|
with self._normalize_errors():
|
||||||
result = self.llm.chat_with_tools(
|
result = self.llm.chat_with_tools(
|
||||||
tools=[tool],
|
tools=[tool],
|
||||||
user_msg=user_msg,
|
user_msg=user_msg,
|
||||||
@@ -173,7 +174,7 @@ class AIClient:
|
|||||||
)
|
)
|
||||||
|
|
||||||
@contextmanager
|
@contextmanager
|
||||||
def _normalize_timeouts(self) -> Iterator[None]:
|
def _normalize_errors(self) -> Iterator[None]:
|
||||||
try:
|
try:
|
||||||
yield
|
yield
|
||||||
except httpx.TimeoutException as exc:
|
except httpx.TimeoutException as exc:
|
||||||
@@ -181,8 +182,23 @@ class AIClient:
|
|||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
if self._is_openai_timeout(exc):
|
if self._is_openai_timeout(exc):
|
||||||
raise LLMTimeoutError from exc
|
raise LLMTimeoutError from exc
|
||||||
|
if self._is_provider_error(exc):
|
||||||
|
raise LLMProviderError from exc
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
def _is_provider_error(self, exc: Exception) -> bool:
|
||||||
|
if self.settings.llm_backend == LLMBackend.OLLAMA:
|
||||||
|
from ollama import ResponseError
|
||||||
|
|
||||||
|
return isinstance(exc, ResponseError)
|
||||||
|
|
||||||
|
if self.settings.llm_backend == LLMBackend.OPENAI_LIKE:
|
||||||
|
from openai import APIStatusError
|
||||||
|
|
||||||
|
return isinstance(exc, APIStatusError)
|
||||||
|
|
||||||
|
return False
|
||||||
|
|
||||||
def _is_openai_timeout(self, exc: Exception) -> bool:
|
def _is_openai_timeout(self, exc: Exception) -> bool:
|
||||||
if self.settings.llm_backend != LLMBackend.OPENAI_LIKE:
|
if self.settings.llm_backend != LLMBackend.OPENAI_LIKE:
|
||||||
return False
|
return False
|
||||||
|
|||||||
@@ -1,2 +1,6 @@
|
|||||||
class LLMTimeoutError(Exception):
|
class LLMTimeoutError(Exception):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class LLMProviderError(Exception):
|
||||||
|
"""The LLM backend rejected the request."""
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ from unittest.mock import MagicMock
|
|||||||
from unittest.mock import patch
|
from unittest.mock import patch
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
|
import ollama
|
||||||
import openai
|
import openai
|
||||||
import pytest
|
import pytest
|
||||||
from llama_index.core.llms.llm import ToolSelection
|
from llama_index.core.llms.llm import ToolSelection
|
||||||
@@ -11,6 +12,7 @@ from llama_index.core.llms.llm import ToolSelection
|
|||||||
from paperless_ai.client import LLM_SYSTEM_PROMPT
|
from paperless_ai.client import LLM_SYSTEM_PROMPT
|
||||||
from paperless_ai.client import PLACEHOLDER_API_KEY
|
from paperless_ai.client import PLACEHOLDER_API_KEY
|
||||||
from paperless_ai.client import AIClient
|
from paperless_ai.client import AIClient
|
||||||
|
from paperless_ai.exceptions import LLMProviderError
|
||||||
from paperless_ai.exceptions import LLMTimeoutError
|
from paperless_ai.exceptions import LLMTimeoutError
|
||||||
|
|
||||||
|
|
||||||
@@ -214,6 +216,52 @@ def test_run_llm_query_openai_timeout_raises_local_error(
|
|||||||
client.run_llm_query("test_prompt")
|
client.run_llm_query("test_prompt")
|
||||||
|
|
||||||
|
|
||||||
|
def test_run_llm_query_openai_status_error_raises_provider_error(
|
||||||
|
mock_ai_config,
|
||||||
|
mock_openai_llm,
|
||||||
|
):
|
||||||
|
mock_ai_config.llm_backend = "openai-like"
|
||||||
|
mock_ai_config.llm_model = "test_model"
|
||||||
|
mock_ai_config.llm_endpoint = "http://test-url"
|
||||||
|
|
||||||
|
request = httpx.Request("POST", "http://test-url/v1/chat/completions")
|
||||||
|
body = {"error": {"message": "Thinking mode does not support this tool_choice"}}
|
||||||
|
mock_openai_llm.return_value.chat_with_tools.side_effect = openai.BadRequestError(
|
||||||
|
"Error code: 400",
|
||||||
|
response=httpx.Response(400, request=request, json=body),
|
||||||
|
body=body,
|
||||||
|
)
|
||||||
|
|
||||||
|
client = AIClient()
|
||||||
|
|
||||||
|
with pytest.raises(LLMProviderError) as exc_info:
|
||||||
|
client.run_llm_query("test_prompt")
|
||||||
|
assert str(exc_info.value) == ""
|
||||||
|
assert isinstance(exc_info.value.__cause__, openai.BadRequestError)
|
||||||
|
|
||||||
|
|
||||||
|
def test_run_llm_query_ollama_response_error_raises_provider_error(
|
||||||
|
mock_ai_config,
|
||||||
|
mock_ollama_llm,
|
||||||
|
):
|
||||||
|
mock_ai_config.llm_backend = "ollama"
|
||||||
|
mock_ai_config.llm_model = "test_model"
|
||||||
|
mock_ai_config.llm_endpoint = "http://test-url"
|
||||||
|
|
||||||
|
response_error = ollama.ResponseError(
|
||||||
|
"confidential provider response",
|
||||||
|
status_code=400,
|
||||||
|
)
|
||||||
|
mock_ollama_llm.return_value.chat.side_effect = response_error
|
||||||
|
|
||||||
|
client = AIClient()
|
||||||
|
|
||||||
|
with pytest.raises(LLMProviderError) as exc_info:
|
||||||
|
client.run_llm_query("test_prompt")
|
||||||
|
assert str(exc_info.value) == ""
|
||||||
|
assert exc_info.value.__cause__ is response_error
|
||||||
|
|
||||||
|
|
||||||
def test_run_llm_query_httpx_timeout_raises_local_error(
|
def test_run_llm_query_httpx_timeout_raises_local_error(
|
||||||
mock_ai_config,
|
mock_ai_config,
|
||||||
mock_ollama_llm,
|
mock_ollama_llm,
|
||||||
|
|||||||
Reference in New Issue
Block a user