diff --git a/src/documents/tasks.py b/src/documents/tasks.py index 182dbd2bd..2d9d6603c 100644 --- a/src/documents/tasks.py +++ b/src/documents/tasks.py @@ -713,8 +713,15 @@ def llmindex_index( ) -@shared_task -def apply_ai_suggestions(action_id: int, document_id: int) -> None: +@shared_task( + bind=True, + autoretry_for=(Exception,), + max_retries=3, + retry_backoff=60, + retry_backoff_max=600, + retry_jitter=True, +) +def apply_ai_suggestions(self, action_id: int, document_id: int) -> None: """ Deferred "apply AI suggestions" workflow action. """ diff --git a/src/documents/tests/test_workflows.py b/src/documents/tests/test_workflows.py index 04afbc6e3..a63ec6d5f 100644 --- a/src/documents/tests/test_workflows.py +++ b/src/documents/tests/test_workflows.py @@ -33,6 +33,7 @@ from documents.file_handling import generate_unique_filename from documents.signals.handlers import run_workflows from documents.workflows.ai import apply_ai_suggestions_to_document from documents.workflows.webhooks import send_webhook +from paperless_ai.exceptions import LLMTimeoutError if TYPE_CHECKING: from django.db.models import QuerySet @@ -5525,7 +5526,7 @@ class TestApplyAISuggestionsWorkflowAction( self.doc, ) - delay.assert_called_once_with(action.pk, self.doc.pk) + delay.assert_called_once_with(action_id=action.pk, document_id=self.doc.pk) def test_consumption_trigger_is_ignored(self) -> None: """ @@ -5605,15 +5606,15 @@ class TestApplyAISuggestionsWorkflowAction( self.assertEqual(changed, []) self.assertIn("AI is not enabled", "".join(cm.output)) - def test_llm_failure_leaves_document_untouched(self) -> None: + def test_invalid_configuration_leaves_document_untouched(self) -> None: """ GIVEN: - - An LLM backend that errors out + - An AI backend that is misconfigured WHEN: - The action is applied THEN: - - The failure is logged and the document is left alone, rather - than the error taking down the whole task + - The failure is logged and the document is left alone. It is not + re-raised, because retrying will not fix a bad configuration """ action = self.make_action() @@ -5629,7 +5630,31 @@ class TestApplyAISuggestionsWorkflowAction( self.assertEqual(changed, []) self.doc.refresh_from_db() self.assertEqual(self.doc.title, "original.pdf") - self.assertIn("Error getting AI suggestions", "".join(cm.output)) + self.assertIn("Invalid AI configuration", "".join(cm.output)) + + def test_transient_llm_failure_is_raised_for_retry(self) -> None: + """ + GIVEN: + - An LLM backend that times out, or rate limits the request + WHEN: + - The action is applied + THEN: + - The error propagates so the queued task can back off and retry, + rather than silently dropping this document's suggestions + """ + action = self.make_action() + + with ( + mock.patch( + "documents.workflows.ai.get_ai_document_classification", + side_effect=LLMTimeoutError(), + ), + self.assertRaises(LLMTimeoutError), + ): + apply_ai_suggestions_to_document(action, self.doc) + + self.doc.refresh_from_db() + self.assertEqual(self.doc.title, "original.pdf") def test_only_matching_objects_are_applied(self) -> None: """ diff --git a/src/documents/workflows/ai.py b/src/documents/workflows/ai.py index 0f9a384f6..f5be3a64b 100644 --- a/src/documents/workflows/ai.py +++ b/src/documents/workflows/ai.py @@ -140,9 +140,12 @@ def apply_ai_suggestions_to_document( owner, get_llm_output_language(ai_config, owner), ) - except Exception: + except ValueError: + # A bad AI config will not fix itself, so swallow it rather than + # letting the caller retry. Timeouts, rate limits, network errors etc + # propagate so the queued task can back off and try again. logger.exception( - "Error getting AI suggestions for document %s", + "Invalid AI configuration, cannot get suggestions for document %s", document.pk, extra={"group": logging_group}, )