Retry celery ping and report warning on no response

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "paperless-ngx"
-version = "2.20.8"
+version = "2.20.7"
 description = "A community-supported supercharged document management system: scan, index and archive all your physical documents"
 readme = "README.md"
 requires-python = ">=3.10"

src-ui/package.json

@@ -1,6 +1,6 @@
 {
   "name": "paperless-ngx-ui",
-  "version": "2.20.8",
+  "version": "2.20.7",
   "scripts": {
     "preinstall": "npx only-allow pnpm",
     "ng": "ng",

src-ui/src/environments/environment.prod.ts

@@ -6,7 +6,7 @@ export const environment = {
   apiVersion: '9', // match src/paperless/settings.py
   appTitle: 'Paperless-ngx',
   tag: 'prod',
-  version: '2.20.8',
+  version: '2.20.7',
   webSocketHost: window.location.host,
   webSocketProtocol: window.location.protocol == 'https:' ? 'wss:' : 'ws:',
   webSocketBaseUrl: base_url.pathname + 'ws/',

src/documents/tests/test_api_status.py

@@ -151,6 +151,50 @@ class TestSystemStatus(APITestCase):
         self.assertEqual(response.status_code, status.HTTP_200_OK)
         self.assertEqual(response.data["tasks"]["celery_status"], "OK")
 
+    @mock.patch("celery.app.control.Inspect.ping")
+    def test_system_status_celery_ping_none(self, mock_ping) -> None:
+        """
+        GIVEN:
+            - Celery ping returns no worker responses
+        WHEN:
+            - The user requests the system status
+        THEN:
+            - The response contains an error celery status
+        """
+        mock_ping.return_value = None
+        self.client.force_login(self.user)
+        response = self.client.get(self.ENDPOINT)
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertEqual(response.data["tasks"]["celery_status"], "WARNING")
+        self.assertEqual(
+            response.data["tasks"]["celery_error"],
+            "No celery workers responded to ping. This may be temporary.",
+        )
+
+    @mock.patch("documents.views.sleep")
+    @mock.patch("celery.app.control.Inspect.ping")
+    def test_system_status_celery_ping_retry_success(
+        self,
+        mock_ping,
+        mock_sleep,
+    ) -> None:
+        """
+        GIVEN:
+            - Celery ping fails once but succeeds on retry
+        WHEN:
+            - The user requests the system status
+        THEN:
+            - The response contains an OK celery status
+        """
+        mock_ping.side_effect = [None, {"hostname": {"ok": "pong"}}]
+        self.client.force_login(self.user)
+        response = self.client.get(self.ENDPOINT)
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertEqual(response.data["tasks"]["celery_status"], "OK")
+        self.assertIsNone(response.data["tasks"]["celery_error"])
+        self.assertEqual(mock_ping.call_count, 2)
+        mock_sleep.assert_called_once_with(0.25)
+
     @override_settings(INDEX_DIR=Path("/tmp/index"))
     @mock.patch("whoosh.index.FileIndex.last_modified")
     def test_system_status_index_ok(self, mock_last_modified):

src/documents/views.py

@@ -10,6 +10,7 @@ from collections import deque
 from datetime import datetime
 from pathlib import Path
 from time import mktime
+from time import sleep
 from typing import Literal
 from unicodedata import normalize
 from urllib.parse import quote
@@ -3035,11 +3036,29 @@ class SystemStatusView(PassUserMixin):
         celery_error = None
         celery_url = None
         try:
-            celery_ping = celery_app.control.inspect().ping()
-            celery_url = next(iter(celery_ping.keys()))
-            first_worker_ping = celery_ping[celery_url]
-            if first_worker_ping["ok"] == "pong":
-                celery_active = "OK"
+            celery_ping = None
+            for ping_attempt in range(3):
+                celery_ping = celery_app.control.inspect().ping()
+                if celery_ping:
+                    break
+                if ping_attempt < 2:
+                    sleep(0.25)
+
+            if not celery_ping:
+                celery_active = "WARNING"
+                celery_error = (
+                    "No celery workers responded to ping. This may be temporary."
+                )
+            else:
+                celery_url, first_worker_ping = next(iter(celery_ping.items()))
+                if (
+                    isinstance(first_worker_ping, dict)
+                    and first_worker_ping.get("ok") == "pong"
+                ):
+                    celery_active = "OK"
+                else:
+                    celery_active = "WARNING"
+                    celery_error = "Celery worker responded unexpectedly."
         except Exception as e:
             celery_active = "ERROR"
             logger.exception(

src/paperless/version.py

@@ -1,6 +1,6 @@
 from typing import Final
 
-__version__: Final[tuple[int, int, int]] = (2, 20, 8)
+__version__: Final[tuple[int, int, int]] = (2, 20, 7)
 # Version string like X.Y.Z
 __full_version_str__: Final[str] = ".".join(map(str, __version__))
 # Version string like X.Y

src/paperless_mail/tests/test_api.py

@@ -272,24 +272,6 @@ class TestAPIMailAccounts(DirectoriesMixin, APITestCase):
         self.assertEqual(response.status_code, status.HTTP_200_OK)
         self.assertEqual(response.data["success"], True)
 
-    def test_mail_account_test_existing_nonexistent_id_forbidden(self):
-        response = self.client.post(
-            f"{self.ENDPOINT}test/",
-            json.dumps(
-                {
-                    "id": 999999,
-                    "imap_server": "server.example.com",
-                    "imap_port": 443,
-                    "imap_security": MailAccount.ImapSecurity.SSL,
-                    "username": "admin",
-                    "password": "******",
-                },
-            ),
-            content_type="application/json",
-        )
-        self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
-        self.assertEqual(response.content.decode(), "Insufficient permissions")
-
     def test_get_mail_accounts_owner_aware(self):
         """
         GIVEN:

src/paperless_mail/tests/test_mail.py

@@ -9,7 +9,6 @@ from datetime import timedelta
 from unittest import mock
 
 import pytest
-from django.contrib.auth.models import Permission
 from django.contrib.auth.models import User
 from django.core.management import call_command
 from django.db import DatabaseError
@@ -1700,10 +1699,6 @@ class TestMailAccountTestView(APITestCase):
             username="testuser",
             password="testpassword",
         )
-        self.user.user_permissions.add(
-            *Permission.objects.filter(codename__in=["add_mailaccount"]),
-        )
-        self.user.save()
         self.client.force_authenticate(user=self.user)
         self.url = "/api/mail_accounts/test/"
 
@@ -1820,54 +1815,6 @@ class TestMailAccountTestView(APITestCase):
             expected_str = "Unable to refresh oauth token"
             self.assertIn(expected_str, error_str)
 
-    def test_mail_account_test_view_existing_forbidden_for_other_owner(self):
-        other_user = User.objects.create_user(
-            username="otheruser",
-            password="testpassword",
-        )
-        existing_account = MailAccount.objects.create(
-            name="Owned account",
-            imap_server="imap.example.com",
-            imap_port=993,
-            imap_security=MailAccount.ImapSecurity.SSL,
-            username="admin",
-            password="secret",
-            owner=other_user,
-        )
-        data = {
-            "id": existing_account.id,
-            "imap_server": "imap.example.com",
-            "imap_port": 993,
-            "imap_security": MailAccount.ImapSecurity.SSL,
-            "username": "admin",
-            "password": "****",
-            "is_token": False,
-        }
-
-        response = self.client.post(self.url, data, format="json")
-
-        self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
-        self.assertEqual(response.content.decode(), "Insufficient permissions")
-
-    def test_mail_account_test_view_requires_add_permission_without_account_id(self):
-        self.user.user_permissions.remove(
-            *Permission.objects.filter(codename__in=["add_mailaccount"]),
-        )
-        self.user.save()
-        data = {
-            "imap_server": "imap.example.com",
-            "imap_port": 993,
-            "imap_security": MailAccount.ImapSecurity.SSL,
-            "username": "admin",
-            "password": "secret",
-            "is_token": False,
-        }
-
-        response = self.client.post(self.url, data, format="json")
-
-        self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
-        self.assertEqual(response.content.decode(), "Insufficient permissions")
-
 
 class TestMailAccountProcess(APITestCase):
     def setUp(self):

src/paperless_mail/views.py

@@ -86,34 +86,13 @@ class MailAccountViewSet(ModelViewSet, PassUserMixin):
         request.data["name"] = datetime.datetime.now().isoformat()
         serializer = self.get_serializer(data=request.data)
         serializer.is_valid(raise_exception=True)
-        existing_account = None
-        account_id = request.data.get("id")
 
-        # testing a new connection requires add permission
-        if account_id is None and not request.user.has_perms(
-            ["paperless_mail.add_mailaccount"],
-        ):
-            return HttpResponseForbidden("Insufficient permissions")
-
-        # testing an existing account requires change permission on that account
-        if account_id is not None:
-            try:
-                existing_account = MailAccount.objects.get(pk=account_id)
-            except (TypeError, ValueError, MailAccount.DoesNotExist):
-                return HttpResponseForbidden("Insufficient permissions")
-
-            if not has_perms_owner_aware(
-                request.user,
-                "change_mailaccount",
-                existing_account,
-            ):
-                return HttpResponseForbidden("Insufficient permissions")
-
-        # account exists, use the password from there instead of ***
+        # account exists, use the password from there instead of *** and refresh_token / expiration
         if (
             len(serializer.validated_data.get("password").replace("*", "")) == 0
-            and existing_account is not None
+            and request.data["id"] is not None
         ):
+            existing_account = MailAccount.objects.get(pk=request.data["id"])
             serializer.validated_data["password"] = existing_account.password
             serializer.validated_data["account_type"] = existing_account.account_type
             serializer.validated_data["refresh_token"] = existing_account.refresh_token
@@ -127,8 +106,7 @@ class MailAccountViewSet(ModelViewSet, PassUserMixin):
         ) as M:
             try:
                 if (
-                    existing_account is not None
-                    and account.is_token
+                    account.is_token
                     and account.expiration is not None
                     and account.expiration < timezone.now()
                 ):
@@ -270,7 +248,6 @@ class OauthCallbackView(GenericAPIView):
                 imap_server=imap_server,
                 refresh_token=refresh_token,
                 expiration=timezone.now() + timedelta(seconds=expires_in),
-                owner=request.user,
                 defaults=defaults,
             )
             return HttpResponseRedirect(

uv.lock

@@ -1991,7 +1991,7 @@ wheels = [
 
 [[package]]
 name = "paperless-ngx"
-version = "2.20.8"
+version = "2.20.7"
 source = { virtual = "." }
 dependencies = [
     { name = "babel", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },