Use locmemcache for ci

Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>

docs/changelog.md

@@ -1,5 +1,41 @@
 # Changelog
 
+## paperless-ngx 3.1.1
+
+### Bug Fixes
+
+- Fix: 3.1.0 llm suggestions remove existing metadata from prompt, dont drop name suggestions [@shamoon](https://github.com/shamoon) ([#13866](https://github.com/paperless-ngx/paperless-ngx/pull/13866))
+- Fix: set global search earlier to avoid awaiting debounce [@shamoon](https://github.com/shamoon) ([#13865](https://github.com/paperless-ngx/paperless-ngx/pull/13865))
+- Fix: responsive sidebar, centralize and make sizes saner [@shamoon](https://github.com/shamoon) ([#13863](https://github.com/paperless-ngx/paperless-ngx/pull/13863))
+- Tweak/fix: show existing count for ai suggestions [@shamoon](https://github.com/shamoon) ([#13861](https://github.com/paperless-ngx/paperless-ngx/pull/13861))
+- Fix: 3.1.0 llm suggestions simplify schema, fix docstrings [@shamoon](https://github.com/shamoon) ([#13850](https://github.com/paperless-ngx/paperless-ngx/pull/13850))
+- Fix: ensure ui reset of suggestionsLoading when changing docs [@shamoon](https://github.com/shamoon) ([#13840](https://github.com/paperless-ngx/paperless-ngx/pull/13840))
+- Fix: always pass a non-empty api key for OpenAI-like servers [@shamoon](https://github.com/shamoon) ([#13838](https://github.com/paperless-ngx/paperless-ngx/pull/13838))
+- Fix: hide slim sidebar scrollbar in browsers with stupid scrollbars [@shamoon](https://github.com/shamoon) ([#13837](https://github.com/paperless-ngx/paperless-ngx/pull/13837))
+- Fix: correct sharelink bundle + document link permissions display bugs [@shamoon](https://github.com/shamoon) ([#13827](https://github.com/paperless-ngx/paperless-ngx/pull/13827))
+- Fix: immediately re-add doc to index after trash restore [@shamoon](https://github.com/shamoon) ([#13818](https://github.com/paperless-ngx/paperless-ngx/pull/13818))
+- Fix: navbar brand anchor size + Safari position jitter [@shamoon](https://github.com/shamoon) ([#13810](https://github.com/paperless-ngx/paperless-ngx/pull/13810))
+
+### All App Changes
+
+<details>
+<summary>12 changes</summary>
+
+- Fix: 3.1.0 llm suggestions remove existing metadata from prompt, dont drop name suggestions [@shamoon](https://github.com/shamoon) ([#13866](https://github.com/paperless-ngx/paperless-ngx/pull/13866))
+- Fix: set global search earlier to avoid awaiting debounce [@shamoon](https://github.com/shamoon) ([#13865](https://github.com/paperless-ngx/paperless-ngx/pull/13865))
+- Fix: responsive sidebar, centralize and make sizes saner [@shamoon](https://github.com/shamoon) ([#13863](https://github.com/paperless-ngx/paperless-ngx/pull/13863))
+- Tweak/fix: show existing count for ai suggestions [@shamoon](https://github.com/shamoon) ([#13861](https://github.com/paperless-ngx/paperless-ngx/pull/13861))
+- Fix: 3.1.0 llm suggestions simplify schema, fix docstrings [@shamoon](https://github.com/shamoon) ([#13850](https://github.com/paperless-ngx/paperless-ngx/pull/13850))
+- Fixhancement: make imap port required, better error display [@shamoon](https://github.com/shamoon) ([#13845](https://github.com/paperless-ngx/paperless-ngx/pull/13845))
+- Fix: ensure ui reset of suggestionsLoading when changing docs [@shamoon](https://github.com/shamoon) ([#13840](https://github.com/paperless-ngx/paperless-ngx/pull/13840))
+- Fix: always pass a non-empty api key for OpenAI-like servers [@shamoon](https://github.com/shamoon) ([#13838](https://github.com/paperless-ngx/paperless-ngx/pull/13838))
+- Fix: hide slim sidebar scrollbar in browsers with stupid scrollbars [@shamoon](https://github.com/shamoon) ([#13837](https://github.com/paperless-ngx/paperless-ngx/pull/13837))
+- Fix: correct sharelink bundle + document link permissions display bugs [@shamoon](https://github.com/shamoon) ([#13827](https://github.com/paperless-ngx/paperless-ngx/pull/13827))
+- Fix: immediately re-add doc to index after trash restore [@shamoon](https://github.com/shamoon) ([#13818](https://github.com/paperless-ngx/paperless-ngx/pull/13818))
+- Fix: navbar brand anchor size + Safari position jitter [@shamoon](https://github.com/shamoon) ([#13810](https://github.com/paperless-ngx/paperless-ngx/pull/13810))
+
+</details>
+
 ## paperless-ngx 3.1.0
 
 ### Features / Enhancements

src/documents/caching.py

@@ -3,13 +3,16 @@ from __future__ import annotations
 import hashlib
 import logging
 import pickle
+import time
 import uuid
 from binascii import hexlify
 from collections import OrderedDict
 from dataclasses import dataclass
+from hashlib import sha256
 from typing import TYPE_CHECKING
 from typing import Any
 from typing import Final
+from uuid import uuid4
 
 from django.conf import settings
 from django.core.cache import cache
@@ -18,6 +21,7 @@ from django.core.cache import caches
 from documents.models import Document
 
 if TYPE_CHECKING:
+    from django.contrib.auth.models import User
     from django.core.cache.backends.base import BaseCache
 
     from documents.classifier import DocumentClassifier
@@ -56,6 +60,9 @@ CLASSIFIER_MODIFIED_KEY: Final[str] = "classifier_modified"
 #          validated separately, so candidate-anchored 1001 results are stale
 LLM_CACHE_CLASSIFIER_VERSION: Final[int] = 1002
 
+# How often a request waiting on llm generation re-checks the cache
+LLM_SUGGESTION_POLL_INTERVAL: Final[float] = 0.5
+
 CACHE_1_MINUTE: Final[int] = 60
 CACHE_5_MINUTES: Final[int] = 5 * CACHE_1_MINUTE
 CACHE_50_MINUTES: Final[int] = 50 * CACHE_1_MINUTE
@@ -257,6 +264,68 @@ def get_llm_suggestion_cache(
     return None
 
 
+def retrieve_llm_suggestions(
+    document: Document,
+    user: User | None,
+    output_language: str | None,
+    *,
+    backend: str,
+    lock_timeout: int,
+) -> dict:
+    """Return cached LLM suggestions, generating them once across workers."""
+    # Lazy import to avoid pulling in the whole AI stuff
+    from paperless_ai.ai_classifier import get_ai_document_classification
+    from paperless_ai.exceptions import LLMTimeoutError
+
+    lock_key = (
+        f"{get_suggestion_cache_key(document.pk)}_llm_lock_"
+        f"{sha256(backend.encode()).hexdigest()}"
+    )
+    waited = False
+
+    while True:
+        cached = get_llm_suggestion_cache(document.pk, backend=backend)
+        if cached is not None:
+            refresh_suggestions_cache(document.pk)
+            return cached.suggestions
+
+        lock_token = uuid4().hex
+        if cache.add(lock_key, lock_token, lock_timeout):
+            if waited:
+                # The generation we were waiting on has ended without caching
+                # anything so it either failed or outlived its lock. Give up
+                # rather than re-running it
+                cache.delete(lock_key)
+                raise LLMTimeoutError
+
+            try:
+                # The cache may have been populated while acquiring the lock.
+                cached = get_llm_suggestion_cache(document.pk, backend=backend)
+                if cached is not None:
+                    refresh_suggestions_cache(document.pk)
+                    return cached.suggestions
+
+                suggestions = get_ai_document_classification(
+                    document,
+                    user,
+                    output_language,
+                )
+                set_llm_suggestions_cache(
+                    document.pk,
+                    suggestions,
+                    backend=backend,
+                )
+                return suggestions
+            finally:
+                # Don't remove lock if this one expired while generation was still running
+                if cache.get(lock_key) == lock_token:
+                    cache.delete(lock_key)
+
+        waited = True
+        # Another worker is generating suggestions, poll to avoid another LLM request
+        time.sleep(LLM_SUGGESTION_POLL_INTERVAL)
+
+
 def set_llm_suggestions_cache(
     document_id: int,
     suggestions: dict,

src/documents/tests/test_api_documents.py

@@ -2486,7 +2486,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         response = self.client.get("/api/documents/34676/suggestions/")
         self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
 
-    @mock.patch("documents.views.get_ai_document_classification")
+    @mock.patch("paperless_ai.ai_classifier.get_ai_document_classification")
     @override_settings(AI_ENABLED=True)
     def test_suggestions_still_uses_classifier_when_ai_enabled(
         self,

src/documents/tests/test_api_trash.py

@@ -7,9 +7,10 @@ from rest_framework import status
 from rest_framework.test import APITestCase
 
 from documents.models import Document
+from documents.tests.utils import DirectoriesMixin
 
 
-class TestTrashAPI(APITestCase):
+class TestTrashAPI(DirectoriesMixin, APITestCase):
     def setUp(self) -> None:
         super().setUp()
 

src/documents/tests/test_caching.py

@@ -1,6 +1,15 @@
 import pickle
+from concurrent.futures import ThreadPoolExecutor
+from threading import Event
+from threading import Lock
+from uuid import uuid4
+
+import pytest
+from django.core.cache.backends.locmem import LocMemCache
 
 from documents.caching import StoredLRUCache
+from documents.caching import retrieve_llm_suggestions
+from paperless_ai.exceptions import LLMTimeoutError
 
 
 def test_lru_cache_entries() -> None:
@@ -43,3 +52,131 @@ def test_stored_lru_cache_key_ttl(mocker) -> None:
     assert key == "test_key"
     assert timeout == 321
     assert pickle.loads(data) == {"x": "X", "y": "Y"}
+
+
+def test_llm_suggestions_are_generated_once_for_concurrent_requests(mocker) -> None:
+    mocker.patch(
+        "documents.caching.cache",
+        LocMemCache(uuid4().hex, {}),
+    )
+    generation_started = Event()
+    finish_generation = Event()
+    waiter_started = Event()
+    release_waiter = Event()
+    call_lock = Lock()
+    calls = 0
+    suggestions = {"title": "Generated once"}
+    document = mocker.Mock(pk=42)
+    user = mocker.Mock()
+
+    def generate(*args) -> dict:
+        nonlocal calls
+        with call_lock:
+            calls += 1
+        generation_started.set()
+        assert finish_generation.wait(timeout=2)
+        return suggestions
+
+    def wait_for_generation(_interval: float) -> None:
+        waiter_started.set()
+        assert release_waiter.wait(timeout=2)
+
+    mock_get_classification = mocker.patch(
+        "paperless_ai.ai_classifier.get_ai_document_classification",
+        side_effect=generate,
+    )
+    mocker.patch("documents.caching.time.sleep", side_effect=wait_for_generation)
+
+    with ThreadPoolExecutor(max_workers=2) as executor:
+        first = executor.submit(
+            retrieve_llm_suggestions,
+            document,
+            user,
+            None,
+            backend="ollama:model",
+            lock_timeout=10,
+        )
+        assert generation_started.wait(timeout=2)
+        second = executor.submit(
+            retrieve_llm_suggestions,
+            document,
+            user,
+            None,
+            backend="ollama:model",
+            lock_timeout=10,
+        )
+        assert waiter_started.wait(timeout=2)
+        finish_generation.set()
+
+        assert first.result(timeout=2) == suggestions
+        release_waiter.set()
+        assert second.result(timeout=2) == suggestions
+
+    assert calls == 1
+    mock_get_classification.assert_called_once_with(document, user, None)
+
+
+def test_llm_suggestions_waiter_does_not_rerun_a_failed_generation(mocker) -> None:
+    """
+    A request queued behind a generation that fails should give up, not take
+    its turn at re-running a query that just failed.
+    """
+    mocker.patch(
+        "documents.caching.cache",
+        LocMemCache(uuid4().hex, {}),
+    )
+    generation_started = Event()
+    fail_generation = Event()
+    waiter_started = Event()
+    release_waiter = Event()
+    call_lock = Lock()
+    calls = 0
+    document = mocker.Mock(pk=43)
+    user = mocker.Mock()
+
+    def generate(*args) -> dict:
+        nonlocal calls
+        with call_lock:
+            calls += 1
+        generation_started.set()
+        assert fail_generation.wait(timeout=2)
+        raise ValueError("Unknown model")
+
+    def wait_for_generation(_interval: float) -> None:
+        waiter_started.set()
+        assert release_waiter.wait(timeout=2)
+
+    mocker.patch(
+        "paperless_ai.ai_classifier.get_ai_document_classification",
+        side_effect=generate,
+    )
+    mocker.patch("documents.caching.time.sleep", side_effect=wait_for_generation)
+
+    with ThreadPoolExecutor(max_workers=2) as executor:
+        first = executor.submit(
+            retrieve_llm_suggestions,
+            document,
+            user,
+            None,
+            backend="ollama:model",
+            lock_timeout=10,
+        )
+        assert generation_started.wait(timeout=2)
+        second = executor.submit(
+            retrieve_llm_suggestions,
+            document,
+            user,
+            None,
+            backend="ollama:model",
+            lock_timeout=10,
+        )
+        assert waiter_started.wait(timeout=2)
+        fail_generation.set()
+
+        with pytest.raises(ValueError, match="Unknown model"):
+            first.result(timeout=2)
+        release_waiter.set()
+        with pytest.raises(LLMTimeoutError):
+            second.result(timeout=2)
+
+    assert calls == 1

src/documents/tests/test_permission_filtering_security.py

@@ -409,6 +409,7 @@ class TestBulkDownloadPermissionChecksRootDocument:
 
 
 @pytest.mark.django_db
+@pytest.mark.usefixtures("_search_index")
 class TestTrashRestorePermissionBoundary:
     def test_restore_rejects_document_without_delete_permission(
         self,

src/documents/tests/test_views.py

@@ -446,7 +446,7 @@ class TestAISuggestions(DirectoriesMixin, TestCase):
         self.assertEqual(response.json()["tags"], [])
         self.assertEqual(response.json()["suggested_tags"], [])
 
-    @patch("documents.views.get_ai_document_classification")
+    @patch("paperless_ai.ai_classifier.get_ai_document_classification")
     @override_settings(
         AI_ENABLED=True,
         LLM_BACKEND="mock_backend",
@@ -496,7 +496,7 @@ class TestAISuggestions(DirectoriesMixin, TestCase):
             None,
         )
 
-    @patch("documents.views.get_ai_document_classification")
+    @patch("paperless_ai.ai_classifier.get_ai_document_classification")
     @override_settings(
         AI_ENABLED=True,
         LLM_BACKEND="mock_backend",
@@ -534,7 +534,7 @@ class TestAISuggestions(DirectoriesMixin, TestCase):
             "KI Title",
         )
 
-    @patch("documents.views.get_ai_document_classification")
+    @patch("paperless_ai.ai_classifier.get_ai_document_classification")
     @override_settings(
         AI_ENABLED=True,
         LLM_BACKEND="mock_backend",
@@ -573,7 +573,7 @@ class TestAISuggestions(DirectoriesMixin, TestCase):
             "Titre IA",
         )
 
-    @patch("documents.views.get_ai_document_classification")
+    @patch("paperless_ai.ai_classifier.get_ai_document_classification")
     @override_settings(
         AI_ENABLED=True,
         LLM_BACKEND="mock_backend",
@@ -609,7 +609,7 @@ class TestAISuggestions(DirectoriesMixin, TestCase):
             ),
         )
 
-    @patch("documents.views.get_ai_document_classification")
+    @patch("paperless_ai.ai_classifier.get_ai_document_classification")
     @override_settings(
         AI_ENABLED=True,
         LLM_BACKEND="mock_backend",
@@ -681,7 +681,7 @@ class TestAISuggestions(DirectoriesMixin, TestCase):
             ),
         )
 
-    @patch("documents.views.get_ai_document_classification")
+    @patch("paperless_ai.ai_classifier.get_ai_document_classification")
     @override_settings(
         AI_ENABLED=True,
         LLM_BACKEND="openai-like",
@@ -710,7 +710,7 @@ class TestAISuggestions(DirectoriesMixin, TestCase):
             get_llm_suggestion_cache(self.document.pk, backend="openai-like"),
         )
 
-    @patch("documents.views.get_ai_document_classification")
+    @patch("paperless_ai.ai_classifier.get_ai_document_classification")
     @override_settings(
         AI_ENABLED=True,
         LLM_BACKEND="openai-like",
@@ -737,7 +737,7 @@ class TestAISuggestions(DirectoriesMixin, TestCase):
             get_llm_suggestion_cache(self.document.pk, backend="openai-like"),
         )
 
-    @patch("documents.views.get_ai_document_classification")
+    @patch("paperless_ai.ai_classifier.get_ai_document_classification")
     @override_settings(
         AI_ENABLED=True,
         LLM_BACKEND="mock_backend",
@@ -775,7 +775,7 @@ class TestAISuggestions(DirectoriesMixin, TestCase):
         self.assertEqual(response.json()["tags"], [self.tag1.pk])
         self.assertEqual(response.json()["suggested_tags"], ["Follow-up"])
 
-    @patch("documents.views.get_ai_document_classification")
+    @patch("paperless_ai.ai_classifier.get_ai_document_classification")
     @override_settings(
         AI_ENABLED=True,
         LLM_BACKEND="mock_backend",
@@ -814,7 +814,7 @@ class TestAISuggestions(DirectoriesMixin, TestCase):
         self.assertEqual(response.json()["tags"], [self.tag1.pk])
         self.assertEqual(response.json()["suggested_tags"], [])
 
-    @patch("documents.views.get_ai_document_classification")
+    @patch("paperless_ai.ai_classifier.get_ai_document_classification")
     @override_settings(
         AI_ENABLED=True,
         LLM_BACKEND="mock_backend",

src/documents/views.py

@@ -116,7 +116,7 @@ from documents.caching import get_suggestion_cache
 from documents.caching import refresh_llm_suggestions_cache
 from documents.caching import refresh_metadata_cache
 from documents.caching import refresh_suggestions_cache
-from documents.caching import set_llm_suggestions_cache
+from documents.caching import retrieve_llm_suggestions
 from documents.caching import set_metadata_cache
 from documents.caching import set_suggestions_cache
 from documents.classifier import load_classifier
@@ -247,7 +247,6 @@ from paperless.parsers.remote import RemoteEngineConfig
 from paperless.serialisers import GroupSerializer
 from paperless.serialisers import UserSerializer
 from paperless.views import StandardPagination
-from paperless_ai.ai_classifier import get_ai_document_classification
 from paperless_ai.ai_classifier import get_llm_output_language
 from paperless_ai.chat import stream_chat_with_documents
 from paperless_ai.exceptions import LLMTimeoutError
@@ -1565,10 +1564,13 @@ class DocumentViewSet(
             llm_suggestions = cached_llm_suggestions.suggestions
         else:
             try:
-                llm_suggestions = get_ai_document_classification(
-                    doc,
-                    request.user,
-                    output_language,
+                llm_suggestions = retrieve_llm_suggestions(
+                    document=doc,
+                    user=request.user,
+                    output_language=output_language,
+                    backend=llm_cache_backend,
+                    # Classification, localization + 30s
+                    lock_timeout=(2 * ai_config.llm_request_timeout) + 30,
                 )
             except ValueError as exc:
                 logger.exception(
@@ -1593,11 +1595,6 @@ class DocumentViewSet(
                     {"ai": [_("AI backend request timed out.")]},
                     status=status.HTTP_503_SERVICE_UNAVAILABLE,
                 )
-            set_llm_suggestions_cache(
-                doc.pk,
-                llm_suggestions,
-                backend=llm_cache_backend,
-            )
 
         tags_choice: TaxonomyChoiceDict = llm_suggestions["tags"]
         correspondents_choice: TaxonomyChoiceDict = llm_suggestions["correspondents"]