don't let waiting requests re-run a failed llm generation

src/documents/caching.py

@@ -56,7 +56,7 @@ CLASSIFIER_MODIFIED_KEY: Final[str] = "classifier_modified"
 #          [...]} per taxonomy field (#13676)
 LLM_CACHE_CLASSIFIER_VERSION: Final[int] = 1001
 
-# How often a request waiting on llm generation re-checks the  cache
+# How often a request waiting on llm generation re-checks the cache
 LLM_SUGGESTION_POLL_INTERVAL: Final[float] = 0.5
 
 CACHE_1_MINUTE: Final[int] = 60
@@ -241,11 +241,13 @@ def retrieve_llm_suggestions(
     """Return cached LLM suggestions, generating them once across workers."""
     # Lazy import to avoid pulling in the whole AI stuff
     from paperless_ai.ai_classifier import get_ai_document_classification
+    from paperless_ai.exceptions import LLMTimeoutError
 
     lock_key = (
         f"{get_suggestion_cache_key(document.pk)}_llm_lock_"
         f"{sha256(backend.encode()).hexdigest()}"
     )
+    waited = False
 
     while True:
         cached = get_llm_suggestion_cache(document.pk, backend=backend)
@@ -255,6 +257,13 @@ def retrieve_llm_suggestions(
 
         lock_token = uuid4().hex
         if cache.add(lock_key, lock_token, lock_timeout):
+            if waited:
+                # The generation we were waiting on has ended without caching
+                # anything so it either failed or outlived its lock. Give up
+                # rather than re-running it
+                cache.delete(lock_key)
+                raise LLMTimeoutError
+
             try:
                 # The cache may have been populated while acquiring the lock.
                 cached = get_llm_suggestion_cache(document.pk, backend=backend)
@@ -274,11 +283,11 @@ def retrieve_llm_suggestions(
                 )
                 return suggestions
             finally:
-                # Do not remove a replacement lock if this one expired while
-                # generation was still running.
+                # Don't remove lock if this one expired while generation was still running
                 if cache.get(lock_key) == lock_token:
                     cache.delete(lock_key)
 
+        waited = True
         # Another worker is generating suggestions, poll to avoid another LLM request
         time.sleep(LLM_SUGGESTION_POLL_INTERVAL)