Experiments with tying the consumer to the task queue a little more, allowing use to determine if we should requeue a file

The tricky bit is figuring out if we should, so we don't loop. Think this works?
Auto translate strings
2026-09-09 03:07:59 +00:00 · 2026-09-08 19:33:59 -07:00 · 2026-09-08 15:57:05 +00:00 · 2026-09-08 15:55:45 +00:00 · 2026-09-08 15:32:41 +00:00 · 2026-09-08 15:30:58 +00:00
23 changed files with 754 additions and 270 deletions
@@ -72,7 +72,7 @@ jobs:
                'You are welcome to open a new issue that describes the problem you observed in your own words.'
              : 'This issue was automatically closed because it was not opened using our bug report form. ' +
                'Issues have to be created through the form so that the details we need to investigate are included.\n\n' +
-                `If the problem is still there, please [open a new issue](${newIssue}) using the form. No other action is needed here.\n\n' +
+                `If the problem is still there, please [open a new issue](${newIssue}) using the form. No other action is needed here.\n\n` +
                'If any part of your report was written by an AI tool or agent, you must say so: undisclosed AI-generated ' +
                `contributions are a violation of our [Code of Conduct](${codeOfConduct}).`;

@@ -25,6 +25,10 @@ jobs:
  pr-bot:
    name: Automated PR Bot
    runs-on: ubuntu-latest
+    # Runs after Anti-slop so the welcome comment can see whether the PR was closed
+    # instead of racing it. Still runs if that job fails, so labeling is not lost.
+    needs: Anti-slop
+    if: ${{ !cancelled() }}
    permissions:
      contents: read
      pull-requests: write
@@ -99,8 +103,25 @@ jobs:
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        with:
          script: |
-            const pr = context.payload.pull_request;
-            const user = pr.user.login;
+            const user = context.payload.pull_request.user.login;
+
+            // Re-read the PR: Anti-slop may have closed and labeled it after the webhook
+            const { data: pr } = await github.rest.pulls.get({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: context.payload.pull_request.number,
+            });
+
+            if (pr.state === 'closed') {
+              core.info('Skipping comment: PR is already closed');
+              return;
+            }
+
+            const labels = pr.labels.map((label) => (typeof label === 'string' ? label : label.name));
+            if (labels.includes('ai')) {
+              core.info('Skipping comment: PR is labeled ai');
+              return;
+            }

            const { data: members } = await github.rest.orgs.listMembers({
              org: 'paperless-ngx',
@@ -501,15 +501,43 @@
          <context context-type="linenumber">30</context>
        </context-group>
      </trans-unit>
+      <trans-unit id="7057826840809102816" datatype="html">
+        <source>This value overrides <x id="INTERPOLATION" equiv-text="{{option.config_key}}"/>, which is set outside Paperless.</source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
+          <context context-type="linenumber">35</context>
+        </context-group>
+      </trans-unit>
+      <trans-unit id="7221396516204435584" datatype="html">
+        <source><x id="INTERPOLATION" equiv-text="{{option.config_key}}"/> is set outside Paperless. Enter a value here to override it.</source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
+          <context context-type="linenumber">37</context>
+        </context-group>
+      </trans-unit>
+      <trans-unit id="8318849619178340389" datatype="html">
+        <source>Use the externally configured value</source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
+          <context context-type="linenumber">42</context>
+        </context-group>
+      </trans-unit>
+      <trans-unit id="6032629623003430385" datatype="html">
+        <source>Reset to external</source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
+          <context context-type="linenumber">43</context>
+        </context-group>
+      </trans-unit>
      <trans-unit id="7808756054397155068" datatype="html">
        <source>Reset</source>
        <context-group purpose="location">
          <context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
-          <context context-type="linenumber">34</context>
+          <context context-type="linenumber">46</context>
        </context-group>
        <context-group purpose="location">
          <context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
-          <context context-type="linenumber">35</context>
+          <context context-type="linenumber">47</context>
        </context-group>
        <context-group purpose="location">
          <context context-type="sourcefile">src/app/components/admin/settings/settings.component.html</context>
@@ -520,7 +548,7 @@
        <source>Enable</source>
        <context-group purpose="location">
          <context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
-          <context context-type="linenumber">43</context>
+          <context context-type="linenumber">56</context>
        </context-group>
        <context-group purpose="location">
          <context context-type="sourcefile">src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html</context>
@@ -531,7 +559,7 @@
        <source>Cancel</source>
        <context-group purpose="location">
          <context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
-          <context context-type="linenumber">67,68</context>
+          <context context-type="linenumber">80,81</context>
        </context-group>
        <context-group purpose="location">
          <context context-type="sourcefile">src/app/components/admin/settings/settings.component.html</context>
@@ -610,7 +638,7 @@
        <source>Save</source>
        <context-group purpose="location">
          <context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
-          <context context-type="linenumber">70,71</context>
+          <context context-type="linenumber">83,84</context>
        </context-group>
        <context-group purpose="location">
          <context context-type="sourcefile">src/app/components/admin/settings/settings.component.html</context>
@@ -681,42 +709,42 @@
        <source>Error retrieving config</source>
        <context-group purpose="location">
          <context context-type="sourcefile">src/app/components/admin/config/config.component.ts</context>
-          <context context-type="linenumber">117</context>
+          <context context-type="linenumber">118</context>
        </context-group>
      </trans-unit>
      <trans-unit id="1172622527269118932" datatype="html">
        <source>Invalid JSON</source>
        <context-group purpose="location">
          <context context-type="sourcefile">src/app/components/admin/config/config.component.ts</context>
-          <context context-type="linenumber">143</context>
+          <context context-type="linenumber">144</context>
        </context-group>
      </trans-unit>
      <trans-unit id="5103146006962696736" datatype="html">
        <source>Configuration updated</source>
        <context-group purpose="location">
          <context context-type="sourcefile">src/app/components/admin/config/config.component.ts</context>
-          <context context-type="linenumber">187</context>
+          <context context-type="linenumber">193</context>
        </context-group>
      </trans-unit>
      <trans-unit id="1664963291286452273" datatype="html">
        <source>An error occurred updating configuration</source>
        <context-group purpose="location">
          <context context-type="sourcefile">src/app/components/admin/config/config.component.ts</context>
-          <context context-type="linenumber">192</context>
+          <context context-type="linenumber">198</context>
        </context-group>
      </trans-unit>
      <trans-unit id="2653081282186526824" datatype="html">
        <source>File successfully updated</source>
        <context-group purpose="location">
          <context context-type="sourcefile">src/app/components/admin/config/config.component.ts</context>
-          <context context-type="linenumber">214</context>
+          <context context-type="linenumber">220</context>
        </context-group>
      </trans-unit>
      <trans-unit id="5902783625859504265" datatype="html">
        <source>An error occurred uploading file</source>
        <context-group purpose="location">
          <context context-type="sourcefile">src/app/components/admin/config/config.component.ts</context>
-          <context context-type="linenumber">219</context>
+          <context context-type="linenumber">225</context>
        </context-group>
      </trans-unit>
      <trans-unit id="4804785061014590286" datatype="html">
@@ -23,17 +23,30 @@
                                    <div class="col">
                                        <div class="card bg-light">
                                            <div class="card-body">
-                                                <div class="card-title d-flex align-items-center">
+                                                <div class="card-title d-flex align-items-center flex-wrap">
                                                    <h6 class="mb-0">
                                                      {{option.title}}
                                                    </h6>
                                                    <a class="btn btn-sm btn-link" title="Read the documentation about this setting" i18n-title [href]="getDocsUrl(option.config_key)" target="_blank" referrerpolicy="no-referrer">
                                                        <i-bs name="info-circle"></i-bs>
                                                    </a>
+                                                    @if (isExternallyConfigured(option.config_key)) {
+                                                      @if (isSet(option.key)) {
+                                                        <span class="badge rounded-pill bg-body-secondary text-dark fw-normal" title="This value overrides {{option.config_key}}, which is set outside Paperless." i18n-title>Overrides external</span>
+                                                      } @else {
+                                                        <span class="badge rounded-pill bg-body-secondary text-dark fw-normal" title="{{option.config_key}} is set outside Paperless. Enter a value here to override it." i18n-title>Set externally</span>
+                                                      }
+                                                    }
                                                    @if (isSet(option.key)) {
-                                                      <button type="button" class="btn btn-sm btn-link text-danger ms-auto pe-0" title="Reset" i18n-title (click)="resetOption(option.key)">
-                                                          <i-bs class="me-1" name="x"></i-bs><ng-container i18n>Reset</ng-container>
-                                                      </button>
+                                                      @if (isExternallyConfigured(option.config_key)) {
+                                                        <button type="button" class="btn btn-sm btn-link text-danger ms-auto pe-0" title="Use the externally configured value" i18n-title (click)="resetOption(option.key)">
+                                                            <i-bs class="me-1" name="x"></i-bs><ng-container i18n>Reset to external</ng-container>
+                                                        </button>
+                                                      } @else {
+                                                        <button type="button" class="btn btn-sm btn-link text-danger ms-auto pe-0" title="Reset" i18n-title (click)="resetOption(option.key)">
+                                                            <i-bs class="me-1" name="x"></i-bs><ng-container i18n>Reset</ng-container>
+                                                        </button>
+                                                      }
                                                    }
                                                </div>
                                                <div class="mb-n3">
@@ -163,6 +163,19 @@ describe('ConfigComponent', () => {
    expect(component.configForm.get('barcodes_enabled').value).toBeNull()
  })

+  it('should identify externally configured options', () => {
+    component.externallyConfiguredVariables = new Set([
+      'PAPERLESS_OCR_LANGUAGE',
+    ])
+
+    expect(
+      component.isExternallyConfigured('PAPERLESS_OCR_LANGUAGE')
+    ).toBeTruthy()
+    expect(
+      component.isExternallyConfigured('PAPERLESS_OCR_OUTPUT_TYPE')
+    ).toBeFalsy()
+  })
+
  it('should group options into sections within a category, or not', () => {
    const sections = component.getCategorySections(ConfigCategory.OCR)
    expect(sections).toEqual([null, ConfigSection.RemoteOCR])
@@ -69,6 +69,7 @@ export class ConfigComponent
  public configForm = new FormGroup({})

  public errors = {}
+  public externallyConfiguredVariables = new Set<string>()

  get optionCategories(): string[] {
    return Object.values(ConfigCategory)
@@ -152,6 +153,9 @@ export class ConfigComponent
  }

  private initialize(config: PaperlessConfig) {
+    this.externallyConfiguredVariables = new Set(
+      config.externally_configured_variables ?? []
+    )
    if (!this.store) {
      this.store = new BehaviorSubject(config)

@@ -162,7 +166,9 @@ export class ConfigComponent
          this.configForm.patchValue(state, { emitEvent: false })
        })

-      this.isDirty$ = dirtyCheck(this.configForm, this.store.asObservable())
+      this.isDirty$ = dirtyCheck(this.configForm, this.store.asObservable(), {
+        excludeKeys: ['externally_configured_variables'],
+      })
    }
    this.configForm.patchValue(config)

@@ -227,6 +233,10 @@ export class ConfigComponent
    return this.configForm.get(key).value != null
  }

+  public isExternallyConfigured(configKey: string): boolean {
+    return this.externallyConfiguredVariables.has(configKey)
+  }
+
  public resetOption(key: string) {
    this.configForm.get(key).setValue(null)
  }
@@ -422,6 +422,7 @@ export const PaperlessConfigOptions: ConfigOption[] = [
 ]

 export interface PaperlessConfig extends ObjectWithId {
+  externally_configured_variables: string[]
  output_type: OutputTypeConfig
  pages: number
  language: string
@@ -12,7 +12,6 @@ from typing import TYPE_CHECKING
 from typing import Any

 from django.contrib.contenttypes.models import ContentType
-from django.core.exceptions import FieldError
 from django.db.models import Case
 from django.db.models import CharField
 from django.db.models import Count
@@ -53,6 +52,7 @@ from documents.models import StoragePath
 from documents.models import Tag
 from documents.permissions import permitted_document_ids
 from documents.permissions import permitted_object_ids
+from documents.versioning import annotate_effective_content

 if TYPE_CHECKING:
    from collections.abc import Callable
@@ -182,14 +182,9 @@ class TitleContentFilter(Filter):
            logger.warning(
                "Deprecated document filter parameter 'title_content' used; use `text` instead.",
            )
-            try:
-                return qs.filter(
-                    Q(title__icontains=value) | Q(effective_content__icontains=value),
-                )
-            except FieldError:
-                return qs.filter(
-                    Q(title__icontains=value) | Q(content__icontains=value),
-                )
+            return annotate_effective_content(qs).filter(
+                Q(title__icontains=value) | Q(effective_content__icontains=value),
+            )
        else:
            return qs

@@ -200,14 +195,9 @@ class EffectiveContentFilter(Filter):
        value = value.strip() if isinstance(value, str) else value
        if not value:
            return qs
-        try:
-            return qs.filter(
-                **{f"effective_content__{self.lookup_expr}": value},
-            )
-        except FieldError:
-            return qs.filter(
-                **{f"content__{self.lookup_expr}": value},
-            )
+        return annotate_effective_content(qs).filter(
+            **{f"effective_content__{self.lookup_expr}": value},
+        )


@extend_schema_field(serializers.BooleanField)
@@ -72,6 +72,24 @@ class TrackedFile:
            return False


+@dataclass(frozen=True, slots=True)
+class QueuedFile:
+    """A file handed to Celery, with enough state to decide when it's safe to re-check."""
+
+    task_id: str
+    size: int
+    mtime: float
+
+    @classmethod
+    def from_path(cls, task_id: str, path: Path) -> QueuedFile | None:
+        """Snapshot the file's size and mtime, or None if it cannot be stat'd."""
+        try:
+            stat = path.stat()
+        except OSError:
+            return None
+        return cls(task_id, stat.st_size, stat.st_mtime)
+
+
 class FileStabilityTracker:
    """
    Tracks file events and determines when files are stable for consumption.
@@ -314,7 +332,7 @@ def _consume_file(
    consumption_dir: Path,
    *,
    subdirs_as_tags: bool,
-) -> bool:
+) -> str | None:
    """
    Queue a file for consumption.

@@ -324,18 +342,18 @@ def _consume_file(
        subdirs_as_tags: Whether to create tags from subdirectory names.

    Returns:
-        True if the file was successfully handed to Celery, False otherwise.
-        Callers must not record the file as queued on failure, or the rescan
-        will never retry it.
+        The Celery task id if the file was successfully handed to Celery,
+        None otherwise. Callers must not record the file as queued on
+        failure, or the rescan will never retry it.
    """
    # Verify file still exists and is accessible
    try:
        if not filepath.is_file():
            logger.debug(f"Not consuming {filepath}: not a file or doesn't exist")
-            return False
+            return None
    except OSError as e:
        logger.warning(f"Not consuming {filepath}: {e}")
-        return False
+        return None

    # Get tags from path if configured
    tag_ids: list[int] | None = None
@@ -348,7 +366,7 @@ def _consume_file(
    # Queue for consumption
    try:
        logger.info(f"Adding {filepath} to the task queue")
-        consume_file.apply_async(
+        result = consume_file.apply_async(
            kwargs={
                "input_doc": ConsumableDocument(
                    source=DocumentSource.ConsumeFolder,
@@ -360,9 +378,9 @@ def _consume_file(
        )
    except Exception:
        logger.exception(f"Error while queuing document {filepath}")
-        return False
+        return None

-    return True
+    return result.id


 class Command(BaseCommand):
@@ -479,18 +497,19 @@ class Command(BaseCommand):
        recursive: bool,
        subdirs_as_tags: bool,
        consumer_filter: ConsumerFilter,
-    ) -> set[Path]:
+    ) -> dict[Path, QueuedFile]:
        """
        Process any existing files in the consumption directory.

-        Returns the set of resolved paths that were queued, so the watch loop
-        can seed its in-flight set and avoid re-queuing them on the first
-        rescan before the consume tasks have removed them from disk.
+        Returns a dict mapping each resolved path that was queued to its
+        QueuedFile state, so the watch loop can seed its in-flight dict and
+        avoid re-queuing them on the first rescan before the consume tasks
+        have removed them from disk.
        """
        logger.info(f"Processing existing files in {directory}")

        glob_pattern = "**/*" if recursive else "*"
-        queued: set[Path] = set()
+        queued: dict[Path, QueuedFile] = {}

        for filepath in directory.glob(glob_pattern):
            # Use filter to check if file should be processed
@@ -500,12 +519,17 @@ class Command(BaseCommand):
            if not consumer_filter(Change.added, str(filepath)):
                continue

-            if _consume_file(
+            task_id = _consume_file(
                filepath=filepath,
                consumption_dir=directory,
                subdirs_as_tags=subdirs_as_tags,
-            ):
-                queued.add(filepath.resolve())
+            )
+            if task_id is None:
+                continue
+
+            entry = QueuedFile.from_path(task_id, filepath)
+            if entry is not None:
+                queued[filepath.resolve()] = entry

        return queued

@@ -516,21 +540,43 @@ class Command(BaseCommand):
        recursive: bool,
        consumer_filter: ConsumerFilter,
        tracker: FileStabilityTracker,
-        queued: set[Path],
+        queued: dict[Path, QueuedFile],
    ) -> None:
        """
        Re-inject on-disk files the watcher never reported into the tracker.

        Acts as a safety net for files stranded by the watcher-recreation gap
-        (see ``rescan_interval_s``). Files already being tracked or already
-        queued and awaiting consumption are skipped, so a file is never queued
-        twice. Queued paths that have since left the directory are pruned so a
-        later file reusing the same name is not skipped forever.
+        (see ``rescan_interval_s``). Files already being tracked, or already
+        queued and still in flight (or completed but with unchanged content),
+        are skipped, so a file is never queued twice and a permanently broken
+        file does not retry forever. Queued paths that have since left the
+        directory are pruned so a later file reusing the same name is not
+        skipped forever.
        """
-        # Prune in-flight paths that have left the directory
-        for path in list(queued):
-            if not path.exists():
-                queued.discard(path)
+        # Long-running process: drop stale DB connections before querying (#4265)
+        db.close_old_connections()
+
+        # Vanished from disk: consumed (or otherwise removed), prune regardless of status
+        for path in [path for path in queued if not path.exists()]:
+            del queued[path]
+
+        if queued:
+            tasks = PaperlessTask.objects.only("task_id", "status").in_bulk(
+                [entry.task_id for entry in queued.values()],
+                field_name="task_id",
+            )
+            for path, entry in list(queued.items()):
+                task = tasks.get(entry.task_id)
+                # No row yet means the task has not started: treat as in flight
+                if task is None or task.status not in PaperlessTask.COMPLETE_STATUSES:
+                    continue
+                try:
+                    current = path.stat()
+                except OSError:
+                    continue
+                # Completed and the content changed: a new file, allow a retry
+                if current.st_size != entry.size or current.st_mtime != entry.mtime:
+                    del queued[path]

        glob_pattern = "**/*" if recursive else "*"

@@ -558,7 +604,7 @@ class Command(BaseCommand):
        polling_interval: float,
        stability_delay: float,
        is_testing: bool,
-        queued: set[Path] | None = None,
+        queued: dict[Path, QueuedFile] | None = None,
    ) -> None:
        """Watch directory for changes and process stable files."""
        use_polling = polling_interval > 0
@@ -567,7 +613,7 @@ class Command(BaseCommand):
        # Resolved paths that have been queued and are awaiting consumption.
        # Seeded from the startup scan so the first rescan does not re-queue
        # files whose consume tasks have not yet removed them from disk.
-        queued = set() if queued is None else queued
+        queued = {} if queued is None else queued

        # Full-glob safety net cadence (0 disables)
        rescan_interval_s = self.rescan_interval_s
@@ -644,7 +690,7 @@ class Command(BaseCommand):
                            # Consumed (or otherwise removed); a later file
                            # reusing this name must not be skipped as
                            # already-queued.
-                            queued.discard(path)
+                            queued.pop(path, None)
                        if not path.is_file():
                            continue
                        if path in queued:
@@ -663,12 +709,17 @@ class Command(BaseCommand):
                        # rescan does not re-queue them while the consume task
                        # has yet to remove them from disk, but does retry a
                        # failed publish instead of stranding it
-                        if _consume_file(
+                        task_id = _consume_file(
                            filepath=stable_path,
                            consumption_dir=directory,
                            subdirs_as_tags=subdirs_as_tags,
-                        ):
-                            queued.add(stable_path)
+                        )
+                        if task_id is None:
+                            continue
+
+                        entry = QueuedFile.from_path(task_id, stable_path)
+                        if entry is not None:
+                            queued[stable_path] = entry

                    # Exit watch loop to reconfigure timeout
                    break
@@ -677,13 +728,16 @@ class Command(BaseCommand):
                if rescan_timeout_ms > 0 and (
                    monotonic() - last_rescan >= rescan_interval_s
                ):
-                    self._rescan_existing_files(
-                        directory=directory,
-                        recursive=recursive,
-                        consumer_filter=consumer_filter,
-                        tracker=tracker,
-                        queued=queued,
-                    )
+                    try:
+                        self._rescan_existing_files(
+                            directory=directory,
+                            recursive=recursive,
+                            consumer_filter=consumer_filter,
+                            tracker=tracker,
+                            queued=queued,
+                        )
+                    except Exception:
+                        logger.exception("Error during consume folder rescan")
                    last_rescan = monotonic()

                # Determine next timeout
@@ -674,6 +674,9 @@ class TagSerializer(MatchingModelSerializer, OwnedObjectSerializer):
            ordering = ordering or (Lower("name"),)
            children = children.order_by(*ordering)

+        if not children:
+            return []
+
        serializer = TagSerializer(
            children,
            many=True,
@@ -35,7 +35,8 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
        THEN:
            - Existing config
        """
-        response = self.client.get(self.ENDPOINT, format="json")
+        with patch.dict("os.environ", {}, clear=True):
+            response = self.client.get(self.ENDPOINT, format="json")

        self.assertEqual(response.status_code, status.HTTP_200_OK)

@@ -45,6 +46,7 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
            response.data[0],
            {
                "id": 1,
+                "externally_configured_variables": [],
                "output_type": None,
                "pages": None,
                "language": None,
@@ -91,6 +93,31 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
            },
        )

+    def test_api_get_config_reports_external_configuration_without_values(self) -> None:
+        with patch.dict(
+            "os.environ",
+            {
+                "PAPERLESS_OCR_LANGUAGE": "eng",
+                "PAPERLESS_REMOTE_OCR_API_KEY": "secret-value",
+                "PAPERLESS_FUTURE_SETTING": "future-value",
+                "UNRELATED_SETTING": "unrelated-value",
+            },
+            clear=True,
+        ):
+            response = self.client.get(self.ENDPOINT, format="json")
+
+        self.assertCountEqual(
+            response.data[0]["externally_configured_variables"],
+            [
+                "PAPERLESS_FUTURE_SETTING",
+                "PAPERLESS_OCR_LANGUAGE",
+                "PAPERLESS_REMOTE_OCR_API_KEY",
+            ],
+        )
+        self.assertNotContains(response, "secret-value")
+        self.assertNotContains(response, "future-value")
+        self.assertNotContains(response, "UNRELATED_SETTING")
+
    def test_api_get_ui_settings_with_config(self) -> None:
        """
        GIVEN:
@@ -2,14 +2,12 @@ from __future__ import annotations

 import datetime
 from typing import TYPE_CHECKING
-from unittest import TestCase
 from unittest import mock

 from auditlog.models import LogEntry  # type: ignore[import-untyped]
 from django.contrib.auth.models import Permission
 from django.contrib.auth.models import User
 from django.contrib.contenttypes.models import ContentType
-from django.core.exceptions import FieldError
 from django.core.files.uploadedfile import SimpleUploadedFile
 from django.test import TestCase as DjangoTestCase
 from django.utils import timezone
@@ -22,6 +20,7 @@ from documents.filters import TitleContentFilter
 from documents.models import Document
 from documents.tests.utils import DirectoriesMixin
 from documents.tests.utils import read_streaming_response
+from documents.versioning import annotate_effective_content
 from documents.views import DocumentSelectionMixin

 if TYPE_CHECKING:
@@ -892,32 +891,104 @@ class TestDocumentVersioningApi(DirectoriesMixin, APITestCase):
        )


-class TestVersionAwareFilters(TestCase):
-    def test_title_content_filter_falls_back_to_content(self) -> None:
-        queryset = mock.Mock()
-        fallback_queryset = mock.Mock()
-        queryset.filter.side_effect = [FieldError("missing field"), fallback_queryset]
+class TestVersionAwareFilters(DjangoTestCase):
+    """
+    The filters annotate effective_content themselves rather than relying on
+    the caller's queryset carrying it, so they stay version-aware on a plain
+    Document queryset (e.g. the bulk-edit "select all matching" path).
+    """

-        result = TitleContentFilter().filter(queryset, " latest ")
+    def setUp(self) -> None:
+        super().setUp()
+        self.root = Document.objects.create(
+            title="root",
+            checksum="root",
+            mime_type="application/pdf",
+            content="superseded-content",
+        )
+        Document.objects.create(
+            title="version",
+            checksum="version",
+            mime_type="application/pdf",
+            root_document=self.root,
+            version_index=1,
+            content="latest-content",
+        )
+        self.unversioned = Document.objects.create(
+            title="unversioned",
+            checksum="unversioned",
+            mime_type="application/pdf",
+            content="latest-content",
+        )

-        self.assertIs(result, fallback_queryset)
-        self.assertEqual(queryset.filter.call_count, 2)
-
-    def test_effective_content_filter_falls_back_to_content_lookup(self) -> None:
-        queryset = mock.Mock()
-        fallback_queryset = mock.Mock()
-        queryset.filter.side_effect = [FieldError("missing field"), fallback_queryset]
-
-        result = EffectiveContentFilter(lookup_expr="icontains").filter(
-            queryset,
+    def test_title_content_filter_matches_latest_version_content(self) -> None:
+        result = TitleContentFilter().filter(
+            Document.objects.filter(root_document__isnull=True),
            " latest ",
        )

-        self.assertIs(result, fallback_queryset)
-        first_kwargs = queryset.filter.call_args_list[0].kwargs
-        second_kwargs = queryset.filter.call_args_list[1].kwargs
-        self.assertEqual(first_kwargs, {"effective_content__icontains": "latest"})
-        self.assertEqual(second_kwargs, {"content__icontains": "latest"})
+        self.assertCountEqual(
+            [doc.id for doc in result],
+            [self.root.id, self.unversioned.id],
+        )
+
+    def test_effective_content_filter_matches_latest_version_content(self) -> None:
+        result = EffectiveContentFilter(lookup_expr="icontains").filter(
+            Document.objects.filter(root_document__isnull=True),
+            " latest ",
+        )
+
+        self.assertCountEqual(
+            [doc.id for doc in result],
+            [self.root.id, self.unversioned.id],
+        )
+
+    def test_effective_content_filter_ignores_superseded_content(self) -> None:
+        result = EffectiveContentFilter(lookup_expr="icontains").filter(
+            Document.objects.filter(root_document__isnull=True),
+            "superseded",
+        )
+
+        self.assertEqual(list(result), [])
+
+    def test_filters_reuse_an_existing_annotation(self) -> None:
+        """
+        Annotating twice under the same alias is an error, so an already
+        annotated queryset (the search path) has to be left alone.
+        """
+        annotated = annotate_effective_content(
+            Document.objects.filter(root_document__isnull=True),
+        )
+        self.assertIs(annotate_effective_content(annotated), annotated)
+
+        result = EffectiveContentFilter(lookup_expr="icontains").filter(
+            annotated,
+            "latest",
+        )
+
+        self.assertCountEqual(
+            [doc.id for doc in result],
+            [self.root.id, self.unversioned.id],
+        )
+
+    def test_bulk_selection_does_not_match_superseded_content(self) -> None:
+        """
+        Bulk edit's "select all matching" builds its own queryset, so before
+        the filters annotated for themselves it matched the root document's
+        superseded content -- selecting documents the list view, filtered by
+        the same term, does not show.
+        """
+        user = User.objects.create_superuser(username="bulk_selection")
+
+        selected = DocumentSelectionMixin()._resolve_document_ids(
+            user=user,
+            validated_data={
+                "all": True,
+                "filters": {"content__icontains": "superseded"},
+            },
+        )
+
+        self.assertEqual(selected, [])

    def test_effective_content_filter_returns_input_for_empty_values(self) -> None:
        queryset = mock.Mock()
@@ -1947,6 +1947,29 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
        self.assertEqual(len(response.data["documents"]), 1)
        self.assertEqual(response.data["documents"][0]["id"], title_match.id)

+    def test_global_search_returns_latest_version_content(self) -> None:
+        root = Document.objects.create(
+            title="bank statement",
+            content="superseded content",
+            checksum="GSV1",
+            pk=23,
+        )
+        Document.objects.create(
+            title="bank statement v2",
+            content="latest content",
+            checksum="GSV2",
+            pk=24,
+            root_document=root,
+            version_index=1,
+        )
+
+        self.client.force_authenticate(self.user)
+
+        response = self.client.get("/api/search/?query=bank&db_only=true")
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        returned = {doc["id"]: doc["content"] for doc in response.data["documents"]}
+        self.assertEqual(returned.get(root.id), "latest content")
+
    def test_global_search_filters_owned_mail_objects(self) -> None:
        user1 = User.objects.create_user("mail-search-user")
        user2 = User.objects.create_user("other-mail-search-user")
@@ -0,0 +1,165 @@
+"""
+Regression test for GH discussion #13969.
+
+A consume-folder file that fails (e.g. a scanner's 0-byte placeholder
+hitting "Unsupported mime type inode/x-empty") must be re-detected once
+its content changes, not permanently stranded in the watcher's queued
+set. See docs/superpowers/specs/2026-09-08-consume-folder-stuck-queue-spec.md.
+"""
+
+from __future__ import annotations
+
+import shutil
+from time import monotonic
+from time import sleep
+from typing import TYPE_CHECKING
+
+import pytest
+
+from documents.management.commands.document_consumer import Command
+from documents.models import PaperlessTask
+from documents.tests.test_management_consumer import consumption_dir  # noqa: F401
+from documents.tests.test_management_consumer import (
+    mock_consume_file_delay,  # noqa: F401
+)
+from documents.tests.test_management_consumer import (
+    mock_supported_extensions,  # noqa: F401
+)
+from documents.tests.test_management_consumer import sample_pdf  # noqa: F401
+from documents.tests.test_management_consumer import scratch_dir  # noqa: F401
+from documents.tests.test_management_consumer import start_consumer  # noqa: F401
+from documents.tests.test_management_consumer import wait_for_mock_call
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+    from pathlib import Path
+    from unittest.mock import MagicMock
+
+    from pytest_mock import MockerFixture
+
+    from documents.tests.test_management_consumer import ConsumerThread
+
+
+@pytest.mark.management
+# transaction=True: the background consumer thread needs to see rows
+# committed by this test.
+@pytest.mark.django_db(transaction=True)
+class TestStuckQueueAfterConsumptionFailure:
+    def test_scanner_placeholder_recovers_after_failure(
+        self,
+        consumption_dir: Path,  # noqa: F811
+        sample_pdf: Path,  # noqa: F811
+        mock_consume_file_delay: MagicMock,  # noqa: F811
+        start_consumer: Callable[..., ConsumerThread],  # noqa: F811
+    ) -> None:
+        """
+        Reproduces discussion #13969: a scanner creates a 0-byte file,
+        consumption fails on it, then the scanner writes real content —
+        the watcher must pick it up on the next rescan instead of
+        ignoring it forever.
+        """
+        apply_async = mock_consume_file_delay.apply_async
+        apply_async.return_value.id = "scan-task-1"
+
+        thread = start_consumer(
+            stability_delay=0.1,
+            rescan_interval=0.3,
+        )
+
+        target = consumption_dir / "scan.pdf"
+        target.write_bytes(b"")  # scanner's 0-byte placeholder
+
+        assert wait_for_mock_call(apply_async, timeout_s=5.0)
+        if thread.exception:
+            raise thread.exception
+        assert apply_async.call_count == 1
+
+        # The Celery task fails on inode/x-empty, exactly as consumer.py's
+        # mime-type check would in production. Create a real PaperlessTask row
+        # with FAILURE status to simulate the task's result when the real
+        # consumer.py rescan queries for it. This exercises the actual
+        # batched query path: PaperlessTask.objects.filter(task_id__in=[...]).
+        # values_list("task_id", "status")
+        PaperlessTask.objects.create(
+            task_id="scan-task-1",
+            task_type=PaperlessTask.TaskType.CONSUME_FILE,
+            status=PaperlessTask.Status.FAILURE,
+        )
+
+        # Scanner finishes writing the real scan.
+        shutil.copy(sample_pdf, target)
+
+        # Needs to clear: rescan_interval (0.3s, until the entry is
+        # released) + a fresh stability_delay (0.1s, before _consume_file
+        # is called again) + polling slop + test margin.
+        deadline = monotonic() + 8.0
+        while apply_async.call_count < 2 and monotonic() < deadline:
+            sleep(0.1)
+
+        if thread.exception:
+            raise thread.exception
+
+        assert apply_async.call_count == 2, (
+            "Expected the file to be re-consumed after the scanner wrote "
+            f"real content, but apply_async was called "
+            f"{apply_async.call_count} time(s)"
+        )
+
+
+@pytest.mark.management
+@pytest.mark.django_db(transaction=True)
+class TestRescanErrorHandling:
+    def test_rescan_exception_does_not_break_the_watch_loop(
+        self,
+        consumption_dir: Path,  # noqa: F811
+        mock_consume_file_delay: MagicMock,  # noqa: F811
+        start_consumer: Callable[..., ConsumerThread],  # noqa: F811
+        mocker: MockerFixture,
+    ) -> None:
+        """
+        A rescan that raises must not kill the watcher, and must not
+        cause the loop to busy-retry the database on every wake. The
+        watch loop updates ``last_rescan`` even when the rescan raises,
+        so a broken rescan still waits a full ``rescan_interval_s``
+        between attempts instead of spinning.
+
+        A file kept perpetually "pending" (rewritten faster than
+        ``stability_delay``) forces the watch loop to wake more often
+        than ``rescan_interval_s``, which is what surfaces the busy-retry
+        regression: with the broken shape, every one of those frequent
+        wakes re-attempts the rescan instead of only every
+        ``rescan_interval_s``.
+        """
+        rescan = mocker.patch.object(
+            Command,
+            "_rescan_existing_files",
+            side_effect=Exception("db down"),
+        )
+
+        thread = start_consumer(
+            stability_delay=0.02,
+            rescan_interval=0.5,
+        )
+
+        # Keep a file perpetually unstable so the watch loop's timeout is
+        # floored at stability_delay (0.02s) rather than rescan_interval_s
+        # (0.5s). Otherwise the loop only wakes every 0.5s regardless of
+        # the rescan bug, and the two shapes would be indistinguishable.
+        target = consumption_dir / "busy.pdf"
+        deadline = monotonic() + 2.0
+        counter = 0
+        while monotonic() < deadline:
+            counter += 1
+            target.write_bytes(f"%PDF-1.4\n{counter}\n".encode())
+            sleep(0.01)
+
+        assert thread.is_alive()
+        if thread.exception:
+            raise thread.exception
+
+        assert rescan.called
+        assert rescan.call_count < 15, (
+            "Expected the rescan to be retried roughly once per "
+            "rescan_interval_s, but it was called "
+            f"{rescan.call_count} time(s), suggesting a busy-loop"
+        )
@@ -33,9 +33,11 @@ from documents.data_models import DocumentSource
 from documents.management.commands.document_consumer import Command
 from documents.management.commands.document_consumer import ConsumerFilter
 from documents.management.commands.document_consumer import FileStabilityTracker
+from documents.management.commands.document_consumer import QueuedFile
 from documents.management.commands.document_consumer import TrackedFile
 from documents.management.commands.document_consumer import _consume_file
 from documents.management.commands.document_consumer import _tags_from_path
+from documents.models import PaperlessTask
 from documents.models import Tag

 if TYPE_CHECKING:
@@ -445,13 +447,14 @@ class TestConsumeFile:
        target = consumption_dir / "document.pdf"
        shutil.copy(sample_pdf, target)

+        mock_consume_file_delay.apply_async.return_value.id = "abc123"
        result = _consume_file(
            filepath=target,
            consumption_dir=consumption_dir,
            subdirs_as_tags=False,
        )

-        assert result is True
+        assert result == mock_consume_file_delay.apply_async.return_value.id
        mock_consume_file_delay.apply_async.assert_called_once()
        call_args = mock_consume_file_delay.apply_async.call_args
        consumable_doc = call_args.kwargs["kwargs"]["input_doc"]
@@ -470,7 +473,7 @@ class TestConsumeFile:
            consumption_dir=consumption_dir,
            subdirs_as_tags=False,
        )
-        assert result is False
+        assert result is None
        mock_consume_file_delay.apply_async.assert_not_called()

    def test_consume_directory(
@@ -487,7 +490,7 @@ class TestConsumeFile:
            consumption_dir=consumption_dir,
            subdirs_as_tags=False,
        )
-        assert result is False
+        assert result is None
        mock_consume_file_delay.apply_async.assert_not_called()

    def test_consume_with_permission_error(
@@ -507,7 +510,7 @@ class TestConsumeFile:
            consumption_dir=consumption_dir,
            subdirs_as_tags=False,
        )
-        assert result is False
+        assert result is None
        mock_consume_file_delay.apply_async.assert_not_called()

    def test_consume_with_apply_async_failure(
@@ -527,7 +530,7 @@ class TestConsumeFile:
            consumption_dir=consumption_dir,
            subdirs_as_tags=False,
        )
-        assert result is False
+        assert result is None

    def test_consume_with_tags_error(
        self,
@@ -545,12 +548,13 @@ class TestConsumeFile:
            side_effect=DatabaseError("Something happened"),
        )

+        mock_consume_file_delay.apply_async.return_value.id = "abc123"
        result = _consume_file(
            filepath=target,
            consumption_dir=consumption_dir,
            subdirs_as_tags=True,
        )
-        assert result is True
+        assert result == "abc123"
        mock_consume_file_delay.apply_async.assert_called_once()
        call_args = mock_consume_file_delay.apply_async.call_args
        overrides = call_args.kwargs["kwargs"]["overrides"]
@@ -1116,6 +1120,7 @@ class TestCommandWatchEdgeCases:
            Tag.objects.all().delete()


+@pytest.mark.django_db
 class TestRescanExistingFiles:
    """
    Unit tests for the rescan safety net.
@@ -1134,12 +1139,23 @@ class TestRescanExistingFiles:
            ignore_patterns=[],
        )

+    def _queued_as_is(self, target: Path, task_id: str) -> dict[Path, QueuedFile]:
+        """A queued entry whose snapshot matches the file's current content."""
+        stat = target.stat()
+        return {
+            target.resolve(): QueuedFile(
+                task_id=task_id,
+                size=stat.st_size,
+                mtime=stat.st_mtime,
+            ),
+        }
+
    def _rescan(
        self,
        directory: Path,
        consumer_filter: ConsumerFilter,
        tracker: FileStabilityTracker,
-        queued: set[Path],
+        queued: dict[Path, QueuedFile],
        *,
        recursive: bool = False,
    ) -> None:
@@ -1162,7 +1178,7 @@ class TestRescanExistingFiles:
        shutil.copy(sample_pdf, target)
        tracker = FileStabilityTracker(stability_delay=0.1)

-        self._rescan(consumption_dir, pdf_only_filter, tracker, set())
+        self._rescan(consumption_dir, pdf_only_filter, tracker, {})

        assert tracker.is_tracking(target) is True
        assert tracker.pending_count == 1
@@ -1179,7 +1195,7 @@ class TestRescanExistingFiles:
        tracker = FileStabilityTracker(stability_delay=0.1)
        tracker.track(target, Change.added)

-        self._rescan(consumption_dir, pdf_only_filter, tracker, set())
+        self._rescan(consumption_dir, pdf_only_filter, tracker, {})

        assert tracker.pending_count == 1

@@ -1193,11 +1209,17 @@ class TestRescanExistingFiles:
        target = consumption_dir / "inflight.pdf"
        shutil.copy(sample_pdf, target)
        tracker = FileStabilityTracker(stability_delay=0.1)
-        queued = {target.resolve()}
+        PaperlessTask.objects.create(
+            task_id="task-inflight",
+            task_type=PaperlessTask.TaskType.CONSUME_FILE,
+            status=PaperlessTask.Status.STARTED,
+        )
+        queued = self._queued_as_is(target, "task-inflight")

        self._rescan(consumption_dir, pdf_only_filter, tracker, queued)

        assert tracker.pending_count == 0
+        assert target.resolve() in queued

    def test_prunes_vanished_queued_paths(
        self,
@@ -1207,7 +1229,7 @@ class TestRescanExistingFiles:
        """Queued paths no longer on disk are dropped so the name can recur."""
        gone = (consumption_dir / "gone.pdf").resolve()
        tracker = FileStabilityTracker(stability_delay=0.1)
-        queued = {gone}
+        queued = {gone: QueuedFile(task_id="task-gone", size=0, mtime=0.0)}

        self._rescan(consumption_dir, pdf_only_filter, tracker, queued)

@@ -1222,7 +1244,7 @@ class TestRescanExistingFiles:
        (consumption_dir / "notes.xyz").write_bytes(b"content")
        tracker = FileStabilityTracker(stability_delay=0.1)

-        self._rescan(consumption_dir, pdf_only_filter, tracker, set())
+        self._rescan(consumption_dir, pdf_only_filter, tracker, {})

        assert tracker.pending_count == 0

@@ -1239,13 +1261,151 @@ class TestRescanExistingFiles:
        shutil.copy(sample_pdf, target)

        shallow = FileStabilityTracker(stability_delay=0.1)
-        self._rescan(consumption_dir, pdf_only_filter, shallow, set())
+        self._rescan(consumption_dir, pdf_only_filter, shallow, {})
        assert shallow.pending_count == 0

        deep = FileStabilityTracker(stability_delay=0.1)
-        self._rescan(consumption_dir, pdf_only_filter, deep, set(), recursive=True)
+        self._rescan(consumption_dir, pdf_only_filter, deep, {}, recursive=True)
        assert deep.is_tracking(target) is True

+    def test_completed_but_content_unchanged_stays_queued(
+        self,
+        consumption_dir: Path,
+        sample_pdf: Path,
+        pdf_only_filter: ConsumerFilter,
+    ) -> None:
+        """
+        A task that failed (or succeeded) but whose file content never
+        changed since being queued stays put — this is what makes a
+        permanently-broken file (e.g. a corrupt PDF) fail once instead of
+        retrying forever (C2).
+        """
+        target = consumption_dir / "broken.pdf"
+        shutil.copy(sample_pdf, target)
+        tracker = FileStabilityTracker(stability_delay=0.1)
+        PaperlessTask.objects.create(
+            task_id="task-failed-unchanged",
+            task_type=PaperlessTask.TaskType.CONSUME_FILE,
+            status=PaperlessTask.Status.FAILURE,
+        )
+        queued = self._queued_as_is(target, "task-failed-unchanged")
+
+        self._rescan(consumption_dir, pdf_only_filter, tracker, queued)
+
+        assert target.resolve() in queued
+        assert tracker.pending_count == 0
+
+    def test_completed_and_content_changed_is_released(
+        self,
+        consumption_dir: Path,
+        sample_pdf: Path,
+        pdf_only_filter: ConsumerFilter,
+    ) -> None:
+        """
+        A task that completed AND whose file content has since changed is
+        released and re-tracked — this is the discussion #13969 fix: the
+        scanner's 0-byte file failed, then real content arrived.
+        """
+        target = consumption_dir / "scanned.pdf"
+        target.write_bytes(b"")  # simulate the 0-byte placeholder that was queued
+        tracker = FileStabilityTracker(stability_delay=0.1)
+        PaperlessTask.objects.create(
+            task_id="task-failed-changed",
+            task_type=PaperlessTask.TaskType.CONSUME_FILE,
+            status=PaperlessTask.Status.FAILURE,
+        )
+        queued = {
+            target.resolve(): QueuedFile(
+                task_id="task-failed-changed",
+                size=0,
+                mtime=0.0,
+            ),
+        }
+        shutil.copy(sample_pdf, target)  # scanner writes real content
+
+        self._rescan(consumption_dir, pdf_only_filter, tracker, queued)
+
+        assert target.resolve() not in queued
+        assert tracker.is_tracking(target.resolve()) is True
+
+    def test_no_paperlesstask_row_stays_queued(
+        self,
+        consumption_dir: Path,
+        sample_pdf: Path,
+        pdf_only_filter: ConsumerFilter,
+    ) -> None:
+        """No matching PaperlessTask row is treated as still in flight, not released."""
+        target = consumption_dir / "no_row.pdf"
+        shutil.copy(sample_pdf, target)
+        tracker = FileStabilityTracker(stability_delay=0.1)
+        queued = self._queued_as_is(target, "task-does-not-exist")
+
+        self._rescan(consumption_dir, pdf_only_filter, tracker, queued)
+
+        assert target.resolve() in queued
+
+    def test_revoked_status_is_treated_as_complete(
+        self,
+        consumption_dir: Path,
+        sample_pdf: Path,
+        pdf_only_filter: ConsumerFilter,
+    ) -> None:
+        """
+        The release guard checks membership in COMPLETE_STATUSES (SUCCESS,
+        FAILURE, REVOKED), not just FAILURE. A cancelled/revoked task (e.g.
+        after a worker restart discards a stale queue entry) whose content
+        has since changed must also be released, not stuck treating REVOKED
+        as still in-flight.
+        """
+        target = consumption_dir / "revoked.pdf"
+        target.write_bytes(b"")
+        tracker = FileStabilityTracker(stability_delay=0.1)
+        PaperlessTask.objects.create(
+            task_id="task-revoked",
+            task_type=PaperlessTask.TaskType.CONSUME_FILE,
+            status=PaperlessTask.Status.REVOKED,
+        )
+        queued = {
+            target.resolve(): QueuedFile(task_id="task-revoked", size=0, mtime=0.0),
+        }
+        shutil.copy(sample_pdf, target)
+
+        self._rescan(consumption_dir, pdf_only_filter, tracker, queued)
+
+        assert target.resolve() not in queued
+        assert tracker.is_tracking(target.resolve()) is True
+
+    def test_rescan_issues_one_batched_query_for_multiple_queued_files(
+        self,
+        consumption_dir: Path,
+        sample_pdf: Path,
+        pdf_only_filter: ConsumerFilter,
+        django_assert_num_queries,
+    ) -> None:
+        """
+        The status lookup for every queued file must be a single batched
+        `task_id__in=[...]` query, not one query per file — otherwise a
+        consume folder with many in-flight files turns every rescan into
+        an N+1.
+        """
+        tracker = FileStabilityTracker(stability_delay=0.1)
+        queued: dict[Path, QueuedFile] = {}
+        for i in range(3):
+            target = consumption_dir / f"doc{i}.pdf"
+            shutil.copy(sample_pdf, target)
+            task_id = f"task-batched-{i}"
+            PaperlessTask.objects.create(
+                task_id=task_id,
+                task_type=PaperlessTask.TaskType.CONSUME_FILE,
+                status=PaperlessTask.Status.STARTED,
+            )
+            queued.update(self._queued_as_is(target, task_id))
+
+        with django_assert_num_queries(1):
+            self._rescan(consumption_dir, pdf_only_filter, tracker, queued)
+
+        assert len(queued) == 3
+

 class TestProcessExistingFilesQueued:
    """Tests that startup processing reports which paths it queued."""
@@ -1258,7 +1418,8 @@ class TestProcessExistingFilesQueued:
        mock_consume_file_delay: MagicMock,
        settings: Settings,
    ) -> None:
-        """The set returned seeds the rescan's queued set, avoiding re-queue."""
+        """The dict returned seeds the rescan's queued dict, avoiding re-queue."""
+        mock_consume_file_delay.apply_async.return_value.id = "startup-task-id"
        target = consumption_dir / "document.pdf"
        shutil.copy(sample_pdf, target)
        settings.CONSUMER_IGNORE_PATTERNS = []
@@ -1271,6 +1432,9 @@ class TestProcessExistingFilesQueued:
        )

        assert target.resolve() in queued
+        entry = queued[target.resolve()]
+        assert entry.task_id == "startup-task-id"
+        assert entry.size == target.stat().st_size


@pytest.mark.management
@@ -1295,11 +1459,13 @@ class TestCommandRetryAfterQueueFailure:
        """A publish failure from the watch loop is retried by the rescan."""
        apply_async = mock_consume_file_delay.apply_async

-        def fail_first_call(*args: object, **kwargs: object) -> None:
+        def fail_first_call(*args: object, **kwargs: object) -> MagicMock | None:
            if apply_async.call_count == 1:
                raise Exception("broker down")
+            return apply_async.return_value

        apply_async.side_effect = fail_first_call
+        apply_async.return_value.id = "task_id_test"

        thread = start_consumer(stability_delay=0.1, rescan_interval=0.3)

@@ -32,7 +32,6 @@ from documents.signals.handlers import update_llm_suggestions_cache
 from documents.tests.utils import DirectoriesMixin
 from documents.tests.utils import read_streaming_response
 from paperless.models import ApplicationConfiguration
-from paperless_ai.exceptions import LLMProviderError
 from paperless_ai.exceptions import LLMTimeoutError


@@ -738,38 +737,6 @@ class TestAISuggestions(DirectoriesMixin, TestCase):
            get_llm_suggestion_cache(self.document.pk, backend="openai-like"),
        )

-    @patch("documents.views.get_ai_document_classification")
-    @override_settings(
-        AI_ENABLED=True,
-        LLM_BACKEND="openai-like",
-    )
-    def test_ai_suggestions_with_llm_provider_error(
-        self,
-        mock_get_ai_classification,
-    ) -> None:
-        mock_get_ai_classification.side_effect = LLMProviderError(
-            "confidential provider response",
-        )
-
-        self.client.force_login(user=self.user)
-        response = self.client.get(
-            f"/api/documents/{self.document.pk}/ai_suggestions/",
-        )
-
-        self.assertEqual(response.status_code, status.HTTP_502_BAD_GATEWAY)
-        self.assertEqual(
-            response.json(),
-            {
-                "ai": [
-                    "AI backend rejected the request. Check logs for details.",
-                ],
-            },
-        )
-        self.assertNotIn("confidential provider response", response.content.decode())
-        self.assertIsNone(
-            get_llm_suggestion_cache(self.document.pk, backend="openai-like"),
-        )
-
    @patch("documents.views.get_ai_document_classification")
    @override_settings(
        AI_ENABLED=True,
@@ -27,10 +27,13 @@ def versions_newest_first(documents: QuerySet[Document]) -> QuerySet[Document]:

 def annotate_effective_content(documents: QuerySet[Document]) -> QuerySet[Document]:
    """
-    Annotates documents with the content of their newest version, falling back
-    to their own, so get_effective_content() can answer from the row rather
-    than querying for the versions of each document
+    Annotates documents with the content of their newest version unless the
+    queryset already carries the annotation, falling back to their own, so
+    get_effective_content() can answer from the row rather than querying for
+    the versions of each document.
    """
+    if "effective_content" in documents.query.annotations:
+        return documents
    return documents.annotate(
        effective_content=Coalesce(
            Subquery(
@@ -232,6 +232,7 @@ from documents.tasks import train_classifier
 from documents.tasks import update_document_parent_tags
 from documents.utils import get_boolean
 from documents.versioning import VersionResolutionError
+from documents.versioning import annotate_effective_content
 from documents.versioning import get_latest_version_for_root
 from documents.versioning import get_request_version_param
 from documents.versioning import get_root_document
@@ -251,7 +252,6 @@ from paperless.views import StandardPagination
 from paperless_ai.ai_classifier import get_ai_document_classification
 from paperless_ai.ai_classifier import get_llm_output_language
 from paperless_ai.chat import stream_chat_with_documents
-from paperless_ai.exceptions import LLMProviderError
 from paperless_ai.exceptions import LLMTimeoutError
 from paperless_ai.matching import extract_unmatched_names
 from paperless_ai.matching import match_correspondents_by_name
@@ -1603,22 +1603,6 @@ class DocumentViewSet(
                    {"ai": [_("AI backend request timed out.")]},
                    status=status.HTTP_503_SERVICE_UNAVAILABLE,
                )
-            except LLMProviderError:
-                logger.exception(
-                    "AI backend rejected the request for document %s",
-                    doc.pk,
-                )
-                return Response(
-                    {
-                        "ai": [
-                            _(
-                                "AI backend rejected the request. "
-                                "Check logs for details.",
-                            ),
-                        ],
-                    },
-                    status=status.HTTP_502_BAD_GATEWAY,
-                )
            set_llm_suggestions_cache(
                doc.pk,
                llm_suggestions,
@@ -3649,8 +3633,13 @@ class GlobalSearchView(PassUserMixin):
        OBJECT_LIMIT = 3
        docs = []
        if request.user.has_perm("documents.view_document"):
-            all_docs = Document.objects.filter(
-                id__in=permitted_document_ids(request.user),
+            # Never more than OBJECT_LIMIT rows come back here, so annotating
+            # is cheap -- and without it these results show the root
+            # document's superseded content.
+            all_docs = annotate_effective_content(
+                Document.objects.filter(
+                    id__in=permitted_document_ids(request.user),
+                ),
            )
            if db_only:
                docs = all_docs.filter(title__icontains=query)[:OBJECT_LIMIT]
@@ -2,7 +2,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: paperless-ngx\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2026-09-07 20:47+0000\n"
+"POT-Creation-Date: 2026-09-08 15:56+0000\n"
 "PO-Revision-Date: 2022-02-17 04:17\n"
 "Last-Translator: \n"
 "Language-Team: English\n"
@@ -21,39 +21,39 @@ msgstr ""
 msgid "Documents"
 msgstr ""

-#: documents/filters.py:473
+#: documents/filters.py:463
 msgid "Value must be valid JSON."
 msgstr ""

-#: documents/filters.py:492
+#: documents/filters.py:482
 msgid "Invalid custom field query expression"
 msgstr ""

-#: documents/filters.py:502
+#: documents/filters.py:492
 msgid "Invalid expression list. Must be nonempty."
 msgstr ""

-#: documents/filters.py:523
+#: documents/filters.py:513
 msgid "Invalid logical operator {op!r}"
 msgstr ""

-#: documents/filters.py:537
+#: documents/filters.py:527
 msgid "Maximum number of query conditions exceeded."
 msgstr ""

-#: documents/filters.py:601
+#: documents/filters.py:591
 msgid "{name!r} is not a valid custom field."
 msgstr ""

-#: documents/filters.py:638
+#: documents/filters.py:628
 msgid "{data_type} does not support query expr {expr!r}."
 msgstr ""

-#: documents/filters.py:757 documents/models.py:136
+#: documents/filters.py:747 documents/models.py:136
 msgid "Maximum nesting depth exceeded."
 msgstr ""

-#: documents/filters.py:1119
+#: documents/filters.py:1109
 msgid "Custom field not found"
 msgstr ""

@@ -1631,49 +1631,49 @@ msgstr ""
 msgid "workflow runs"
 msgstr ""

-#: documents/serialisers.py:524 documents/serialisers.py:878
-#: documents/serialisers.py:2838 documents/views.py:314 documents/views.py:2624
+#: documents/serialisers.py:524 documents/serialisers.py:881
+#: documents/serialisers.py:2841 documents/views.py:315 documents/views.py:2625
 #: paperless_mail/serialisers.py:156
 msgid "Insufficient permissions."
 msgstr ""

-#: documents/serialisers.py:714
+#: documents/serialisers.py:717
 msgid "Invalid color."
 msgstr ""

-#: documents/serialisers.py:2315
+#: documents/serialisers.py:2318
 #, python-format
 msgid "File type %(type)s not supported"
 msgstr ""

-#: documents/serialisers.py:2359
+#: documents/serialisers.py:2362
 #, python-format
 msgid "Custom field id must be an integer: %(id)s"
 msgstr ""

-#: documents/serialisers.py:2366
+#: documents/serialisers.py:2369
 #, python-format
 msgid "Custom field with id %(id)s does not exist"
 msgstr ""

-#: documents/serialisers.py:2383 documents/serialisers.py:2393
+#: documents/serialisers.py:2386 documents/serialisers.py:2396
 msgid ""
 "Custom fields must be a list of integers or an object mapping ids to values."
 msgstr ""

-#: documents/serialisers.py:2388
+#: documents/serialisers.py:2391
 msgid "Some custom fields don't exist or were specified twice."
 msgstr ""

-#: documents/serialisers.py:2535
+#: documents/serialisers.py:2538
 msgid "Invalid variable detected."
 msgstr ""

-#: documents/serialisers.py:2894
+#: documents/serialisers.py:2897
 msgid "Duplicate document identifiers are not allowed."
 msgstr ""

-#: documents/serialisers.py:2924 documents/views.py:4626
+#: documents/serialisers.py:2927 documents/views.py:4632
 #, python-format
 msgid "Documents not found: %(ids)s"
 msgstr ""
@@ -1941,36 +1941,36 @@ msgstr ""
 msgid "Unable to parse URI {value}"
 msgstr ""

-#: documents/views.py:307 documents/views.py:2621
+#: documents/views.py:308 documents/views.py:2622
 msgid "Invalid more_like_id"
 msgstr ""

-#: documents/views.py:1591
+#: documents/views.py:1592
 msgid "Invalid AI configuration."
 msgstr ""

-#: documents/views.py:1602
+#: documents/views.py:1603
 msgid "AI backend request timed out."
 msgstr ""

-#: documents/views.py:2446 documents/views.py:2767
+#: documents/views.py:2447 documents/views.py:2768
 msgid "Specify only one of text, title_search, query, or more_like_id."
 msgstr ""

-#: documents/views.py:4639
+#: documents/views.py:4645
 #, python-format
 msgid "Insufficient permissions to share document %(id)s."
 msgstr ""

-#: documents/views.py:4685
+#: documents/views.py:4691
 msgid "Bundle is already being processed."
 msgstr ""

-#: documents/views.py:4749
+#: documents/views.py:4755
 msgid "The share link bundle is still being prepared. Please try again later."
 msgstr ""

-#: documents/views.py:4763
+#: documents/views.py:4769
 msgid "The share link bundle is unavailable."
 msgstr ""

@@ -1,4 +1,5 @@
 import logging
+import os
 from io import BytesIO

 import magic
@@ -212,6 +213,7 @@ class ProfileSerializer(PasswordValidationMixin, serializers.ModelSerializer[Use
 class ApplicationConfigurationSerializer(
    serializers.ModelSerializer[ApplicationConfiguration],
 ):
+    externally_configured_variables = serializers.SerializerMethodField()
    user_args = serializers.JSONField(binary=True, allow_null=True)
    barcode_tag_mapping = serializers.JSONField(binary=True, allow_null=True)
    llm_api_key = ObfuscatedPasswordField(
@@ -227,6 +229,12 @@ class ApplicationConfigurationSerializer(

    OBFUSCATED_FIELDS = ("llm_api_key", "remote_ocr_api_key")

+    def get_externally_configured_variables(
+        self,
+        instance: ApplicationConfiguration,
+    ) -> list[str]:
+        return sorted(name for name in os.environ if name.startswith("PAPERLESS_"))
+
    def run_validation(self, data):
        # Empty strings treated as None to avoid unexpected behavior
        if "user_args" in data and data["user_args"] == "":
@@ -22,7 +22,6 @@ from paperless.network import validate_outbound_http_url
 from paperless_ai.base_model import ClassificationSuggestions
 from paperless_ai.base_model import DocumentClassifierSchema
 from paperless_ai.base_model import model_to_classification_suggestions
-from paperless_ai.exceptions import LLMProviderError
 from paperless_ai.exceptions import LLMTimeoutError

 logger = logging.getLogger("paperless_ai.client")
@@ -133,7 +132,7 @@ class AIClient:
        from llama_index.core.llms import ChatMessage

        if self.settings.llm_backend == LLMBackend.OLLAMA:
-            with self._normalize_errors():
+            with self._normalize_timeouts():
                result = self.llm.chat(
                    [ChatMessage(role="user", content=prompt)],
                    format=DocumentClassifierSchema.model_json_schema(),
@@ -154,7 +153,7 @@ class AIClient:
            content=f"{prompt}\n\n"
            f"Answer by calling the {tool.metadata.name} tool. Do not write the answer as text.",
        )
-        with self._normalize_errors():
+        with self._normalize_timeouts():
            result = self.llm.chat_with_tools(
                tools=[tool],
                user_msg=user_msg,
@@ -174,7 +173,7 @@ class AIClient:
        )

    @contextmanager
-    def _normalize_errors(self) -> Iterator[None]:
+    def _normalize_timeouts(self) -> Iterator[None]:
        try:
            yield
        except httpx.TimeoutException as exc:
@@ -182,23 +181,8 @@ class AIClient:
        except Exception as exc:
            if self._is_openai_timeout(exc):
                raise LLMTimeoutError from exc
-            if self._is_provider_error(exc):
-                raise LLMProviderError from exc
            raise

-    def _is_provider_error(self, exc: Exception) -> bool:
-        if self.settings.llm_backend == LLMBackend.OLLAMA:
-            from ollama import ResponseError
-
-            return isinstance(exc, ResponseError)
-
-        if self.settings.llm_backend == LLMBackend.OPENAI_LIKE:
-            from openai import APIStatusError
-
-            return isinstance(exc, APIStatusError)
-
-        return False
-
    def _is_openai_timeout(self, exc: Exception) -> bool:
        if self.settings.llm_backend != LLMBackend.OPENAI_LIKE:
            return False
@@ -1,6 +1,2 @@
 class LLMTimeoutError(Exception):
    pass
-
-
-class LLMProviderError(Exception):
-    """The LLM backend rejected the request."""
@@ -4,7 +4,6 @@ from unittest.mock import MagicMock
 from unittest.mock import patch

 import httpx
-import ollama
 import openai
 import pytest
 from llama_index.core.llms.llm import ToolSelection
@@ -12,7 +11,6 @@ from llama_index.core.llms.llm import ToolSelection
 from paperless_ai.client import LLM_SYSTEM_PROMPT
 from paperless_ai.client import PLACEHOLDER_API_KEY
 from paperless_ai.client import AIClient
-from paperless_ai.exceptions import LLMProviderError
 from paperless_ai.exceptions import LLMTimeoutError


@@ -216,52 +214,6 @@ def test_run_llm_query_openai_timeout_raises_local_error(
        client.run_llm_query("test_prompt")


-def test_run_llm_query_openai_status_error_raises_provider_error(
-    mock_ai_config,
-    mock_openai_llm,
-):
-    mock_ai_config.llm_backend = "openai-like"
-    mock_ai_config.llm_model = "test_model"
-    mock_ai_config.llm_endpoint = "http://test-url"
-
-    request = httpx.Request("POST", "http://test-url/v1/chat/completions")
-    body = {"error": {"message": "Thinking mode does not support this tool_choice"}}
-    mock_openai_llm.return_value.chat_with_tools.side_effect = openai.BadRequestError(
-        "Error code: 400",
-        response=httpx.Response(400, request=request, json=body),
-        body=body,
-    )
-
-    client = AIClient()
-
-    with pytest.raises(LLMProviderError) as exc_info:
-        client.run_llm_query("test_prompt")
-    assert str(exc_info.value) == ""
-    assert isinstance(exc_info.value.__cause__, openai.BadRequestError)
-
-
-def test_run_llm_query_ollama_response_error_raises_provider_error(
-    mock_ai_config,
-    mock_ollama_llm,
-):
-    mock_ai_config.llm_backend = "ollama"
-    mock_ai_config.llm_model = "test_model"
-    mock_ai_config.llm_endpoint = "http://test-url"
-
-    response_error = ollama.ResponseError(
-        "confidential provider response",
-        status_code=400,
-    )
-    mock_ollama_llm.return_value.chat.side_effect = response_error
-
-    client = AIClient()
-
-    with pytest.raises(LLMProviderError) as exc_info:
-        client.run_llm_query("test_prompt")
-    assert str(exc_info.value) == ""
-    assert exc_info.value.__cause__ is response_error
-
-
 def test_run_llm_query_httpx_timeout_raises_local_error(
    mock_ai_config,
    mock_ollama_llm,
Author	SHA1	Message	Date
stumpylog	cf252b144c	Experiments with tying the consumer to the task queue a little more, allowing use to determine if we should requeue a file The tricky bit is figuring out if we should, so we don't loop. Think this works?	2026-09-08 19:33:59 -07:00
GitHub Actions	b989b74140	Auto translate strings	2026-09-08 15:57:05 +00:00
shamoon	5194f47291	Performance: ensure version-aware content filters on querysets (#13792 )	2026-09-08 15:55:45 +00:00
GitHub Actions	714885d7a5	Auto translate strings	2026-09-08 15:32:41 +00:00
Trenton H	73e777a48c	Fix: skip nested TagSerializer construction when a tag has no children (#14039 ) TagSerializer.get_children() built a full nested TagSerializer(many=True) for every tag, even when it had zero children, likely the common case for most tags and maybe even most installs. Constructing a DRF ModelSerializer isn't free (field introspection, deepcopy of declared fields, i18n lookups all re-run per instantiation), so this scaled GET /api/tags/ linearly with tag count in pure Python overhead, unrelated to SQL query count.	2026-09-08 15:30:58 +00:00
GitHub Actions	e9141366bb	Auto translate strings	2026-09-08 14:40:59 +00:00
shamoon	7813375123	Enhancement (QoL): surface externally-set options in Config UI (#13989 )	2026-09-08 14:39:22 +00:00
shamoon	0132c7bd6e	Fix pr-bot timing	2026-09-07 22:59:41 -07:00