Compare commits

..
Author SHA1 Message Date
GitHub Actions b989b74140 Auto translate strings 2026-09-08 15:57:05 +00:00
shamoon 5194f47291 Performance: ensure version-aware content filters on querysets (#13792) 2026-09-08 15:55:45 +00:00
GitHub Actions 714885d7a5 Auto translate strings 2026-09-08 15:32:41 +00:00
Trenton H 73e777a48c Fix: skip nested TagSerializer construction when a tag has no children (#14039)
TagSerializer.get_children() built a full nested TagSerializer(many=True)
for every tag, even when it had zero children, likely the common case for
most tags and maybe even most installs. Constructing a DRF ModelSerializer isn't
free (field introspection, deepcopy of declared fields, i18n lookups
all re-run per instantiation), so this scaled GET /api/tags/ linearly
with tag count in pure Python overhead, unrelated to SQL query count.
2026-09-08 15:30:58 +00:00
GitHub Actions e9141366bb Auto translate strings 2026-09-08 14:40:59 +00:00
shamoon 7813375123 Enhancement (QoL): surface externally-set options in Config UI (#13989) 2026-09-08 14:39:22 +00:00
shamoon 0132c7bd6e Fix pr-bot timing 2026-09-07 22:59:41 -07:00
20 changed files with 314 additions and 215 deletions
+1 -1
View File
@@ -72,7 +72,7 @@ jobs:
'You are welcome to open a new issue that describes the problem you observed in your own words.' 'You are welcome to open a new issue that describes the problem you observed in your own words.'
: 'This issue was automatically closed because it was not opened using our bug report form. ' + : 'This issue was automatically closed because it was not opened using our bug report form. ' +
'Issues have to be created through the form so that the details we need to investigate are included.\n\n' + 'Issues have to be created through the form so that the details we need to investigate are included.\n\n' +
`If the problem is still there, please [open a new issue](${newIssue}) using the form. No other action is needed here.\n\n' + `If the problem is still there, please [open a new issue](${newIssue}) using the form. No other action is needed here.\n\n` +
'If any part of your report was written by an AI tool or agent, you must say so: undisclosed AI-generated ' + 'If any part of your report was written by an AI tool or agent, you must say so: undisclosed AI-generated ' +
`contributions are a violation of our [Code of Conduct](${codeOfConduct}).`; `contributions are a violation of our [Code of Conduct](${codeOfConduct}).`;
+23 -2
View File
@@ -25,6 +25,10 @@ jobs:
pr-bot: pr-bot:
name: Automated PR Bot name: Automated PR Bot
runs-on: ubuntu-latest runs-on: ubuntu-latest
# Runs after Anti-slop so the welcome comment can see whether the PR was closed
# instead of racing it. Still runs if that job fails, so labeling is not lost.
needs: Anti-slop
if: ${{ !cancelled() }}
permissions: permissions:
contents: read contents: read
pull-requests: write pull-requests: write
@@ -99,8 +103,25 @@ jobs:
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with: with:
script: | script: |
const pr = context.payload.pull_request; const user = context.payload.pull_request.user.login;
const user = pr.user.login;
// Re-read the PR: Anti-slop may have closed and labeled it after the webhook
const { data: pr } = await github.rest.pulls.get({
owner: context.repo.owner,
repo: context.repo.repo,
pull_number: context.payload.pull_request.number,
});
if (pr.state === 'closed') {
core.info('Skipping comment: PR is already closed');
return;
}
const labels = pr.labels.map((label) => (typeof label === 'string' ? label : label.name));
if (labels.includes('ai')) {
core.info('Skipping comment: PR is labeled ai');
return;
}
const { data: members } = await github.rest.orgs.listMembers({ const { data: members } = await github.rest.orgs.listMembers({
org: 'paperless-ngx', org: 'paperless-ngx',
+39 -11
View File
@@ -501,15 +501,43 @@
<context context-type="linenumber">30</context> <context context-type="linenumber">30</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="7057826840809102816" datatype="html">
<source>This value overrides <x id="INTERPOLATION" equiv-text="{{option.config_key}}"/>, which is set outside Paperless.</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
<context context-type="linenumber">35</context>
</context-group>
</trans-unit>
<trans-unit id="7221396516204435584" datatype="html">
<source><x id="INTERPOLATION" equiv-text="{{option.config_key}}"/> is set outside Paperless. Enter a value here to override it.</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
<context context-type="linenumber">37</context>
</context-group>
</trans-unit>
<trans-unit id="8318849619178340389" datatype="html">
<source>Use the externally configured value</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
<context context-type="linenumber">42</context>
</context-group>
</trans-unit>
<trans-unit id="6032629623003430385" datatype="html">
<source>Reset to external</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
<context context-type="linenumber">43</context>
</context-group>
</trans-unit>
<trans-unit id="7808756054397155068" datatype="html"> <trans-unit id="7808756054397155068" datatype="html">
<source>Reset</source> <source>Reset</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/admin/config/config.component.html</context> <context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
<context context-type="linenumber">34</context> <context context-type="linenumber">46</context>
</context-group> </context-group>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/admin/config/config.component.html</context> <context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
<context context-type="linenumber">35</context> <context context-type="linenumber">47</context>
</context-group> </context-group>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/admin/settings/settings.component.html</context> <context context-type="sourcefile">src/app/components/admin/settings/settings.component.html</context>
@@ -520,7 +548,7 @@
<source>Enable</source> <source>Enable</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/admin/config/config.component.html</context> <context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
<context context-type="linenumber">43</context> <context context-type="linenumber">56</context>
</context-group> </context-group>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html</context> <context context-type="sourcefile">src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html</context>
@@ -531,7 +559,7 @@
<source>Cancel</source> <source>Cancel</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/admin/config/config.component.html</context> <context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
<context context-type="linenumber">67,68</context> <context context-type="linenumber">80,81</context>
</context-group> </context-group>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/admin/settings/settings.component.html</context> <context context-type="sourcefile">src/app/components/admin/settings/settings.component.html</context>
@@ -610,7 +638,7 @@
<source>Save</source> <source>Save</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/admin/config/config.component.html</context> <context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
<context context-type="linenumber">70,71</context> <context context-type="linenumber">83,84</context>
</context-group> </context-group>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/admin/settings/settings.component.html</context> <context context-type="sourcefile">src/app/components/admin/settings/settings.component.html</context>
@@ -681,42 +709,42 @@
<source>Error retrieving config</source> <source>Error retrieving config</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/admin/config/config.component.ts</context> <context context-type="sourcefile">src/app/components/admin/config/config.component.ts</context>
<context context-type="linenumber">117</context> <context context-type="linenumber">118</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="1172622527269118932" datatype="html"> <trans-unit id="1172622527269118932" datatype="html">
<source>Invalid JSON</source> <source>Invalid JSON</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/admin/config/config.component.ts</context> <context context-type="sourcefile">src/app/components/admin/config/config.component.ts</context>
<context context-type="linenumber">143</context> <context context-type="linenumber">144</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="5103146006962696736" datatype="html"> <trans-unit id="5103146006962696736" datatype="html">
<source>Configuration updated</source> <source>Configuration updated</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/admin/config/config.component.ts</context> <context context-type="sourcefile">src/app/components/admin/config/config.component.ts</context>
<context context-type="linenumber">187</context> <context context-type="linenumber">193</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="1664963291286452273" datatype="html"> <trans-unit id="1664963291286452273" datatype="html">
<source>An error occurred updating configuration</source> <source>An error occurred updating configuration</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/admin/config/config.component.ts</context> <context context-type="sourcefile">src/app/components/admin/config/config.component.ts</context>
<context context-type="linenumber">192</context> <context context-type="linenumber">198</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="2653081282186526824" datatype="html"> <trans-unit id="2653081282186526824" datatype="html">
<source>File successfully updated</source> <source>File successfully updated</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/admin/config/config.component.ts</context> <context context-type="sourcefile">src/app/components/admin/config/config.component.ts</context>
<context context-type="linenumber">214</context> <context context-type="linenumber">220</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="5902783625859504265" datatype="html"> <trans-unit id="5902783625859504265" datatype="html">
<source>An error occurred uploading file</source> <source>An error occurred uploading file</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/admin/config/config.component.ts</context> <context context-type="sourcefile">src/app/components/admin/config/config.component.ts</context>
<context context-type="linenumber">219</context> <context context-type="linenumber">225</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="4804785061014590286" datatype="html"> <trans-unit id="4804785061014590286" datatype="html">
@@ -23,18 +23,31 @@
<div class="col"> <div class="col">
<div class="card bg-light"> <div class="card bg-light">
<div class="card-body"> <div class="card-body">
<div class="card-title d-flex align-items-center"> <div class="card-title d-flex align-items-center flex-wrap">
<h6 class="mb-0"> <h6 class="mb-0">
{{option.title}} {{option.title}}
</h6> </h6>
<a class="btn btn-sm btn-link" title="Read the documentation about this setting" i18n-title [href]="getDocsUrl(option.config_key)" target="_blank" referrerpolicy="no-referrer"> <a class="btn btn-sm btn-link" title="Read the documentation about this setting" i18n-title [href]="getDocsUrl(option.config_key)" target="_blank" referrerpolicy="no-referrer">
<i-bs name="info-circle"></i-bs> <i-bs name="info-circle"></i-bs>
</a> </a>
@if (isExternallyConfigured(option.config_key)) {
@if (isSet(option.key)) { @if (isSet(option.key)) {
<span class="badge rounded-pill bg-body-secondary text-dark fw-normal" title="This value overrides {{option.config_key}}, which is set outside Paperless." i18n-title>Overrides external</span>
} @else {
<span class="badge rounded-pill bg-body-secondary text-dark fw-normal" title="{{option.config_key}} is set outside Paperless. Enter a value here to override it." i18n-title>Set externally</span>
}
}
@if (isSet(option.key)) {
@if (isExternallyConfigured(option.config_key)) {
<button type="button" class="btn btn-sm btn-link text-danger ms-auto pe-0" title="Use the externally configured value" i18n-title (click)="resetOption(option.key)">
<i-bs class="me-1" name="x"></i-bs><ng-container i18n>Reset to external</ng-container>
</button>
} @else {
<button type="button" class="btn btn-sm btn-link text-danger ms-auto pe-0" title="Reset" i18n-title (click)="resetOption(option.key)"> <button type="button" class="btn btn-sm btn-link text-danger ms-auto pe-0" title="Reset" i18n-title (click)="resetOption(option.key)">
<i-bs class="me-1" name="x"></i-bs><ng-container i18n>Reset</ng-container> <i-bs class="me-1" name="x"></i-bs><ng-container i18n>Reset</ng-container>
</button> </button>
} }
}
</div> </div>
<div class="mb-n3"> <div class="mb-n3">
@switch (option.type) { @switch (option.type) {
@@ -163,6 +163,19 @@ describe('ConfigComponent', () => {
expect(component.configForm.get('barcodes_enabled').value).toBeNull() expect(component.configForm.get('barcodes_enabled').value).toBeNull()
}) })
it('should identify externally configured options', () => {
component.externallyConfiguredVariables = new Set([
'PAPERLESS_OCR_LANGUAGE',
])
expect(
component.isExternallyConfigured('PAPERLESS_OCR_LANGUAGE')
).toBeTruthy()
expect(
component.isExternallyConfigured('PAPERLESS_OCR_OUTPUT_TYPE')
).toBeFalsy()
})
it('should group options into sections within a category, or not', () => { it('should group options into sections within a category, or not', () => {
const sections = component.getCategorySections(ConfigCategory.OCR) const sections = component.getCategorySections(ConfigCategory.OCR)
expect(sections).toEqual([null, ConfigSection.RemoteOCR]) expect(sections).toEqual([null, ConfigSection.RemoteOCR])
@@ -69,6 +69,7 @@ export class ConfigComponent
public configForm = new FormGroup({}) public configForm = new FormGroup({})
public errors = {} public errors = {}
public externallyConfiguredVariables = new Set<string>()
get optionCategories(): string[] { get optionCategories(): string[] {
return Object.values(ConfigCategory) return Object.values(ConfigCategory)
@@ -152,6 +153,9 @@ export class ConfigComponent
} }
private initialize(config: PaperlessConfig) { private initialize(config: PaperlessConfig) {
this.externallyConfiguredVariables = new Set(
config.externally_configured_variables ?? []
)
if (!this.store) { if (!this.store) {
this.store = new BehaviorSubject(config) this.store = new BehaviorSubject(config)
@@ -162,7 +166,9 @@ export class ConfigComponent
this.configForm.patchValue(state, { emitEvent: false }) this.configForm.patchValue(state, { emitEvent: false })
}) })
this.isDirty$ = dirtyCheck(this.configForm, this.store.asObservable()) this.isDirty$ = dirtyCheck(this.configForm, this.store.asObservable(), {
excludeKeys: ['externally_configured_variables'],
})
} }
this.configForm.patchValue(config) this.configForm.patchValue(config)
@@ -227,6 +233,10 @@ export class ConfigComponent
return this.configForm.get(key).value != null return this.configForm.get(key).value != null
} }
public isExternallyConfigured(configKey: string): boolean {
return this.externallyConfiguredVariables.has(configKey)
}
public resetOption(key: string) { public resetOption(key: string) {
this.configForm.get(key).setValue(null) this.configForm.get(key).setValue(null)
} }
+1
View File
@@ -422,6 +422,7 @@ export const PaperlessConfigOptions: ConfigOption[] = [
] ]
export interface PaperlessConfig extends ObjectWithId { export interface PaperlessConfig extends ObjectWithId {
externally_configured_variables: string[]
output_type: OutputTypeConfig output_type: OutputTypeConfig
pages: number pages: number
language: string language: string
+3 -13
View File
@@ -12,7 +12,6 @@ from typing import TYPE_CHECKING
from typing import Any from typing import Any
from django.contrib.contenttypes.models import ContentType from django.contrib.contenttypes.models import ContentType
from django.core.exceptions import FieldError
from django.db.models import Case from django.db.models import Case
from django.db.models import CharField from django.db.models import CharField
from django.db.models import Count from django.db.models import Count
@@ -53,6 +52,7 @@ from documents.models import StoragePath
from documents.models import Tag from documents.models import Tag
from documents.permissions import permitted_document_ids from documents.permissions import permitted_document_ids
from documents.permissions import permitted_object_ids from documents.permissions import permitted_object_ids
from documents.versioning import annotate_effective_content
if TYPE_CHECKING: if TYPE_CHECKING:
from collections.abc import Callable from collections.abc import Callable
@@ -182,14 +182,9 @@ class TitleContentFilter(Filter):
logger.warning( logger.warning(
"Deprecated document filter parameter 'title_content' used; use `text` instead.", "Deprecated document filter parameter 'title_content' used; use `text` instead.",
) )
try: return annotate_effective_content(qs).filter(
return qs.filter(
Q(title__icontains=value) | Q(effective_content__icontains=value), Q(title__icontains=value) | Q(effective_content__icontains=value),
) )
except FieldError:
return qs.filter(
Q(title__icontains=value) | Q(content__icontains=value),
)
else: else:
return qs return qs
@@ -200,14 +195,9 @@ class EffectiveContentFilter(Filter):
value = value.strip() if isinstance(value, str) else value value = value.strip() if isinstance(value, str) else value
if not value: if not value:
return qs return qs
try: return annotate_effective_content(qs).filter(
return qs.filter(
**{f"effective_content__{self.lookup_expr}": value}, **{f"effective_content__{self.lookup_expr}": value},
) )
except FieldError:
return qs.filter(
**{f"content__{self.lookup_expr}": value},
)
@extend_schema_field(serializers.BooleanField) @extend_schema_field(serializers.BooleanField)
+3
View File
@@ -674,6 +674,9 @@ class TagSerializer(MatchingModelSerializer, OwnedObjectSerializer):
ordering = ordering or (Lower("name"),) ordering = ordering or (Lower("name"),)
children = children.order_by(*ordering) children = children.order_by(*ordering)
if not children:
return []
serializer = TagSerializer( serializer = TagSerializer(
children, children,
many=True, many=True,
@@ -35,6 +35,7 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
THEN: THEN:
- Existing config - Existing config
""" """
with patch.dict("os.environ", {}, clear=True):
response = self.client.get(self.ENDPOINT, format="json") response = self.client.get(self.ENDPOINT, format="json")
self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.status_code, status.HTTP_200_OK)
@@ -45,6 +46,7 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
response.data[0], response.data[0],
{ {
"id": 1, "id": 1,
"externally_configured_variables": [],
"output_type": None, "output_type": None,
"pages": None, "pages": None,
"language": None, "language": None,
@@ -91,6 +93,31 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
}, },
) )
def test_api_get_config_reports_external_configuration_without_values(self) -> None:
with patch.dict(
"os.environ",
{
"PAPERLESS_OCR_LANGUAGE": "eng",
"PAPERLESS_REMOTE_OCR_API_KEY": "secret-value",
"PAPERLESS_FUTURE_SETTING": "future-value",
"UNRELATED_SETTING": "unrelated-value",
},
clear=True,
):
response = self.client.get(self.ENDPOINT, format="json")
self.assertCountEqual(
response.data[0]["externally_configured_variables"],
[
"PAPERLESS_FUTURE_SETTING",
"PAPERLESS_OCR_LANGUAGE",
"PAPERLESS_REMOTE_OCR_API_KEY",
],
)
self.assertNotContains(response, "secret-value")
self.assertNotContains(response, "future-value")
self.assertNotContains(response, "UNRELATED_SETTING")
def test_api_get_ui_settings_with_config(self) -> None: def test_api_get_ui_settings_with_config(self) -> None:
""" """
GIVEN: GIVEN:
@@ -2,14 +2,12 @@ from __future__ import annotations
import datetime import datetime
from typing import TYPE_CHECKING from typing import TYPE_CHECKING
from unittest import TestCase
from unittest import mock from unittest import mock
from auditlog.models import LogEntry # type: ignore[import-untyped] from auditlog.models import LogEntry # type: ignore[import-untyped]
from django.contrib.auth.models import Permission from django.contrib.auth.models import Permission
from django.contrib.auth.models import User from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType from django.contrib.contenttypes.models import ContentType
from django.core.exceptions import FieldError
from django.core.files.uploadedfile import SimpleUploadedFile from django.core.files.uploadedfile import SimpleUploadedFile
from django.test import TestCase as DjangoTestCase from django.test import TestCase as DjangoTestCase
from django.utils import timezone from django.utils import timezone
@@ -22,6 +20,7 @@ from documents.filters import TitleContentFilter
from documents.models import Document from documents.models import Document
from documents.tests.utils import DirectoriesMixin from documents.tests.utils import DirectoriesMixin
from documents.tests.utils import read_streaming_response from documents.tests.utils import read_streaming_response
from documents.versioning import annotate_effective_content
from documents.views import DocumentSelectionMixin from documents.views import DocumentSelectionMixin
if TYPE_CHECKING: if TYPE_CHECKING:
@@ -892,32 +891,104 @@ class TestDocumentVersioningApi(DirectoriesMixin, APITestCase):
) )
class TestVersionAwareFilters(TestCase): class TestVersionAwareFilters(DjangoTestCase):
def test_title_content_filter_falls_back_to_content(self) -> None: """
queryset = mock.Mock() The filters annotate effective_content themselves rather than relying on
fallback_queryset = mock.Mock() the caller's queryset carrying it, so they stay version-aware on a plain
queryset.filter.side_effect = [FieldError("missing field"), fallback_queryset] Document queryset (e.g. the bulk-edit "select all matching" path).
"""
result = TitleContentFilter().filter(queryset, " latest ") def setUp(self) -> None:
super().setUp()
self.root = Document.objects.create(
title="root",
checksum="root",
mime_type="application/pdf",
content="superseded-content",
)
Document.objects.create(
title="version",
checksum="version",
mime_type="application/pdf",
root_document=self.root,
version_index=1,
content="latest-content",
)
self.unversioned = Document.objects.create(
title="unversioned",
checksum="unversioned",
mime_type="application/pdf",
content="latest-content",
)
self.assertIs(result, fallback_queryset) def test_title_content_filter_matches_latest_version_content(self) -> None:
self.assertEqual(queryset.filter.call_count, 2) result = TitleContentFilter().filter(
Document.objects.filter(root_document__isnull=True),
def test_effective_content_filter_falls_back_to_content_lookup(self) -> None:
queryset = mock.Mock()
fallback_queryset = mock.Mock()
queryset.filter.side_effect = [FieldError("missing field"), fallback_queryset]
result = EffectiveContentFilter(lookup_expr="icontains").filter(
queryset,
" latest ", " latest ",
) )
self.assertIs(result, fallback_queryset) self.assertCountEqual(
first_kwargs = queryset.filter.call_args_list[0].kwargs [doc.id for doc in result],
second_kwargs = queryset.filter.call_args_list[1].kwargs [self.root.id, self.unversioned.id],
self.assertEqual(first_kwargs, {"effective_content__icontains": "latest"}) )
self.assertEqual(second_kwargs, {"content__icontains": "latest"})
def test_effective_content_filter_matches_latest_version_content(self) -> None:
result = EffectiveContentFilter(lookup_expr="icontains").filter(
Document.objects.filter(root_document__isnull=True),
" latest ",
)
self.assertCountEqual(
[doc.id for doc in result],
[self.root.id, self.unversioned.id],
)
def test_effective_content_filter_ignores_superseded_content(self) -> None:
result = EffectiveContentFilter(lookup_expr="icontains").filter(
Document.objects.filter(root_document__isnull=True),
"superseded",
)
self.assertEqual(list(result), [])
def test_filters_reuse_an_existing_annotation(self) -> None:
"""
Annotating twice under the same alias is an error, so an already
annotated queryset (the search path) has to be left alone.
"""
annotated = annotate_effective_content(
Document.objects.filter(root_document__isnull=True),
)
self.assertIs(annotate_effective_content(annotated), annotated)
result = EffectiveContentFilter(lookup_expr="icontains").filter(
annotated,
"latest",
)
self.assertCountEqual(
[doc.id for doc in result],
[self.root.id, self.unversioned.id],
)
def test_bulk_selection_does_not_match_superseded_content(self) -> None:
"""
Bulk edit's "select all matching" builds its own queryset, so before
the filters annotated for themselves it matched the root document's
superseded content -- selecting documents the list view, filtered by
the same term, does not show.
"""
user = User.objects.create_superuser(username="bulk_selection")
selected = DocumentSelectionMixin()._resolve_document_ids(
user=user,
validated_data={
"all": True,
"filters": {"content__icontains": "superseded"},
},
)
self.assertEqual(selected, [])
def test_effective_content_filter_returns_input_for_empty_values(self) -> None: def test_effective_content_filter_returns_input_for_empty_values(self) -> None:
queryset = mock.Mock() queryset = mock.Mock()
+23
View File
@@ -1947,6 +1947,29 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
self.assertEqual(len(response.data["documents"]), 1) self.assertEqual(len(response.data["documents"]), 1)
self.assertEqual(response.data["documents"][0]["id"], title_match.id) self.assertEqual(response.data["documents"][0]["id"], title_match.id)
def test_global_search_returns_latest_version_content(self) -> None:
root = Document.objects.create(
title="bank statement",
content="superseded content",
checksum="GSV1",
pk=23,
)
Document.objects.create(
title="bank statement v2",
content="latest content",
checksum="GSV2",
pk=24,
root_document=root,
version_index=1,
)
self.client.force_authenticate(self.user)
response = self.client.get("/api/search/?query=bank&db_only=true")
self.assertEqual(response.status_code, status.HTTP_200_OK)
returned = {doc["id"]: doc["content"] for doc in response.data["documents"]}
self.assertEqual(returned.get(root.id), "latest content")
def test_global_search_filters_owned_mail_objects(self) -> None: def test_global_search_filters_owned_mail_objects(self) -> None:
user1 = User.objects.create_user("mail-search-user") user1 = User.objects.create_user("mail-search-user")
user2 = User.objects.create_user("other-mail-search-user") user2 = User.objects.create_user("other-mail-search-user")
-33
View File
@@ -32,7 +32,6 @@ from documents.signals.handlers import update_llm_suggestions_cache
from documents.tests.utils import DirectoriesMixin from documents.tests.utils import DirectoriesMixin
from documents.tests.utils import read_streaming_response from documents.tests.utils import read_streaming_response
from paperless.models import ApplicationConfiguration from paperless.models import ApplicationConfiguration
from paperless_ai.exceptions import LLMProviderError
from paperless_ai.exceptions import LLMTimeoutError from paperless_ai.exceptions import LLMTimeoutError
@@ -738,38 +737,6 @@ class TestAISuggestions(DirectoriesMixin, TestCase):
get_llm_suggestion_cache(self.document.pk, backend="openai-like"), get_llm_suggestion_cache(self.document.pk, backend="openai-like"),
) )
@patch("documents.views.get_ai_document_classification")
@override_settings(
AI_ENABLED=True,
LLM_BACKEND="openai-like",
)
def test_ai_suggestions_with_llm_provider_error(
self,
mock_get_ai_classification,
) -> None:
mock_get_ai_classification.side_effect = LLMProviderError(
"confidential provider response",
)
self.client.force_login(user=self.user)
response = self.client.get(
f"/api/documents/{self.document.pk}/ai_suggestions/",
)
self.assertEqual(response.status_code, status.HTTP_502_BAD_GATEWAY)
self.assertEqual(
response.json(),
{
"ai": [
"AI backend rejected the request. Check logs for details.",
],
},
)
self.assertNotIn("confidential provider response", response.content.decode())
self.assertIsNone(
get_llm_suggestion_cache(self.document.pk, backend="openai-like"),
)
@patch("documents.views.get_ai_document_classification") @patch("documents.views.get_ai_document_classification")
@override_settings( @override_settings(
AI_ENABLED=True, AI_ENABLED=True,
+6 -3
View File
@@ -27,10 +27,13 @@ def versions_newest_first(documents: QuerySet[Document]) -> QuerySet[Document]:
def annotate_effective_content(documents: QuerySet[Document]) -> QuerySet[Document]: def annotate_effective_content(documents: QuerySet[Document]) -> QuerySet[Document]:
""" """
Annotates documents with the content of their newest version, falling back Annotates documents with the content of their newest version unless the
to their own, so get_effective_content() can answer from the row rather queryset already carries the annotation, falling back to their own, so
than querying for the versions of each document get_effective_content() can answer from the row rather than querying for
the versions of each document.
""" """
if "effective_content" in documents.query.annotations:
return documents
return documents.annotate( return documents.annotate(
effective_content=Coalesce( effective_content=Coalesce(
Subquery( Subquery(
+7 -18
View File
@@ -232,6 +232,7 @@ from documents.tasks import train_classifier
from documents.tasks import update_document_parent_tags from documents.tasks import update_document_parent_tags
from documents.utils import get_boolean from documents.utils import get_boolean
from documents.versioning import VersionResolutionError from documents.versioning import VersionResolutionError
from documents.versioning import annotate_effective_content
from documents.versioning import get_latest_version_for_root from documents.versioning import get_latest_version_for_root
from documents.versioning import get_request_version_param from documents.versioning import get_request_version_param
from documents.versioning import get_root_document from documents.versioning import get_root_document
@@ -251,7 +252,6 @@ from paperless.views import StandardPagination
from paperless_ai.ai_classifier import get_ai_document_classification from paperless_ai.ai_classifier import get_ai_document_classification
from paperless_ai.ai_classifier import get_llm_output_language from paperless_ai.ai_classifier import get_llm_output_language
from paperless_ai.chat import stream_chat_with_documents from paperless_ai.chat import stream_chat_with_documents
from paperless_ai.exceptions import LLMProviderError
from paperless_ai.exceptions import LLMTimeoutError from paperless_ai.exceptions import LLMTimeoutError
from paperless_ai.matching import extract_unmatched_names from paperless_ai.matching import extract_unmatched_names
from paperless_ai.matching import match_correspondents_by_name from paperless_ai.matching import match_correspondents_by_name
@@ -1603,22 +1603,6 @@ class DocumentViewSet(
{"ai": [_("AI backend request timed out.")]}, {"ai": [_("AI backend request timed out.")]},
status=status.HTTP_503_SERVICE_UNAVAILABLE, status=status.HTTP_503_SERVICE_UNAVAILABLE,
) )
except LLMProviderError:
logger.exception(
"AI backend rejected the request for document %s",
doc.pk,
)
return Response(
{
"ai": [
_(
"AI backend rejected the request. "
"Check logs for details.",
),
],
},
status=status.HTTP_502_BAD_GATEWAY,
)
set_llm_suggestions_cache( set_llm_suggestions_cache(
doc.pk, doc.pk,
llm_suggestions, llm_suggestions,
@@ -3649,8 +3633,13 @@ class GlobalSearchView(PassUserMixin):
OBJECT_LIMIT = 3 OBJECT_LIMIT = 3
docs = [] docs = []
if request.user.has_perm("documents.view_document"): if request.user.has_perm("documents.view_document"):
all_docs = Document.objects.filter( # Never more than OBJECT_LIMIT rows come back here, so annotating
# is cheap -- and without it these results show the root
# document's superseded content.
all_docs = annotate_effective_content(
Document.objects.filter(
id__in=permitted_document_ids(request.user), id__in=permitted_document_ids(request.user),
),
) )
if db_only: if db_only:
docs = all_docs.filter(title__icontains=query)[:OBJECT_LIMIT] docs = all_docs.filter(title__icontains=query)[:OBJECT_LIMIT]
+29 -29
View File
@@ -2,7 +2,7 @@ msgid ""
msgstr "" msgstr ""
"Project-Id-Version: paperless-ngx\n" "Project-Id-Version: paperless-ngx\n"
"Report-Msgid-Bugs-To: \n" "Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2026-09-07 20:47+0000\n" "POT-Creation-Date: 2026-09-08 15:56+0000\n"
"PO-Revision-Date: 2022-02-17 04:17\n" "PO-Revision-Date: 2022-02-17 04:17\n"
"Last-Translator: \n" "Last-Translator: \n"
"Language-Team: English\n" "Language-Team: English\n"
@@ -21,39 +21,39 @@ msgstr ""
msgid "Documents" msgid "Documents"
msgstr "" msgstr ""
#: documents/filters.py:473 #: documents/filters.py:463
msgid "Value must be valid JSON." msgid "Value must be valid JSON."
msgstr "" msgstr ""
#: documents/filters.py:492 #: documents/filters.py:482
msgid "Invalid custom field query expression" msgid "Invalid custom field query expression"
msgstr "" msgstr ""
#: documents/filters.py:502 #: documents/filters.py:492
msgid "Invalid expression list. Must be nonempty." msgid "Invalid expression list. Must be nonempty."
msgstr "" msgstr ""
#: documents/filters.py:523 #: documents/filters.py:513
msgid "Invalid logical operator {op!r}" msgid "Invalid logical operator {op!r}"
msgstr "" msgstr ""
#: documents/filters.py:537 #: documents/filters.py:527
msgid "Maximum number of query conditions exceeded." msgid "Maximum number of query conditions exceeded."
msgstr "" msgstr ""
#: documents/filters.py:601 #: documents/filters.py:591
msgid "{name!r} is not a valid custom field." msgid "{name!r} is not a valid custom field."
msgstr "" msgstr ""
#: documents/filters.py:638 #: documents/filters.py:628
msgid "{data_type} does not support query expr {expr!r}." msgid "{data_type} does not support query expr {expr!r}."
msgstr "" msgstr ""
#: documents/filters.py:757 documents/models.py:136 #: documents/filters.py:747 documents/models.py:136
msgid "Maximum nesting depth exceeded." msgid "Maximum nesting depth exceeded."
msgstr "" msgstr ""
#: documents/filters.py:1119 #: documents/filters.py:1109
msgid "Custom field not found" msgid "Custom field not found"
msgstr "" msgstr ""
@@ -1631,49 +1631,49 @@ msgstr ""
msgid "workflow runs" msgid "workflow runs"
msgstr "" msgstr ""
#: documents/serialisers.py:524 documents/serialisers.py:878 #: documents/serialisers.py:524 documents/serialisers.py:881
#: documents/serialisers.py:2838 documents/views.py:314 documents/views.py:2624 #: documents/serialisers.py:2841 documents/views.py:315 documents/views.py:2625
#: paperless_mail/serialisers.py:156 #: paperless_mail/serialisers.py:156
msgid "Insufficient permissions." msgid "Insufficient permissions."
msgstr "" msgstr ""
#: documents/serialisers.py:714 #: documents/serialisers.py:717
msgid "Invalid color." msgid "Invalid color."
msgstr "" msgstr ""
#: documents/serialisers.py:2315 #: documents/serialisers.py:2318
#, python-format #, python-format
msgid "File type %(type)s not supported" msgid "File type %(type)s not supported"
msgstr "" msgstr ""
#: documents/serialisers.py:2359 #: documents/serialisers.py:2362
#, python-format #, python-format
msgid "Custom field id must be an integer: %(id)s" msgid "Custom field id must be an integer: %(id)s"
msgstr "" msgstr ""
#: documents/serialisers.py:2366 #: documents/serialisers.py:2369
#, python-format #, python-format
msgid "Custom field with id %(id)s does not exist" msgid "Custom field with id %(id)s does not exist"
msgstr "" msgstr ""
#: documents/serialisers.py:2383 documents/serialisers.py:2393 #: documents/serialisers.py:2386 documents/serialisers.py:2396
msgid "" msgid ""
"Custom fields must be a list of integers or an object mapping ids to values." "Custom fields must be a list of integers or an object mapping ids to values."
msgstr "" msgstr ""
#: documents/serialisers.py:2388 #: documents/serialisers.py:2391
msgid "Some custom fields don't exist or were specified twice." msgid "Some custom fields don't exist or were specified twice."
msgstr "" msgstr ""
#: documents/serialisers.py:2535 #: documents/serialisers.py:2538
msgid "Invalid variable detected." msgid "Invalid variable detected."
msgstr "" msgstr ""
#: documents/serialisers.py:2894 #: documents/serialisers.py:2897
msgid "Duplicate document identifiers are not allowed." msgid "Duplicate document identifiers are not allowed."
msgstr "" msgstr ""
#: documents/serialisers.py:2924 documents/views.py:4626 #: documents/serialisers.py:2927 documents/views.py:4632
#, python-format #, python-format
msgid "Documents not found: %(ids)s" msgid "Documents not found: %(ids)s"
msgstr "" msgstr ""
@@ -1941,36 +1941,36 @@ msgstr ""
msgid "Unable to parse URI {value}" msgid "Unable to parse URI {value}"
msgstr "" msgstr ""
#: documents/views.py:307 documents/views.py:2621 #: documents/views.py:308 documents/views.py:2622
msgid "Invalid more_like_id" msgid "Invalid more_like_id"
msgstr "" msgstr ""
#: documents/views.py:1591 #: documents/views.py:1592
msgid "Invalid AI configuration." msgid "Invalid AI configuration."
msgstr "" msgstr ""
#: documents/views.py:1602 #: documents/views.py:1603
msgid "AI backend request timed out." msgid "AI backend request timed out."
msgstr "" msgstr ""
#: documents/views.py:2446 documents/views.py:2767 #: documents/views.py:2447 documents/views.py:2768
msgid "Specify only one of text, title_search, query, or more_like_id." msgid "Specify only one of text, title_search, query, or more_like_id."
msgstr "" msgstr ""
#: documents/views.py:4639 #: documents/views.py:4645
#, python-format #, python-format
msgid "Insufficient permissions to share document %(id)s." msgid "Insufficient permissions to share document %(id)s."
msgstr "" msgstr ""
#: documents/views.py:4685 #: documents/views.py:4691
msgid "Bundle is already being processed." msgid "Bundle is already being processed."
msgstr "" msgstr ""
#: documents/views.py:4749 #: documents/views.py:4755
msgid "The share link bundle is still being prepared. Please try again later." msgid "The share link bundle is still being prepared. Please try again later."
msgstr "" msgstr ""
#: documents/views.py:4763 #: documents/views.py:4769
msgid "The share link bundle is unavailable." msgid "The share link bundle is unavailable."
msgstr "" msgstr ""
+8
View File
@@ -1,4 +1,5 @@
import logging import logging
import os
from io import BytesIO from io import BytesIO
import magic import magic
@@ -212,6 +213,7 @@ class ProfileSerializer(PasswordValidationMixin, serializers.ModelSerializer[Use
class ApplicationConfigurationSerializer( class ApplicationConfigurationSerializer(
serializers.ModelSerializer[ApplicationConfiguration], serializers.ModelSerializer[ApplicationConfiguration],
): ):
externally_configured_variables = serializers.SerializerMethodField()
user_args = serializers.JSONField(binary=True, allow_null=True) user_args = serializers.JSONField(binary=True, allow_null=True)
barcode_tag_mapping = serializers.JSONField(binary=True, allow_null=True) barcode_tag_mapping = serializers.JSONField(binary=True, allow_null=True)
llm_api_key = ObfuscatedPasswordField( llm_api_key = ObfuscatedPasswordField(
@@ -227,6 +229,12 @@ class ApplicationConfigurationSerializer(
OBFUSCATED_FIELDS = ("llm_api_key", "remote_ocr_api_key") OBFUSCATED_FIELDS = ("llm_api_key", "remote_ocr_api_key")
def get_externally_configured_variables(
self,
instance: ApplicationConfiguration,
) -> list[str]:
return sorted(name for name in os.environ if name.startswith("PAPERLESS_"))
def run_validation(self, data): def run_validation(self, data):
# Empty strings treated as None to avoid unexpected behavior # Empty strings treated as None to avoid unexpected behavior
if "user_args" in data and data["user_args"] == "": if "user_args" in data and data["user_args"] == "":
+3 -19
View File
@@ -22,7 +22,6 @@ from paperless.network import validate_outbound_http_url
from paperless_ai.base_model import ClassificationSuggestions from paperless_ai.base_model import ClassificationSuggestions
from paperless_ai.base_model import DocumentClassifierSchema from paperless_ai.base_model import DocumentClassifierSchema
from paperless_ai.base_model import model_to_classification_suggestions from paperless_ai.base_model import model_to_classification_suggestions
from paperless_ai.exceptions import LLMProviderError
from paperless_ai.exceptions import LLMTimeoutError from paperless_ai.exceptions import LLMTimeoutError
logger = logging.getLogger("paperless_ai.client") logger = logging.getLogger("paperless_ai.client")
@@ -133,7 +132,7 @@ class AIClient:
from llama_index.core.llms import ChatMessage from llama_index.core.llms import ChatMessage
if self.settings.llm_backend == LLMBackend.OLLAMA: if self.settings.llm_backend == LLMBackend.OLLAMA:
with self._normalize_errors(): with self._normalize_timeouts():
result = self.llm.chat( result = self.llm.chat(
[ChatMessage(role="user", content=prompt)], [ChatMessage(role="user", content=prompt)],
format=DocumentClassifierSchema.model_json_schema(), format=DocumentClassifierSchema.model_json_schema(),
@@ -154,7 +153,7 @@ class AIClient:
content=f"{prompt}\n\n" content=f"{prompt}\n\n"
f"Answer by calling the {tool.metadata.name} tool. Do not write the answer as text.", f"Answer by calling the {tool.metadata.name} tool. Do not write the answer as text.",
) )
with self._normalize_errors(): with self._normalize_timeouts():
result = self.llm.chat_with_tools( result = self.llm.chat_with_tools(
tools=[tool], tools=[tool],
user_msg=user_msg, user_msg=user_msg,
@@ -174,7 +173,7 @@ class AIClient:
) )
@contextmanager @contextmanager
def _normalize_errors(self) -> Iterator[None]: def _normalize_timeouts(self) -> Iterator[None]:
try: try:
yield yield
except httpx.TimeoutException as exc: except httpx.TimeoutException as exc:
@@ -182,23 +181,8 @@ class AIClient:
except Exception as exc: except Exception as exc:
if self._is_openai_timeout(exc): if self._is_openai_timeout(exc):
raise LLMTimeoutError from exc raise LLMTimeoutError from exc
if self._is_provider_error(exc):
raise LLMProviderError from exc
raise raise
def _is_provider_error(self, exc: Exception) -> bool:
if self.settings.llm_backend == LLMBackend.OLLAMA:
from ollama import ResponseError
return isinstance(exc, ResponseError)
if self.settings.llm_backend == LLMBackend.OPENAI_LIKE:
from openai import APIStatusError
return isinstance(exc, APIStatusError)
return False
def _is_openai_timeout(self, exc: Exception) -> bool: def _is_openai_timeout(self, exc: Exception) -> bool:
if self.settings.llm_backend != LLMBackend.OPENAI_LIKE: if self.settings.llm_backend != LLMBackend.OPENAI_LIKE:
return False return False
-4
View File
@@ -1,6 +1,2 @@
class LLMTimeoutError(Exception): class LLMTimeoutError(Exception):
pass pass
class LLMProviderError(Exception):
"""The LLM backend rejected the request."""
-48
View File
@@ -4,7 +4,6 @@ from unittest.mock import MagicMock
from unittest.mock import patch from unittest.mock import patch
import httpx import httpx
import ollama
import openai import openai
import pytest import pytest
from llama_index.core.llms.llm import ToolSelection from llama_index.core.llms.llm import ToolSelection
@@ -12,7 +11,6 @@ from llama_index.core.llms.llm import ToolSelection
from paperless_ai.client import LLM_SYSTEM_PROMPT from paperless_ai.client import LLM_SYSTEM_PROMPT
from paperless_ai.client import PLACEHOLDER_API_KEY from paperless_ai.client import PLACEHOLDER_API_KEY
from paperless_ai.client import AIClient from paperless_ai.client import AIClient
from paperless_ai.exceptions import LLMProviderError
from paperless_ai.exceptions import LLMTimeoutError from paperless_ai.exceptions import LLMTimeoutError
@@ -216,52 +214,6 @@ def test_run_llm_query_openai_timeout_raises_local_error(
client.run_llm_query("test_prompt") client.run_llm_query("test_prompt")
def test_run_llm_query_openai_status_error_raises_provider_error(
mock_ai_config,
mock_openai_llm,
):
mock_ai_config.llm_backend = "openai-like"
mock_ai_config.llm_model = "test_model"
mock_ai_config.llm_endpoint = "http://test-url"
request = httpx.Request("POST", "http://test-url/v1/chat/completions")
body = {"error": {"message": "Thinking mode does not support this tool_choice"}}
mock_openai_llm.return_value.chat_with_tools.side_effect = openai.BadRequestError(
"Error code: 400",
response=httpx.Response(400, request=request, json=body),
body=body,
)
client = AIClient()
with pytest.raises(LLMProviderError) as exc_info:
client.run_llm_query("test_prompt")
assert str(exc_info.value) == ""
assert isinstance(exc_info.value.__cause__, openai.BadRequestError)
def test_run_llm_query_ollama_response_error_raises_provider_error(
mock_ai_config,
mock_ollama_llm,
):
mock_ai_config.llm_backend = "ollama"
mock_ai_config.llm_model = "test_model"
mock_ai_config.llm_endpoint = "http://test-url"
response_error = ollama.ResponseError(
"confidential provider response",
status_code=400,
)
mock_ollama_llm.return_value.chat.side_effect = response_error
client = AIClient()
with pytest.raises(LLMProviderError) as exc_info:
client.run_llm_query("test_prompt")
assert str(exc_info.value) == ""
assert exc_info.value.__cause__ is response_error
def test_run_llm_query_httpx_timeout_raises_local_error( def test_run_llm_query_httpx_timeout_raises_local_error(
mock_ai_config, mock_ai_config,
mock_ollama_llm, mock_ollama_llm,