Compare commits

...
Author SHA1 Message Date
stumpylog 1f8cf4cd6e Fix: consolidate and extend unicode NFC normalization for filenames and matching
Consolidates the scattered unicodedata.normalize("NFC", ...) calls introduced
by my earlier filename/path normalization work into a single
documents.utils.normalize_unicode() helper, and closes several gaps where
NFD-normalized filenames could still slip through unnormalized:

- Document.get_public_filename() now normalizes, fixing exported filenames
  built from an NFD title/correspondent name (the default, non-format export
  path was not covered by the earlier fix).
- DocumentViewSet.update_version() now normalizes the uploaded filename,
  matching PostDocumentView's existing behavior.
- ConsumerPlugin normalizes self.filename once at consumption time, covering
  the title fallback and Document.original_filename for every document
  source (consume folder, mail, API, barcode splits).
- Workflow trigger and mail rule filename/path matching (documents/matching.py,
  paperless_mail/mail.py) now normalize the document/attachment side before
  comparing, so an NFD filename matches an NFC-typed filter pattern instead of
  silently failing to match.
- WorkflowTriggerSerializer and MailRuleSerializer normalize filter_filename/
  filter_path and the attachment include/exclude patterns once at write time,
  so the read side isn't re-normalizing an already-canonical value on every
  match.
2026-09-08 16:00:16 -07:00
GitHub Actions b989b74140 Auto translate strings 2026-09-08 15:57:05 +00:00
shamoon 5194f47291 Performance: ensure version-aware content filters on querysets (#13792) 2026-09-08 15:55:45 +00:00
GitHub Actions 714885d7a5 Auto translate strings 2026-09-08 15:32:41 +00:00
Trenton H 73e777a48c Fix: skip nested TagSerializer construction when a tag has no children (#14039)
TagSerializer.get_children() built a full nested TagSerializer(many=True)
for every tag, even when it had zero children, likely the common case for
most tags and maybe even most installs. Constructing a DRF ModelSerializer isn't
free (field introspection, deepcopy of declared fields, i18n lookups
all re-run per instantiation), so this scaled GET /api/tags/ linearly
with tag count in pure Python overhead, unrelated to SQL query count.
2026-09-08 15:30:58 +00:00
GitHub Actions e9141366bb Auto translate strings 2026-09-08 14:40:59 +00:00
shamoon 7813375123 Enhancement (QoL): surface externally-set options in Config UI (#13989) 2026-09-08 14:39:22 +00:00
shamoon 0132c7bd6e Fix pr-bot timing 2026-09-07 22:59:41 -07:00
27 changed files with 588 additions and 123 deletions
+1 -1
View File
@@ -72,7 +72,7 @@ jobs:
'You are welcome to open a new issue that describes the problem you observed in your own words.' 'You are welcome to open a new issue that describes the problem you observed in your own words.'
: 'This issue was automatically closed because it was not opened using our bug report form. ' + : 'This issue was automatically closed because it was not opened using our bug report form. ' +
'Issues have to be created through the form so that the details we need to investigate are included.\n\n' + 'Issues have to be created through the form so that the details we need to investigate are included.\n\n' +
`If the problem is still there, please [open a new issue](${newIssue}) using the form. No other action is needed here.\n\n' + `If the problem is still there, please [open a new issue](${newIssue}) using the form. No other action is needed here.\n\n` +
'If any part of your report was written by an AI tool or agent, you must say so: undisclosed AI-generated ' + 'If any part of your report was written by an AI tool or agent, you must say so: undisclosed AI-generated ' +
`contributions are a violation of our [Code of Conduct](${codeOfConduct}).`; `contributions are a violation of our [Code of Conduct](${codeOfConduct}).`;
+23 -2
View File
@@ -25,6 +25,10 @@ jobs:
pr-bot: pr-bot:
name: Automated PR Bot name: Automated PR Bot
runs-on: ubuntu-latest runs-on: ubuntu-latest
# Runs after Anti-slop so the welcome comment can see whether the PR was closed
# instead of racing it. Still runs if that job fails, so labeling is not lost.
needs: Anti-slop
if: ${{ !cancelled() }}
permissions: permissions:
contents: read contents: read
pull-requests: write pull-requests: write
@@ -99,8 +103,25 @@ jobs:
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with: with:
script: | script: |
const pr = context.payload.pull_request; const user = context.payload.pull_request.user.login;
const user = pr.user.login;
// Re-read the PR: Anti-slop may have closed and labeled it after the webhook
const { data: pr } = await github.rest.pulls.get({
owner: context.repo.owner,
repo: context.repo.repo,
pull_number: context.payload.pull_request.number,
});
if (pr.state === 'closed') {
core.info('Skipping comment: PR is already closed');
return;
}
const labels = pr.labels.map((label) => (typeof label === 'string' ? label : label.name));
if (labels.includes('ai')) {
core.info('Skipping comment: PR is labeled ai');
return;
}
const { data: members } = await github.rest.orgs.listMembers({ const { data: members } = await github.rest.orgs.listMembers({
org: 'paperless-ngx', org: 'paperless-ngx',
+39 -11
View File
@@ -501,15 +501,43 @@
<context context-type="linenumber">30</context> <context context-type="linenumber">30</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="7057826840809102816" datatype="html">
<source>This value overrides <x id="INTERPOLATION" equiv-text="{{option.config_key}}"/>, which is set outside Paperless.</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
<context context-type="linenumber">35</context>
</context-group>
</trans-unit>
<trans-unit id="7221396516204435584" datatype="html">
<source><x id="INTERPOLATION" equiv-text="{{option.config_key}}"/> is set outside Paperless. Enter a value here to override it.</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
<context context-type="linenumber">37</context>
</context-group>
</trans-unit>
<trans-unit id="8318849619178340389" datatype="html">
<source>Use the externally configured value</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
<context context-type="linenumber">42</context>
</context-group>
</trans-unit>
<trans-unit id="6032629623003430385" datatype="html">
<source>Reset to external</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
<context context-type="linenumber">43</context>
</context-group>
</trans-unit>
<trans-unit id="7808756054397155068" datatype="html"> <trans-unit id="7808756054397155068" datatype="html">
<source>Reset</source> <source>Reset</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/admin/config/config.component.html</context> <context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
<context context-type="linenumber">34</context> <context context-type="linenumber">46</context>
</context-group> </context-group>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/admin/config/config.component.html</context> <context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
<context context-type="linenumber">35</context> <context context-type="linenumber">47</context>
</context-group> </context-group>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/admin/settings/settings.component.html</context> <context context-type="sourcefile">src/app/components/admin/settings/settings.component.html</context>
@@ -520,7 +548,7 @@
<source>Enable</source> <source>Enable</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/admin/config/config.component.html</context> <context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
<context context-type="linenumber">43</context> <context context-type="linenumber">56</context>
</context-group> </context-group>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html</context> <context context-type="sourcefile">src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html</context>
@@ -531,7 +559,7 @@
<source>Cancel</source> <source>Cancel</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/admin/config/config.component.html</context> <context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
<context context-type="linenumber">67,68</context> <context context-type="linenumber">80,81</context>
</context-group> </context-group>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/admin/settings/settings.component.html</context> <context context-type="sourcefile">src/app/components/admin/settings/settings.component.html</context>
@@ -610,7 +638,7 @@
<source>Save</source> <source>Save</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/admin/config/config.component.html</context> <context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
<context context-type="linenumber">70,71</context> <context context-type="linenumber">83,84</context>
</context-group> </context-group>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/admin/settings/settings.component.html</context> <context context-type="sourcefile">src/app/components/admin/settings/settings.component.html</context>
@@ -681,42 +709,42 @@
<source>Error retrieving config</source> <source>Error retrieving config</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/admin/config/config.component.ts</context> <context context-type="sourcefile">src/app/components/admin/config/config.component.ts</context>
<context context-type="linenumber">117</context> <context context-type="linenumber">118</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="1172622527269118932" datatype="html"> <trans-unit id="1172622527269118932" datatype="html">
<source>Invalid JSON</source> <source>Invalid JSON</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/admin/config/config.component.ts</context> <context context-type="sourcefile">src/app/components/admin/config/config.component.ts</context>
<context context-type="linenumber">143</context> <context context-type="linenumber">144</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="5103146006962696736" datatype="html"> <trans-unit id="5103146006962696736" datatype="html">
<source>Configuration updated</source> <source>Configuration updated</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/admin/config/config.component.ts</context> <context context-type="sourcefile">src/app/components/admin/config/config.component.ts</context>
<context context-type="linenumber">187</context> <context context-type="linenumber">193</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="1664963291286452273" datatype="html"> <trans-unit id="1664963291286452273" datatype="html">
<source>An error occurred updating configuration</source> <source>An error occurred updating configuration</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/admin/config/config.component.ts</context> <context context-type="sourcefile">src/app/components/admin/config/config.component.ts</context>
<context context-type="linenumber">192</context> <context context-type="linenumber">198</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="2653081282186526824" datatype="html"> <trans-unit id="2653081282186526824" datatype="html">
<source>File successfully updated</source> <source>File successfully updated</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/admin/config/config.component.ts</context> <context context-type="sourcefile">src/app/components/admin/config/config.component.ts</context>
<context context-type="linenumber">214</context> <context context-type="linenumber">220</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="5902783625859504265" datatype="html"> <trans-unit id="5902783625859504265" datatype="html">
<source>An error occurred uploading file</source> <source>An error occurred uploading file</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/admin/config/config.component.ts</context> <context context-type="sourcefile">src/app/components/admin/config/config.component.ts</context>
<context context-type="linenumber">219</context> <context context-type="linenumber">225</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="4804785061014590286" datatype="html"> <trans-unit id="4804785061014590286" datatype="html">
@@ -23,17 +23,30 @@
<div class="col"> <div class="col">
<div class="card bg-light"> <div class="card bg-light">
<div class="card-body"> <div class="card-body">
<div class="card-title d-flex align-items-center"> <div class="card-title d-flex align-items-center flex-wrap">
<h6 class="mb-0"> <h6 class="mb-0">
{{option.title}} {{option.title}}
</h6> </h6>
<a class="btn btn-sm btn-link" title="Read the documentation about this setting" i18n-title [href]="getDocsUrl(option.config_key)" target="_blank" referrerpolicy="no-referrer"> <a class="btn btn-sm btn-link" title="Read the documentation about this setting" i18n-title [href]="getDocsUrl(option.config_key)" target="_blank" referrerpolicy="no-referrer">
<i-bs name="info-circle"></i-bs> <i-bs name="info-circle"></i-bs>
</a> </a>
@if (isExternallyConfigured(option.config_key)) {
@if (isSet(option.key)) {
<span class="badge rounded-pill bg-body-secondary text-dark fw-normal" title="This value overrides {{option.config_key}}, which is set outside Paperless." i18n-title>Overrides external</span>
} @else {
<span class="badge rounded-pill bg-body-secondary text-dark fw-normal" title="{{option.config_key}} is set outside Paperless. Enter a value here to override it." i18n-title>Set externally</span>
}
}
@if (isSet(option.key)) { @if (isSet(option.key)) {
<button type="button" class="btn btn-sm btn-link text-danger ms-auto pe-0" title="Reset" i18n-title (click)="resetOption(option.key)"> @if (isExternallyConfigured(option.config_key)) {
<i-bs class="me-1" name="x"></i-bs><ng-container i18n>Reset</ng-container> <button type="button" class="btn btn-sm btn-link text-danger ms-auto pe-0" title="Use the externally configured value" i18n-title (click)="resetOption(option.key)">
</button> <i-bs class="me-1" name="x"></i-bs><ng-container i18n>Reset to external</ng-container>
</button>
} @else {
<button type="button" class="btn btn-sm btn-link text-danger ms-auto pe-0" title="Reset" i18n-title (click)="resetOption(option.key)">
<i-bs class="me-1" name="x"></i-bs><ng-container i18n>Reset</ng-container>
</button>
}
} }
</div> </div>
<div class="mb-n3"> <div class="mb-n3">
@@ -163,6 +163,19 @@ describe('ConfigComponent', () => {
expect(component.configForm.get('barcodes_enabled').value).toBeNull() expect(component.configForm.get('barcodes_enabled').value).toBeNull()
}) })
it('should identify externally configured options', () => {
component.externallyConfiguredVariables = new Set([
'PAPERLESS_OCR_LANGUAGE',
])
expect(
component.isExternallyConfigured('PAPERLESS_OCR_LANGUAGE')
).toBeTruthy()
expect(
component.isExternallyConfigured('PAPERLESS_OCR_OUTPUT_TYPE')
).toBeFalsy()
})
it('should group options into sections within a category, or not', () => { it('should group options into sections within a category, or not', () => {
const sections = component.getCategorySections(ConfigCategory.OCR) const sections = component.getCategorySections(ConfigCategory.OCR)
expect(sections).toEqual([null, ConfigSection.RemoteOCR]) expect(sections).toEqual([null, ConfigSection.RemoteOCR])
@@ -69,6 +69,7 @@ export class ConfigComponent
public configForm = new FormGroup({}) public configForm = new FormGroup({})
public errors = {} public errors = {}
public externallyConfiguredVariables = new Set<string>()
get optionCategories(): string[] { get optionCategories(): string[] {
return Object.values(ConfigCategory) return Object.values(ConfigCategory)
@@ -152,6 +153,9 @@ export class ConfigComponent
} }
private initialize(config: PaperlessConfig) { private initialize(config: PaperlessConfig) {
this.externallyConfiguredVariables = new Set(
config.externally_configured_variables ?? []
)
if (!this.store) { if (!this.store) {
this.store = new BehaviorSubject(config) this.store = new BehaviorSubject(config)
@@ -162,7 +166,9 @@ export class ConfigComponent
this.configForm.patchValue(state, { emitEvent: false }) this.configForm.patchValue(state, { emitEvent: false })
}) })
this.isDirty$ = dirtyCheck(this.configForm, this.store.asObservable()) this.isDirty$ = dirtyCheck(this.configForm, this.store.asObservable(), {
excludeKeys: ['externally_configured_variables'],
})
} }
this.configForm.patchValue(config) this.configForm.patchValue(config)
@@ -227,6 +233,10 @@ export class ConfigComponent
return this.configForm.get(key).value != null return this.configForm.get(key).value != null
} }
public isExternallyConfigured(configKey: string): boolean {
return this.externallyConfiguredVariables.has(configKey)
}
public resetOption(key: string) { public resetOption(key: string) {
this.configForm.get(key).setValue(null) this.configForm.get(key).setValue(null)
} }
+1
View File
@@ -422,6 +422,7 @@ export const PaperlessConfigOptions: ConfigOption[] = [
] ]
export interface PaperlessConfig extends ObjectWithId { export interface PaperlessConfig extends ObjectWithId {
externally_configured_variables: string[]
output_type: OutputTypeConfig output_type: OutputTypeConfig
pages: number pages: number
language: string language: string
+4 -1
View File
@@ -52,6 +52,7 @@ from documents.templating.workflows import parse_w_workflow_placeholders
from documents.utils import compute_checksum from documents.utils import compute_checksum
from documents.utils import copy_basic_file_stats from documents.utils import copy_basic_file_stats
from documents.utils import copy_file_with_basic_stats from documents.utils import copy_file_with_basic_stats
from documents.utils import normalize_unicode
from documents.utils import run_subprocess from documents.utils import run_subprocess
from paperless.config import OcrConfig from paperless.config import OcrConfig
from paperless.config import RemoteOCRConfig from paperless.config import RemoteOCRConfig
@@ -201,7 +202,9 @@ class ConsumerPluginMixin:
self.renew_logging_group() self.renew_logging_group()
self.filename = self.metadata.filename or self.input_doc.original_file.name self.filename = normalize_unicode(
self.metadata.filename or self.input_doc.original_file.name,
)
def _send_progress( def _send_progress(
self, self,
+7 -17
View File
@@ -12,7 +12,6 @@ from typing import TYPE_CHECKING
from typing import Any from typing import Any
from django.contrib.contenttypes.models import ContentType from django.contrib.contenttypes.models import ContentType
from django.core.exceptions import FieldError
from django.db.models import Case from django.db.models import Case
from django.db.models import CharField from django.db.models import CharField
from django.db.models import Count from django.db.models import Count
@@ -53,6 +52,7 @@ from documents.models import StoragePath
from documents.models import Tag from documents.models import Tag
from documents.permissions import permitted_document_ids from documents.permissions import permitted_document_ids
from documents.permissions import permitted_object_ids from documents.permissions import permitted_object_ids
from documents.versioning import annotate_effective_content
if TYPE_CHECKING: if TYPE_CHECKING:
from collections.abc import Callable from collections.abc import Callable
@@ -182,14 +182,9 @@ class TitleContentFilter(Filter):
logger.warning( logger.warning(
"Deprecated document filter parameter 'title_content' used; use `text` instead.", "Deprecated document filter parameter 'title_content' used; use `text` instead.",
) )
try: return annotate_effective_content(qs).filter(
return qs.filter( Q(title__icontains=value) | Q(effective_content__icontains=value),
Q(title__icontains=value) | Q(effective_content__icontains=value), )
)
except FieldError:
return qs.filter(
Q(title__icontains=value) | Q(content__icontains=value),
)
else: else:
return qs return qs
@@ -200,14 +195,9 @@ class EffectiveContentFilter(Filter):
value = value.strip() if isinstance(value, str) else value value = value.strip() if isinstance(value, str) else value
if not value: if not value:
return qs return qs
try: return annotate_effective_content(qs).filter(
return qs.filter( **{f"effective_content__{self.lookup_expr}": value},
**{f"effective_content__{self.lookup_expr}": value}, )
)
except FieldError:
return qs.filter(
**{f"content__{self.lookup_expr}": value},
)
@extend_schema_field(serializers.BooleanField) @extend_schema_field(serializers.BooleanField)
+10 -6
View File
@@ -21,6 +21,7 @@ from documents.models import Workflow
from documents.models import WorkflowTrigger from documents.models import WorkflowTrigger
from documents.permissions import permitted_object_ids from documents.permissions import permitted_object_ids
from documents.regex import safe_regex_search from documents.regex import safe_regex_search
from documents.utils import normalize_unicode
if TYPE_CHECKING: if TYPE_CHECKING:
from django.db.models import QuerySet from django.db.models import QuerySet
@@ -311,11 +312,12 @@ def consumable_document_matches_workflow(
trigger_matched = False trigger_matched = False
# Document filename vs trigger filename # Document filename vs trigger filename
document_filename = normalize_unicode(document.original_file.name)
if ( if (
trigger.filter_filename is not None trigger.filter_filename is not None
and len(trigger.filter_filename) > 0 and len(trigger.filter_filename) > 0
and not fnmatch( and not fnmatch(
document.original_file.name.lower(), document_filename.lower(),
trigger.filter_filename.lower(), trigger.filter_filename.lower(),
) )
): ):
@@ -328,10 +330,12 @@ def consumable_document_matches_workflow(
# Document path vs trigger path # Document path vs trigger path
# Use the original_path if set, else us the original_file # Use the original_path if set, else us the original_file
match_against = ( match_against = normalize_unicode(
document.original_path str(
if document.original_path is not None document.original_path
else document.original_file if document.original_path is not None
else document.original_file,
),
) )
if ( if (
@@ -536,7 +540,7 @@ def existing_document_matches_workflow(
and len(trigger.filter_filename) > 0 and len(trigger.filter_filename) > 0
and document.original_filename is not None and document.original_filename is not None
and not fnmatch( and not fnmatch(
document.original_filename.lower(), normalize_unicode(document.original_filename).lower(),
trigger.filter_filename.lower(), trigger.filter_filename.lower(),
) )
): ):
+2 -1
View File
@@ -27,6 +27,7 @@ from django_softdelete.models import SoftDeleteModel
from documents.data_models import DocumentSource from documents.data_models import DocumentSource
from documents.parsers import get_default_file_extension from documents.parsers import get_default_file_extension
from documents.utils import normalize_unicode
class ModelWithOwner(models.Model): class ModelWithOwner(models.Model):
@@ -467,7 +468,7 @@ class Document(SoftDeleteModel, ModelWithOwner): # type: ignore[django-manager-
context_document = ( context_document = (
self.root_document if self.root_document_id is not None else self self.root_document if self.root_document_id is not None else self
) )
result = str(context_document) result = normalize_unicode(str(context_document))
if counter: if counter:
result += f"_{counter:02}" result += f"_{counter:02}"
+11
View File
@@ -87,6 +87,7 @@ from documents.regex import validate_regex_pattern
from documents.templating.filepath import validate_filepath_template_and_render from documents.templating.filepath import validate_filepath_template_and_render
from documents.templating.utils import convert_format_str_to_template_format from documents.templating.utils import convert_format_str_to_template_format
from documents.templating.workflows import validate_workflow_template from documents.templating.workflows import validate_workflow_template
from documents.utils import normalize_unicode
from documents.validators import uri_validator from documents.validators import uri_validator
from documents.validators import url_validator from documents.validators import url_validator
from documents.versioning import sort_versions_newest_first from documents.versioning import sort_versions_newest_first
@@ -674,6 +675,9 @@ class TagSerializer(MatchingModelSerializer, OwnedObjectSerializer):
ordering = ordering or (Lower("name"),) ordering = ordering or (Lower("name"),)
children = children.order_by(*ordering) children = children.order_by(*ordering)
if not children:
return []
serializer = TagSerializer( serializer = TagSerializer(
children, children,
many=True, many=True,
@@ -3117,6 +3121,13 @@ class WorkflowTriggerSerializer(serializers.ModelSerializer[WorkflowTrigger]):
): ):
attrs["filter_path"] = None attrs["filter_path"] = None
# Normalize once at write time, since these are matched against many
# documents but edited rarely
if attrs.get("filter_filename") is not None:
attrs["filter_filename"] = normalize_unicode(attrs["filter_filename"])
if attrs.get("filter_path") is not None:
attrs["filter_path"] = normalize_unicode(attrs["filter_path"])
if ( if (
"filter_custom_field_query" in attrs "filter_custom_field_query" in attrs
and attrs["filter_custom_field_query"] is not None and attrs["filter_custom_field_query"] is not None
+11 -13
View File
@@ -1,7 +1,6 @@
import logging import logging
import os import os
import re import re
import unicodedata
from collections.abc import Iterable from collections.abc import Iterable
from pathlib import PurePath from pathlib import PurePath
@@ -26,6 +25,7 @@ from documents.templating.environment import _template_environment
from documents.templating.filters import format_datetime from documents.templating.filters import format_datetime
from documents.templating.filters import get_cf_value from documents.templating.filters import get_cf_value
from documents.templating.filters import localize_date from documents.templating.filters import localize_date
from documents.utils import normalize_unicode
logger = logging.getLogger("paperless.templating") logger = logging.getLogger("paperless.templating")
@@ -42,7 +42,7 @@ class FilePathTemplate(Template):
3. Removing extra spaces before and after forward slashes 3. Removing extra spaces before and after forward slashes
4. Preserving spaces in other parts of the path 4. Preserving spaces in other parts of the path
""" """
value = unicodedata.normalize("NFC", value) value = normalize_unicode(value)
value = value.replace("\n", "").replace("\r", "") value = value.replace("\n", "").replace("\r", "")
value = re.sub(r"\s*/\s*", "/", value) value = re.sub(r"\s*/\s*", "/", value)
@@ -184,17 +184,17 @@ def get_basic_metadata_context(
""" """
return { return {
"title": pathvalidate.sanitize_filename( "title": pathvalidate.sanitize_filename(
unicodedata.normalize("NFC", document.title), normalize_unicode(document.title),
replacement_text="-", replacement_text="-",
), ),
"correspondent": pathvalidate.sanitize_filename( "correspondent": pathvalidate.sanitize_filename(
unicodedata.normalize("NFC", document.correspondent.name), normalize_unicode(document.correspondent.name),
replacement_text="-", replacement_text="-",
) )
if document.correspondent if document.correspondent
else no_value_default, else no_value_default,
"document_type": pathvalidate.sanitize_filename( "document_type": pathvalidate.sanitize_filename(
unicodedata.normalize("NFC", document.document_type.name), normalize_unicode(document.document_type.name),
replacement_text="-", replacement_text="-",
) )
if document.document_type if document.document_type
@@ -205,8 +205,7 @@ def get_basic_metadata_context(
"owner_username": document.owner.username "owner_username": document.owner.username
if document.owner if document.owner
else no_value_default, else no_value_default,
"original_name": unicodedata.normalize( "original_name": normalize_unicode(
"NFC",
PurePath(document.original_filename).with_suffix("").name, PurePath(document.original_filename).with_suffix("").name,
) )
if document.original_filename if document.original_filename
@@ -275,12 +274,12 @@ def get_tags_context(tags: Iterable[Tag]) -> dict[str, str | list[str]]:
return { return {
"tag_list": pathvalidate.sanitize_filename( "tag_list": pathvalidate.sanitize_filename(
",".join( ",".join(
sorted(unicodedata.normalize("NFC", tag.name) for tag in tags), sorted(normalize_unicode(tag.name) for tag in tags),
), ),
replacement_text="-", replacement_text="-",
), ),
# Assumed to be ordered, but a template could loop through to find what they want # Assumed to be ordered, but a template could loop through to find what they want
"tag_name_list": [unicodedata.normalize("NFC", x.name) for x in tags], "tag_name_list": [normalize_unicode(x.name) for x in tags],
} }
@@ -307,7 +306,7 @@ def get_custom_fields_context(
CustomField.FieldDataType.LONG_TEXT, CustomField.FieldDataType.LONG_TEXT,
}: }:
value = pathvalidate.sanitize_filename( value = pathvalidate.sanitize_filename(
unicodedata.normalize("NFC", field_instance.value), normalize_unicode(field_instance.value),
replacement_text="-", replacement_text="-",
) )
elif ( elif (
@@ -316,8 +315,7 @@ def get_custom_fields_context(
): ):
options = field_instance.field.extra_data["select_options"] options = field_instance.field.extra_data["select_options"]
value = pathvalidate.sanitize_filename( value = pathvalidate.sanitize_filename(
unicodedata.normalize( normalize_unicode(
"NFC",
next( next(
option["label"] option["label"]
for option in options for option in options
@@ -330,7 +328,7 @@ def get_custom_fields_context(
value = field_instance.value value = field_instance.value
field_data["custom_fields"][ field_data["custom_fields"][
pathvalidate.sanitize_filename( pathvalidate.sanitize_filename(
unicodedata.normalize("NFC", field_instance.field.name), normalize_unicode(field_instance.field.name),
replacement_text="-", replacement_text="-",
) )
] = { ] = {
+28 -1
View File
@@ -35,7 +35,8 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
THEN: THEN:
- Existing config - Existing config
""" """
response = self.client.get(self.ENDPOINT, format="json") with patch.dict("os.environ", {}, clear=True):
response = self.client.get(self.ENDPOINT, format="json")
self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.status_code, status.HTTP_200_OK)
@@ -45,6 +46,7 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
response.data[0], response.data[0],
{ {
"id": 1, "id": 1,
"externally_configured_variables": [],
"output_type": None, "output_type": None,
"pages": None, "pages": None,
"language": None, "language": None,
@@ -91,6 +93,31 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
}, },
) )
def test_api_get_config_reports_external_configuration_without_values(self) -> None:
with patch.dict(
"os.environ",
{
"PAPERLESS_OCR_LANGUAGE": "eng",
"PAPERLESS_REMOTE_OCR_API_KEY": "secret-value",
"PAPERLESS_FUTURE_SETTING": "future-value",
"UNRELATED_SETTING": "unrelated-value",
},
clear=True,
):
response = self.client.get(self.ENDPOINT, format="json")
self.assertCountEqual(
response.data[0]["externally_configured_variables"],
[
"PAPERLESS_FUTURE_SETTING",
"PAPERLESS_OCR_LANGUAGE",
"PAPERLESS_REMOTE_OCR_API_KEY",
],
)
self.assertNotContains(response, "secret-value")
self.assertNotContains(response, "future-value")
self.assertNotContains(response, "UNRELATED_SETTING")
def test_api_get_ui_settings_with_config(self) -> None: def test_api_get_ui_settings_with_config(self) -> None:
""" """
GIVEN: GIVEN:
@@ -2,14 +2,12 @@ from __future__ import annotations
import datetime import datetime
from typing import TYPE_CHECKING from typing import TYPE_CHECKING
from unittest import TestCase
from unittest import mock from unittest import mock
from auditlog.models import LogEntry # type: ignore[import-untyped] from auditlog.models import LogEntry # type: ignore[import-untyped]
from django.contrib.auth.models import Permission from django.contrib.auth.models import Permission
from django.contrib.auth.models import User from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType from django.contrib.contenttypes.models import ContentType
from django.core.exceptions import FieldError
from django.core.files.uploadedfile import SimpleUploadedFile from django.core.files.uploadedfile import SimpleUploadedFile
from django.test import TestCase as DjangoTestCase from django.test import TestCase as DjangoTestCase
from django.utils import timezone from django.utils import timezone
@@ -22,6 +20,7 @@ from documents.filters import TitleContentFilter
from documents.models import Document from documents.models import Document
from documents.tests.utils import DirectoriesMixin from documents.tests.utils import DirectoriesMixin
from documents.tests.utils import read_streaming_response from documents.tests.utils import read_streaming_response
from documents.versioning import annotate_effective_content
from documents.views import DocumentSelectionMixin from documents.views import DocumentSelectionMixin
if TYPE_CHECKING: if TYPE_CHECKING:
@@ -892,32 +891,104 @@ class TestDocumentVersioningApi(DirectoriesMixin, APITestCase):
) )
class TestVersionAwareFilters(TestCase): class TestVersionAwareFilters(DjangoTestCase):
def test_title_content_filter_falls_back_to_content(self) -> None: """
queryset = mock.Mock() The filters annotate effective_content themselves rather than relying on
fallback_queryset = mock.Mock() the caller's queryset carrying it, so they stay version-aware on a plain
queryset.filter.side_effect = [FieldError("missing field"), fallback_queryset] Document queryset (e.g. the bulk-edit "select all matching" path).
"""
result = TitleContentFilter().filter(queryset, " latest ") def setUp(self) -> None:
super().setUp()
self.root = Document.objects.create(
title="root",
checksum="root",
mime_type="application/pdf",
content="superseded-content",
)
Document.objects.create(
title="version",
checksum="version",
mime_type="application/pdf",
root_document=self.root,
version_index=1,
content="latest-content",
)
self.unversioned = Document.objects.create(
title="unversioned",
checksum="unversioned",
mime_type="application/pdf",
content="latest-content",
)
self.assertIs(result, fallback_queryset) def test_title_content_filter_matches_latest_version_content(self) -> None:
self.assertEqual(queryset.filter.call_count, 2) result = TitleContentFilter().filter(
Document.objects.filter(root_document__isnull=True),
def test_effective_content_filter_falls_back_to_content_lookup(self) -> None:
queryset = mock.Mock()
fallback_queryset = mock.Mock()
queryset.filter.side_effect = [FieldError("missing field"), fallback_queryset]
result = EffectiveContentFilter(lookup_expr="icontains").filter(
queryset,
" latest ", " latest ",
) )
self.assertIs(result, fallback_queryset) self.assertCountEqual(
first_kwargs = queryset.filter.call_args_list[0].kwargs [doc.id for doc in result],
second_kwargs = queryset.filter.call_args_list[1].kwargs [self.root.id, self.unversioned.id],
self.assertEqual(first_kwargs, {"effective_content__icontains": "latest"}) )
self.assertEqual(second_kwargs, {"content__icontains": "latest"})
def test_effective_content_filter_matches_latest_version_content(self) -> None:
result = EffectiveContentFilter(lookup_expr="icontains").filter(
Document.objects.filter(root_document__isnull=True),
" latest ",
)
self.assertCountEqual(
[doc.id for doc in result],
[self.root.id, self.unversioned.id],
)
def test_effective_content_filter_ignores_superseded_content(self) -> None:
result = EffectiveContentFilter(lookup_expr="icontains").filter(
Document.objects.filter(root_document__isnull=True),
"superseded",
)
self.assertEqual(list(result), [])
def test_filters_reuse_an_existing_annotation(self) -> None:
"""
Annotating twice under the same alias is an error, so an already
annotated queryset (the search path) has to be left alone.
"""
annotated = annotate_effective_content(
Document.objects.filter(root_document__isnull=True),
)
self.assertIs(annotate_effective_content(annotated), annotated)
result = EffectiveContentFilter(lookup_expr="icontains").filter(
annotated,
"latest",
)
self.assertCountEqual(
[doc.id for doc in result],
[self.root.id, self.unversioned.id],
)
def test_bulk_selection_does_not_match_superseded_content(self) -> None:
"""
Bulk edit's "select all matching" builds its own queryset, so before
the filters annotated for themselves it matched the root document's
superseded content -- selecting documents the list view, filtered by
the same term, does not show.
"""
user = User.objects.create_superuser(username="bulk_selection")
selected = DocumentSelectionMixin()._resolve_document_ids(
user=user,
validated_data={
"all": True,
"filters": {"content__icontains": "superseded"},
},
)
self.assertEqual(selected, [])
def test_effective_content_filter_returns_input_for_empty_values(self) -> None: def test_effective_content_filter_returns_input_for_empty_values(self) -> None:
queryset = mock.Mock() queryset = mock.Mock()
+23
View File
@@ -1947,6 +1947,29 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
self.assertEqual(len(response.data["documents"]), 1) self.assertEqual(len(response.data["documents"]), 1)
self.assertEqual(response.data["documents"][0]["id"], title_match.id) self.assertEqual(response.data["documents"][0]["id"], title_match.id)
def test_global_search_returns_latest_version_content(self) -> None:
root = Document.objects.create(
title="bank statement",
content="superseded content",
checksum="GSV1",
pk=23,
)
Document.objects.create(
title="bank statement v2",
content="latest content",
checksum="GSV2",
pk=24,
root_document=root,
version_index=1,
)
self.client.force_authenticate(self.user)
response = self.client.get("/api/search/?query=bank&db_only=true")
self.assertEqual(response.status_code, status.HTTP_200_OK)
returned = {doc["id"]: doc["content"] for doc in response.data["documents"]}
self.assertEqual(returned.get(root.id), "latest content")
def test_global_search_filters_owned_mail_objects(self) -> None: def test_global_search_filters_owned_mail_objects(self) -> None:
user1 = User.objects.create_user("mail-search-user") user1 = User.objects.create_user("mail-search-user")
user2 = User.objects.create_user("other-mail-search-user") user2 = User.objects.create_user("other-mail-search-user")
@@ -0,0 +1,69 @@
import unicodedata
from typing import TYPE_CHECKING
from unittest import mock
import celery.result
import pytest
from django.core.files.uploadedfile import SimpleUploadedFile
from documents.models import Document
if TYPE_CHECKING:
from documents.data_models import ConsumableDocument
@pytest.fixture()
def consume_file_mock():
with mock.patch("documents.tasks.consume_file.apply_async") as m:
m.return_value = celery.result.AsyncResult(id="test-task-id")
yield m
@pytest.fixture()
def directories(tmp_path, settings, _media_settings):
scratch = tmp_path / "scratch"
scratch.mkdir()
settings.SCRATCH_DIR = scratch
return scratch
@pytest.mark.django_db
class TestUpdateVersionNFCNormalization:
def test_nfd_filename_normalized_to_nfc(
self,
admin_client,
consume_file_mock: mock.MagicMock,
directories,
):
"""Uploaded new-version file with NFD filename must have its temp name stored as NFC."""
document = Document.objects.create(
title="Test",
content="content",
checksum="checksum",
mime_type="application/pdf",
)
nfd = unicodedata.normalize("NFD", "Rechnung März.pdf")
nfc = unicodedata.normalize("NFC", "Rechnung März.pdf")
assert nfd != nfc
uploaded = SimpleUploadedFile(
nfd,
b"%PDF-1.4 test",
content_type="application/pdf",
)
response = admin_client.post(
f"/api/documents/{document.pk}/update_version/",
{"document": uploaded},
)
assert response.status_code == 200
task_kwargs = consume_file_mock.call_args.kwargs["kwargs"]
input_doc: ConsumableDocument = task_kwargs["input_doc"]
assert input_doc.original_file.name == nfc, (
f"Expected NFC filename {nfc!r}, got {input_doc.original_file.name!r}"
)
assert unicodedata.is_normalized("NFC", input_doc.original_file.name)
@@ -0,0 +1,48 @@
import unicodedata
from datetime import date
import pytest
from documents.models import Correspondent
from documents.models import Document
@pytest.mark.django_db
class TestGetPublicFilenameNfc:
def test_normalizes_nfd_title_to_nfc(self) -> None:
nfd_title = unicodedata.normalize("NFD", "Gehaltserhöhung")
assert not unicodedata.is_normalized("NFC", nfd_title)
doc = Document(
mime_type="application/pdf",
title=nfd_title,
created=date(2025, 10, 17),
)
result = doc.get_public_filename()
assert unicodedata.is_normalized("NFC", result)
assert (
result
== "2025-10-17 "
+ unicodedata.normalize(
"NFC",
nfd_title,
)
+ ".pdf"
)
def test_normalizes_nfd_correspondent_name_to_nfc(self) -> None:
nfd_name = unicodedata.normalize("NFD", "Müller GmbH")
correspondent = Correspondent.objects.create(name=nfd_name)
doc = Document.objects.create(
mime_type="application/pdf",
title="Rechnung",
created=date(2025, 10, 17),
correspondent=correspondent,
)
result = doc.get_public_filename()
assert unicodedata.is_normalized("NFC", result)
+80
View File
@@ -0,0 +1,80 @@
import unicodedata
import pytest
from documents.data_models import ConsumableDocument
from documents.data_models import DocumentSource
from documents.matching import consumable_document_matches_workflow
from documents.matching import existing_document_matches_workflow
from documents.models import Document
from documents.models import Workflow
from documents.models import WorkflowTrigger
@pytest.mark.django_db
class TestMatchingNfcNormalization:
def test_consumable_document_filename_nfd_matches_nfc_pattern(
self,
tmp_path,
) -> None:
"""
GIVEN:
- A file on disk whose name is NFD-normalized
- A workflow trigger filename filter typed as NFC
WHEN:
- The consumable document is checked against the trigger
THEN:
- It matches, because both sides are normalized before comparing
"""
nfd_name = unicodedata.normalize("NFD", "Gehaltserhöhung.pdf")
nfc_pattern = unicodedata.normalize("NFC", "*Gehaltserhöhung*")
assert nfd_name != unicodedata.normalize("NFC", nfd_name)
file_path = tmp_path / nfd_name
file_path.write_bytes(b"%PDF-1.4 test")
document = ConsumableDocument(
source=DocumentSource.ConsumeFolder,
original_file=file_path,
)
trigger = WorkflowTrigger(
type=WorkflowTrigger.WorkflowTriggerType.CONSUMPTION,
filter_filename=nfc_pattern,
sources=[],
)
matched, reason = consumable_document_matches_workflow(document, trigger)
assert matched, reason
def test_existing_document_filename_nfd_matches_nfc_pattern(self) -> None:
"""
GIVEN:
- A Document whose original_filename is NFD-normalized (e.g. from
before normalization was applied at consumption time)
- A workflow trigger filename filter typed as NFC
WHEN:
- The document is checked against the trigger
THEN:
- It matches, because both sides are normalized before comparing
"""
nfd_name = unicodedata.normalize("NFD", "Gehaltserhöhung.pdf")
nfc_pattern = unicodedata.normalize("NFC", "*Gehaltserhöhung*")
document = Document.objects.create(
title="Test",
content="content",
checksum="checksum",
mime_type="application/pdf",
original_filename=nfd_name,
)
workflow = Workflow.objects.create(name="Test workflow", order=0)
trigger = WorkflowTrigger.objects.create(
type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED,
filter_filename=nfc_pattern,
)
workflow.triggers.add(trigger)
matched, reason = existing_document_matches_workflow(document, trigger)
assert matched, reason
@@ -0,0 +1,6 @@
from documents.utils import normalize_unicode
class TestNormalizeUnicode:
def test_none_passes_through(self) -> None:
assert normalize_unicode(None) is None
+20
View File
@@ -1,6 +1,7 @@
import hashlib import hashlib
import logging import logging
import shutil import shutil
import unicodedata
from collections.abc import Callable from collections.abc import Callable
from collections.abc import Iterable from collections.abc import Iterable
from collections.abc import Iterator from collections.abc import Iterator
@@ -31,6 +32,25 @@ def identity(iterable: Iterable[_T]) -> Iterable[_T]:
return iterable return iterable
def normalize_unicode(value: str | None) -> str | None:
"""
Normalize a string to Unicode NFC form, or return None unchanged.
This is the single normalization pass for any user- or filesystem-supplied
text that ends up in a filename, path, or is compared/matched against one
(titles, correspondent/tag/type names, uploaded filenames, workflow and
mail rule filename/path filters). Composed (NFC) and decomposed (NFD)
forms of the same visible text are different byte sequences, which breaks
exact comparisons and filesystem lookups even though the text looks
identical. Always normalize through this function rather than calling
unicodedata.normalize() directly, so every call site agrees on the same
form.
"""
if value is None:
return None
return unicodedata.normalize("NFC", value)
class QuerySetStream(Generic[_M]): class QuerySetStream(Generic[_M]):
"""Stream a QuerySet via .iterator(chunk_size=...) instead of """Stream a QuerySet via .iterator(chunk_size=...) instead of
materializing it (plus any prefetch caches) all at once, while still materializing it (plus any prefetch caches) all at once, while still
+6 -3
View File
@@ -27,10 +27,13 @@ def versions_newest_first(documents: QuerySet[Document]) -> QuerySet[Document]:
def annotate_effective_content(documents: QuerySet[Document]) -> QuerySet[Document]: def annotate_effective_content(documents: QuerySet[Document]) -> QuerySet[Document]:
""" """
Annotates documents with the content of their newest version, falling back Annotates documents with the content of their newest version unless the
to their own, so get_effective_content() can answer from the row rather queryset already carries the annotation, falling back to their own, so
than querying for the versions of each document get_effective_content() can answer from the row rather than querying for
the versions of each document.
""" """
if "effective_content" in documents.query.annotations:
return documents
return documents.annotate( return documents.annotate(
effective_content=Coalesce( effective_content=Coalesce(
Subquery( Subquery(
+11 -3
View File
@@ -231,7 +231,9 @@ from documents.tasks import sanity_check
from documents.tasks import train_classifier from documents.tasks import train_classifier
from documents.tasks import update_document_parent_tags from documents.tasks import update_document_parent_tags
from documents.utils import get_boolean from documents.utils import get_boolean
from documents.utils import normalize_unicode
from documents.versioning import VersionResolutionError from documents.versioning import VersionResolutionError
from documents.versioning import annotate_effective_content
from documents.versioning import get_latest_version_for_root from documents.versioning import get_latest_version_for_root
from documents.versioning import get_request_version_param from documents.versioning import get_request_version_param
from documents.versioning import get_root_document from documents.versioning import get_root_document
@@ -2067,6 +2069,7 @@ class DocumentViewSet(
try: try:
doc_name, doc_data = serializer.validated_data.get("document") doc_name, doc_data = serializer.validated_data.get("document")
doc_name = normalize_unicode(doc_name)
version_label = serializer.validated_data.get("version_label") version_label = serializer.validated_data.get("version_label")
t = int(mktime(datetime.now().timetuple())) t = int(mktime(datetime.now().timetuple()))
@@ -3333,7 +3336,7 @@ class PostDocumentView(GenericAPIView[Any]):
serializer.is_valid(raise_exception=True) serializer.is_valid(raise_exception=True)
doc_name, doc_data = serializer.validated_data.get("document") doc_name, doc_data = serializer.validated_data.get("document")
doc_name = normalize("NFC", doc_name) doc_name = normalize_unicode(doc_name)
correspondent_id = serializer.validated_data.get("correspondent") correspondent_id = serializer.validated_data.get("correspondent")
document_type_id = serializer.validated_data.get("document_type") document_type_id = serializer.validated_data.get("document_type")
storage_path_id = serializer.validated_data.get("storage_path") storage_path_id = serializer.validated_data.get("storage_path")
@@ -3632,8 +3635,13 @@ class GlobalSearchView(PassUserMixin):
OBJECT_LIMIT = 3 OBJECT_LIMIT = 3
docs = [] docs = []
if request.user.has_perm("documents.view_document"): if request.user.has_perm("documents.view_document"):
all_docs = Document.objects.filter( # Never more than OBJECT_LIMIT rows come back here, so annotating
id__in=permitted_document_ids(request.user), # is cheap -- and without it these results show the root
# document's superseded content.
all_docs = annotate_effective_content(
Document.objects.filter(
id__in=permitted_document_ids(request.user),
),
) )
if db_only: if db_only:
docs = all_docs.filter(title__icontains=query)[:OBJECT_LIMIT] docs = all_docs.filter(title__icontains=query)[:OBJECT_LIMIT]
+29 -29
View File
@@ -2,7 +2,7 @@ msgid ""
msgstr "" msgstr ""
"Project-Id-Version: paperless-ngx\n" "Project-Id-Version: paperless-ngx\n"
"Report-Msgid-Bugs-To: \n" "Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2026-09-07 20:47+0000\n" "POT-Creation-Date: 2026-09-08 15:56+0000\n"
"PO-Revision-Date: 2022-02-17 04:17\n" "PO-Revision-Date: 2022-02-17 04:17\n"
"Last-Translator: \n" "Last-Translator: \n"
"Language-Team: English\n" "Language-Team: English\n"
@@ -21,39 +21,39 @@ msgstr ""
msgid "Documents" msgid "Documents"
msgstr "" msgstr ""
#: documents/filters.py:473 #: documents/filters.py:463
msgid "Value must be valid JSON." msgid "Value must be valid JSON."
msgstr "" msgstr ""
#: documents/filters.py:492 #: documents/filters.py:482
msgid "Invalid custom field query expression" msgid "Invalid custom field query expression"
msgstr "" msgstr ""
#: documents/filters.py:502 #: documents/filters.py:492
msgid "Invalid expression list. Must be nonempty." msgid "Invalid expression list. Must be nonempty."
msgstr "" msgstr ""
#: documents/filters.py:523 #: documents/filters.py:513
msgid "Invalid logical operator {op!r}" msgid "Invalid logical operator {op!r}"
msgstr "" msgstr ""
#: documents/filters.py:537 #: documents/filters.py:527
msgid "Maximum number of query conditions exceeded." msgid "Maximum number of query conditions exceeded."
msgstr "" msgstr ""
#: documents/filters.py:601 #: documents/filters.py:591
msgid "{name!r} is not a valid custom field." msgid "{name!r} is not a valid custom field."
msgstr "" msgstr ""
#: documents/filters.py:638 #: documents/filters.py:628
msgid "{data_type} does not support query expr {expr!r}." msgid "{data_type} does not support query expr {expr!r}."
msgstr "" msgstr ""
#: documents/filters.py:757 documents/models.py:136 #: documents/filters.py:747 documents/models.py:136
msgid "Maximum nesting depth exceeded." msgid "Maximum nesting depth exceeded."
msgstr "" msgstr ""
#: documents/filters.py:1119 #: documents/filters.py:1109
msgid "Custom field not found" msgid "Custom field not found"
msgstr "" msgstr ""
@@ -1631,49 +1631,49 @@ msgstr ""
msgid "workflow runs" msgid "workflow runs"
msgstr "" msgstr ""
#: documents/serialisers.py:524 documents/serialisers.py:878 #: documents/serialisers.py:524 documents/serialisers.py:881
#: documents/serialisers.py:2838 documents/views.py:314 documents/views.py:2624 #: documents/serialisers.py:2841 documents/views.py:315 documents/views.py:2625
#: paperless_mail/serialisers.py:156 #: paperless_mail/serialisers.py:156
msgid "Insufficient permissions." msgid "Insufficient permissions."
msgstr "" msgstr ""
#: documents/serialisers.py:714 #: documents/serialisers.py:717
msgid "Invalid color." msgid "Invalid color."
msgstr "" msgstr ""
#: documents/serialisers.py:2315 #: documents/serialisers.py:2318
#, python-format #, python-format
msgid "File type %(type)s not supported" msgid "File type %(type)s not supported"
msgstr "" msgstr ""
#: documents/serialisers.py:2359 #: documents/serialisers.py:2362
#, python-format #, python-format
msgid "Custom field id must be an integer: %(id)s" msgid "Custom field id must be an integer: %(id)s"
msgstr "" msgstr ""
#: documents/serialisers.py:2366 #: documents/serialisers.py:2369
#, python-format #, python-format
msgid "Custom field with id %(id)s does not exist" msgid "Custom field with id %(id)s does not exist"
msgstr "" msgstr ""
#: documents/serialisers.py:2383 documents/serialisers.py:2393 #: documents/serialisers.py:2386 documents/serialisers.py:2396
msgid "" msgid ""
"Custom fields must be a list of integers or an object mapping ids to values." "Custom fields must be a list of integers or an object mapping ids to values."
msgstr "" msgstr ""
#: documents/serialisers.py:2388 #: documents/serialisers.py:2391
msgid "Some custom fields don't exist or were specified twice." msgid "Some custom fields don't exist or were specified twice."
msgstr "" msgstr ""
#: documents/serialisers.py:2535 #: documents/serialisers.py:2538
msgid "Invalid variable detected." msgid "Invalid variable detected."
msgstr "" msgstr ""
#: documents/serialisers.py:2894 #: documents/serialisers.py:2897
msgid "Duplicate document identifiers are not allowed." msgid "Duplicate document identifiers are not allowed."
msgstr "" msgstr ""
#: documents/serialisers.py:2924 documents/views.py:4626 #: documents/serialisers.py:2927 documents/views.py:4632
#, python-format #, python-format
msgid "Documents not found: %(ids)s" msgid "Documents not found: %(ids)s"
msgstr "" msgstr ""
@@ -1941,36 +1941,36 @@ msgstr ""
msgid "Unable to parse URI {value}" msgid "Unable to parse URI {value}"
msgstr "" msgstr ""
#: documents/views.py:307 documents/views.py:2621 #: documents/views.py:308 documents/views.py:2622
msgid "Invalid more_like_id" msgid "Invalid more_like_id"
msgstr "" msgstr ""
#: documents/views.py:1591 #: documents/views.py:1592
msgid "Invalid AI configuration." msgid "Invalid AI configuration."
msgstr "" msgstr ""
#: documents/views.py:1602 #: documents/views.py:1603
msgid "AI backend request timed out." msgid "AI backend request timed out."
msgstr "" msgstr ""
#: documents/views.py:2446 documents/views.py:2767 #: documents/views.py:2447 documents/views.py:2768
msgid "Specify only one of text, title_search, query, or more_like_id." msgid "Specify only one of text, title_search, query, or more_like_id."
msgstr "" msgstr ""
#: documents/views.py:4639 #: documents/views.py:4645
#, python-format #, python-format
msgid "Insufficient permissions to share document %(id)s." msgid "Insufficient permissions to share document %(id)s."
msgstr "" msgstr ""
#: documents/views.py:4685 #: documents/views.py:4691
msgid "Bundle is already being processed." msgid "Bundle is already being processed."
msgstr "" msgstr ""
#: documents/views.py:4749 #: documents/views.py:4755
msgid "The share link bundle is still being prepared. Please try again later." msgid "The share link bundle is still being prepared. Please try again later."
msgstr "" msgstr ""
#: documents/views.py:4763 #: documents/views.py:4769
msgid "The share link bundle is unavailable." msgid "The share link bundle is unavailable."
msgstr "" msgstr ""
+8
View File
@@ -1,4 +1,5 @@
import logging import logging
import os
from io import BytesIO from io import BytesIO
import magic import magic
@@ -212,6 +213,7 @@ class ProfileSerializer(PasswordValidationMixin, serializers.ModelSerializer[Use
class ApplicationConfigurationSerializer( class ApplicationConfigurationSerializer(
serializers.ModelSerializer[ApplicationConfiguration], serializers.ModelSerializer[ApplicationConfiguration],
): ):
externally_configured_variables = serializers.SerializerMethodField()
user_args = serializers.JSONField(binary=True, allow_null=True) user_args = serializers.JSONField(binary=True, allow_null=True)
barcode_tag_mapping = serializers.JSONField(binary=True, allow_null=True) barcode_tag_mapping = serializers.JSONField(binary=True, allow_null=True)
llm_api_key = ObfuscatedPasswordField( llm_api_key = ObfuscatedPasswordField(
@@ -227,6 +229,12 @@ class ApplicationConfigurationSerializer(
OBFUSCATED_FIELDS = ("llm_api_key", "remote_ocr_api_key") OBFUSCATED_FIELDS = ("llm_api_key", "remote_ocr_api_key")
def get_externally_configured_variables(
self,
instance: ApplicationConfiguration,
) -> list[str]:
return sorted(name for name in os.environ if name.startswith("PAPERLESS_"))
def run_validation(self, data): def run_validation(self, data):
# Empty strings treated as None to avoid unexpected behavior # Empty strings treated as None to avoid unexpected behavior
if "user_args" in data and data["user_args"] == "": if "user_args" in data and data["user_args"] == "":
+9 -7
View File
@@ -6,7 +6,6 @@ import socket
import ssl import ssl
import tempfile import tempfile
import traceback import traceback
import unicodedata
from datetime import date from datetime import date
from datetime import timedelta from datetime import timedelta
from fnmatch import fnmatch from fnmatch import fnmatch
@@ -45,6 +44,7 @@ from documents.models import Correspondent
from documents.models import PaperlessTask from documents.models import PaperlessTask
from documents.parsers import is_mime_type_supported from documents.parsers import is_mime_type_supported
from documents.tasks import consume_file from documents.tasks import consume_file
from documents.utils import normalize_unicode
from paperless.network import is_public_ip from paperless.network import is_public_ip
from paperless.network import resolve_hostname_ips from paperless.network import resolve_hostname_ips
from paperless_mail.models import MailAccount from paperless_mail.models import MailAccount
@@ -617,10 +617,10 @@ class MailAccountHandler(LoggingMixin):
rule: MailRule, rule: MailRule,
) -> str | None: ) -> str | None:
if rule.assign_title_from == MailRule.TitleSource.FROM_SUBJECT: if rule.assign_title_from == MailRule.TitleSource.FROM_SUBJECT:
return unicodedata.normalize("NFC", message.subject) return normalize_unicode(message.subject)
elif rule.assign_title_from == MailRule.TitleSource.FROM_FILENAME: elif rule.assign_title_from == MailRule.TitleSource.FROM_FILENAME:
return unicodedata.normalize("NFC", Path(att.filename).stem) return normalize_unicode(Path(att.filename).stem)
elif rule.assign_title_from == MailRule.TitleSource.NONE: elif rule.assign_title_from == MailRule.TitleSource.NONE:
return None return None
@@ -1004,6 +1004,8 @@ class MailAccountHandler(LoggingMixin):
consume_tasks = [] consume_tasks = []
for att in message.attachments: for att in message.attachments:
attachment_filename = normalize_unicode(att.filename)
if ( if (
att.content_disposition != "attachment" att.content_disposition != "attachment"
and rule.attachment_type and rule.attachment_type
@@ -1018,7 +1020,7 @@ class MailAccountHandler(LoggingMixin):
if not self.filename_inclusion_matches( if not self.filename_inclusion_matches(
rule.filter_attachment_filename_include, rule.filter_attachment_filename_include,
att.filename, attachment_filename,
): ):
# Force the filename and pattern to the lowercase # Force the filename and pattern to the lowercase
# as this is system dependent otherwise # as this is system dependent otherwise
@@ -1030,7 +1032,7 @@ class MailAccountHandler(LoggingMixin):
continue continue
elif self.filename_exclusion_matches( elif self.filename_exclusion_matches(
rule.filter_attachment_filename_exclude, rule.filter_attachment_filename_exclude,
att.filename, attachment_filename,
): ):
self.log.debug( self.log.debug(
f"Rule {rule}: " f"Rule {rule}: "
@@ -1064,7 +1066,7 @@ class MailAccountHandler(LoggingMixin):
) )
attachment_name = pathvalidate.sanitize_filename( attachment_name = pathvalidate.sanitize_filename(
unicodedata.normalize("NFC", att.filename), attachment_filename,
) )
if attachment_name: if attachment_name:
temp_filename = temp_dir / attachment_name temp_filename = temp_dir / attachment_name
@@ -1175,7 +1177,7 @@ class MailAccountHandler(LoggingMixin):
doc_overrides = DocumentMetadataOverrides( doc_overrides = DocumentMetadataOverrides(
title=message.subject, title=message.subject,
filename=pathvalidate.sanitize_filename( filename=pathvalidate.sanitize_filename(
unicodedata.normalize("NFC", f"{message.subject}.eml"), normalize_unicode(f"{message.subject}.eml"),
), ),
correspondent_id=correspondent.id if correspondent else None, correspondent_id=correspondent.id if correspondent else None,
document_type_id=doc_type.id if doc_type else None, document_type_id=doc_type.id if doc_type else None,
+7
View File
@@ -8,6 +8,7 @@ from documents.serialisers import CorrespondentField
from documents.serialisers import DocumentTypeField from documents.serialisers import DocumentTypeField
from documents.serialisers import OwnedObjectSerializer from documents.serialisers import OwnedObjectSerializer
from documents.serialisers import TagsField from documents.serialisers import TagsField
from documents.utils import normalize_unicode
from paperless_mail.models import MailAccount from paperless_mail.models import MailAccount
from paperless_mail.models import MailRule from paperless_mail.models import MailRule
from paperless_mail.models import ProcessedMail from paperless_mail.models import ProcessedMail
@@ -161,6 +162,12 @@ class MailRuleSerializer(OwnedObjectSerializer):
raise serializers.ValidationError("Maximum mail age is unreasonably large.") raise serializers.ValidationError("Maximum mail age is unreasonably large.")
return value return value
def validate_filter_attachment_filename_include(self, value):
return normalize_unicode(value)
def validate_filter_attachment_filename_exclude(self, value):
return normalize_unicode(value)
class ProcessedMailSerializer(OwnedObjectSerializer): class ProcessedMailSerializer(OwnedObjectSerializer):
class Meta: class Meta: