mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-09-09 11:17:58 +00:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 shamoon
|
069529203f | ||
|
shamoon
|
0bd02c0b5c |
@@ -72,7 +72,7 @@ jobs:
|
||||
'You are welcome to open a new issue that describes the problem you observed in your own words.'
|
||||
: 'This issue was automatically closed because it was not opened using our bug report form. ' +
|
||||
'Issues have to be created through the form so that the details we need to investigate are included.\n\n' +
|
||||
`If the problem is still there, please [open a new issue](${newIssue}) using the form. No other action is needed here.\n\n` +
|
||||
`If the problem is still there, please [open a new issue](${newIssue}) using the form. No other action is needed here.\n\n' +
|
||||
'If any part of your report was written by an AI tool or agent, you must say so: undisclosed AI-generated ' +
|
||||
`contributions are a violation of our [Code of Conduct](${codeOfConduct}).`;
|
||||
|
||||
|
||||
@@ -25,10 +25,6 @@ jobs:
|
||||
pr-bot:
|
||||
name: Automated PR Bot
|
||||
runs-on: ubuntu-latest
|
||||
# Runs after Anti-slop so the welcome comment can see whether the PR was closed
|
||||
# instead of racing it. Still runs if that job fails, so labeling is not lost.
|
||||
needs: Anti-slop
|
||||
if: ${{ !cancelled() }}
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
@@ -103,25 +99,8 @@ jobs:
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
const user = context.payload.pull_request.user.login;
|
||||
|
||||
// Re-read the PR: Anti-slop may have closed and labeled it after the webhook
|
||||
const { data: pr } = await github.rest.pulls.get({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
pull_number: context.payload.pull_request.number,
|
||||
});
|
||||
|
||||
if (pr.state === 'closed') {
|
||||
core.info('Skipping comment: PR is already closed');
|
||||
return;
|
||||
}
|
||||
|
||||
const labels = pr.labels.map((label) => (typeof label === 'string' ? label : label.name));
|
||||
if (labels.includes('ai')) {
|
||||
core.info('Skipping comment: PR is labeled ai');
|
||||
return;
|
||||
}
|
||||
const pr = context.payload.pull_request;
|
||||
const user = pr.user.login;
|
||||
|
||||
const { data: members } = await github.rest.orgs.listMembers({
|
||||
org: 'paperless-ngx',
|
||||
|
||||
+11
-39
@@ -501,43 +501,15 @@
|
||||
<context context-type="linenumber">30</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="7057826840809102816" datatype="html">
|
||||
<source>This value overrides <x id="INTERPOLATION" equiv-text="{{option.config_key}}"/>, which is set outside Paperless.</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
|
||||
<context context-type="linenumber">35</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="7221396516204435584" datatype="html">
|
||||
<source><x id="INTERPOLATION" equiv-text="{{option.config_key}}"/> is set outside Paperless. Enter a value here to override it.</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
|
||||
<context context-type="linenumber">37</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="8318849619178340389" datatype="html">
|
||||
<source>Use the externally configured value</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
|
||||
<context context-type="linenumber">42</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="6032629623003430385" datatype="html">
|
||||
<source>Reset to external</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
|
||||
<context context-type="linenumber">43</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="7808756054397155068" datatype="html">
|
||||
<source>Reset</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
|
||||
<context context-type="linenumber">46</context>
|
||||
<context context-type="linenumber">34</context>
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
|
||||
<context context-type="linenumber">47</context>
|
||||
<context context-type="linenumber">35</context>
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/admin/settings/settings.component.html</context>
|
||||
@@ -548,7 +520,7 @@
|
||||
<source>Enable</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
|
||||
<context context-type="linenumber">56</context>
|
||||
<context context-type="linenumber">43</context>
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html</context>
|
||||
@@ -559,7 +531,7 @@
|
||||
<source>Cancel</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
|
||||
<context context-type="linenumber">80,81</context>
|
||||
<context context-type="linenumber">67,68</context>
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/admin/settings/settings.component.html</context>
|
||||
@@ -638,7 +610,7 @@
|
||||
<source>Save</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
|
||||
<context context-type="linenumber">83,84</context>
|
||||
<context context-type="linenumber">70,71</context>
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/admin/settings/settings.component.html</context>
|
||||
@@ -709,42 +681,42 @@
|
||||
<source>Error retrieving config</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/admin/config/config.component.ts</context>
|
||||
<context context-type="linenumber">118</context>
|
||||
<context context-type="linenumber">117</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="1172622527269118932" datatype="html">
|
||||
<source>Invalid JSON</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/admin/config/config.component.ts</context>
|
||||
<context context-type="linenumber">144</context>
|
||||
<context context-type="linenumber">143</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="5103146006962696736" datatype="html">
|
||||
<source>Configuration updated</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/admin/config/config.component.ts</context>
|
||||
<context context-type="linenumber">193</context>
|
||||
<context context-type="linenumber">187</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="1664963291286452273" datatype="html">
|
||||
<source>An error occurred updating configuration</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/admin/config/config.component.ts</context>
|
||||
<context context-type="linenumber">198</context>
|
||||
<context context-type="linenumber">192</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="2653081282186526824" datatype="html">
|
||||
<source>File successfully updated</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/admin/config/config.component.ts</context>
|
||||
<context context-type="linenumber">220</context>
|
||||
<context context-type="linenumber">214</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="5902783625859504265" datatype="html">
|
||||
<source>An error occurred uploading file</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/admin/config/config.component.ts</context>
|
||||
<context context-type="linenumber">225</context>
|
||||
<context context-type="linenumber">219</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="4804785061014590286" datatype="html">
|
||||
|
||||
@@ -23,30 +23,17 @@
|
||||
<div class="col">
|
||||
<div class="card bg-light">
|
||||
<div class="card-body">
|
||||
<div class="card-title d-flex align-items-center flex-wrap">
|
||||
<div class="card-title d-flex align-items-center">
|
||||
<h6 class="mb-0">
|
||||
{{option.title}}
|
||||
</h6>
|
||||
<a class="btn btn-sm btn-link" title="Read the documentation about this setting" i18n-title [href]="getDocsUrl(option.config_key)" target="_blank" referrerpolicy="no-referrer">
|
||||
<i-bs name="info-circle"></i-bs>
|
||||
</a>
|
||||
@if (isExternallyConfigured(option.config_key)) {
|
||||
@if (isSet(option.key)) {
|
||||
<span class="badge rounded-pill bg-body-secondary text-dark fw-normal" title="This value overrides {{option.config_key}}, which is set outside Paperless." i18n-title>Overrides external</span>
|
||||
} @else {
|
||||
<span class="badge rounded-pill bg-body-secondary text-dark fw-normal" title="{{option.config_key}} is set outside Paperless. Enter a value here to override it." i18n-title>Set externally</span>
|
||||
}
|
||||
}
|
||||
@if (isSet(option.key)) {
|
||||
@if (isExternallyConfigured(option.config_key)) {
|
||||
<button type="button" class="btn btn-sm btn-link text-danger ms-auto pe-0" title="Use the externally configured value" i18n-title (click)="resetOption(option.key)">
|
||||
<i-bs class="me-1" name="x"></i-bs><ng-container i18n>Reset to external</ng-container>
|
||||
</button>
|
||||
} @else {
|
||||
<button type="button" class="btn btn-sm btn-link text-danger ms-auto pe-0" title="Reset" i18n-title (click)="resetOption(option.key)">
|
||||
<i-bs class="me-1" name="x"></i-bs><ng-container i18n>Reset</ng-container>
|
||||
</button>
|
||||
}
|
||||
<button type="button" class="btn btn-sm btn-link text-danger ms-auto pe-0" title="Reset" i18n-title (click)="resetOption(option.key)">
|
||||
<i-bs class="me-1" name="x"></i-bs><ng-container i18n>Reset</ng-container>
|
||||
</button>
|
||||
}
|
||||
</div>
|
||||
<div class="mb-n3">
|
||||
|
||||
@@ -163,19 +163,6 @@ describe('ConfigComponent', () => {
|
||||
expect(component.configForm.get('barcodes_enabled').value).toBeNull()
|
||||
})
|
||||
|
||||
it('should identify externally configured options', () => {
|
||||
component.externallyConfiguredVariables = new Set([
|
||||
'PAPERLESS_OCR_LANGUAGE',
|
||||
])
|
||||
|
||||
expect(
|
||||
component.isExternallyConfigured('PAPERLESS_OCR_LANGUAGE')
|
||||
).toBeTruthy()
|
||||
expect(
|
||||
component.isExternallyConfigured('PAPERLESS_OCR_OUTPUT_TYPE')
|
||||
).toBeFalsy()
|
||||
})
|
||||
|
||||
it('should group options into sections within a category, or not', () => {
|
||||
const sections = component.getCategorySections(ConfigCategory.OCR)
|
||||
expect(sections).toEqual([null, ConfigSection.RemoteOCR])
|
||||
|
||||
@@ -69,7 +69,6 @@ export class ConfigComponent
|
||||
public configForm = new FormGroup({})
|
||||
|
||||
public errors = {}
|
||||
public externallyConfiguredVariables = new Set<string>()
|
||||
|
||||
get optionCategories(): string[] {
|
||||
return Object.values(ConfigCategory)
|
||||
@@ -153,9 +152,6 @@ export class ConfigComponent
|
||||
}
|
||||
|
||||
private initialize(config: PaperlessConfig) {
|
||||
this.externallyConfiguredVariables = new Set(
|
||||
config.externally_configured_variables ?? []
|
||||
)
|
||||
if (!this.store) {
|
||||
this.store = new BehaviorSubject(config)
|
||||
|
||||
@@ -166,9 +162,7 @@ export class ConfigComponent
|
||||
this.configForm.patchValue(state, { emitEvent: false })
|
||||
})
|
||||
|
||||
this.isDirty$ = dirtyCheck(this.configForm, this.store.asObservable(), {
|
||||
excludeKeys: ['externally_configured_variables'],
|
||||
})
|
||||
this.isDirty$ = dirtyCheck(this.configForm, this.store.asObservable())
|
||||
}
|
||||
this.configForm.patchValue(config)
|
||||
|
||||
@@ -233,10 +227,6 @@ export class ConfigComponent
|
||||
return this.configForm.get(key).value != null
|
||||
}
|
||||
|
||||
public isExternallyConfigured(configKey: string): boolean {
|
||||
return this.externallyConfiguredVariables.has(configKey)
|
||||
}
|
||||
|
||||
public resetOption(key: string) {
|
||||
this.configForm.get(key).setValue(null)
|
||||
}
|
||||
|
||||
@@ -422,7 +422,6 @@ export const PaperlessConfigOptions: ConfigOption[] = [
|
||||
]
|
||||
|
||||
export interface PaperlessConfig extends ObjectWithId {
|
||||
externally_configured_variables: string[]
|
||||
output_type: OutputTypeConfig
|
||||
pages: number
|
||||
language: string
|
||||
|
||||
@@ -52,7 +52,6 @@ from documents.templating.workflows import parse_w_workflow_placeholders
|
||||
from documents.utils import compute_checksum
|
||||
from documents.utils import copy_basic_file_stats
|
||||
from documents.utils import copy_file_with_basic_stats
|
||||
from documents.utils import normalize_unicode
|
||||
from documents.utils import run_subprocess
|
||||
from paperless.config import OcrConfig
|
||||
from paperless.config import RemoteOCRConfig
|
||||
@@ -202,9 +201,7 @@ class ConsumerPluginMixin:
|
||||
|
||||
self.renew_logging_group()
|
||||
|
||||
self.filename = normalize_unicode(
|
||||
self.metadata.filename or self.input_doc.original_file.name,
|
||||
)
|
||||
self.filename = self.metadata.filename or self.input_doc.original_file.name
|
||||
|
||||
def _send_progress(
|
||||
self,
|
||||
|
||||
@@ -12,6 +12,7 @@ from typing import TYPE_CHECKING
|
||||
from typing import Any
|
||||
|
||||
from django.contrib.contenttypes.models import ContentType
|
||||
from django.core.exceptions import FieldError
|
||||
from django.db.models import Case
|
||||
from django.db.models import CharField
|
||||
from django.db.models import Count
|
||||
@@ -52,7 +53,6 @@ from documents.models import StoragePath
|
||||
from documents.models import Tag
|
||||
from documents.permissions import permitted_document_ids
|
||||
from documents.permissions import permitted_object_ids
|
||||
from documents.versioning import annotate_effective_content
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
@@ -182,9 +182,14 @@ class TitleContentFilter(Filter):
|
||||
logger.warning(
|
||||
"Deprecated document filter parameter 'title_content' used; use `text` instead.",
|
||||
)
|
||||
return annotate_effective_content(qs).filter(
|
||||
Q(title__icontains=value) | Q(effective_content__icontains=value),
|
||||
)
|
||||
try:
|
||||
return qs.filter(
|
||||
Q(title__icontains=value) | Q(effective_content__icontains=value),
|
||||
)
|
||||
except FieldError:
|
||||
return qs.filter(
|
||||
Q(title__icontains=value) | Q(content__icontains=value),
|
||||
)
|
||||
else:
|
||||
return qs
|
||||
|
||||
@@ -195,9 +200,14 @@ class EffectiveContentFilter(Filter):
|
||||
value = value.strip() if isinstance(value, str) else value
|
||||
if not value:
|
||||
return qs
|
||||
return annotate_effective_content(qs).filter(
|
||||
**{f"effective_content__{self.lookup_expr}": value},
|
||||
)
|
||||
try:
|
||||
return qs.filter(
|
||||
**{f"effective_content__{self.lookup_expr}": value},
|
||||
)
|
||||
except FieldError:
|
||||
return qs.filter(
|
||||
**{f"content__{self.lookup_expr}": value},
|
||||
)
|
||||
|
||||
|
||||
@extend_schema_field(serializers.BooleanField)
|
||||
|
||||
@@ -21,7 +21,6 @@ from documents.models import Workflow
|
||||
from documents.models import WorkflowTrigger
|
||||
from documents.permissions import permitted_object_ids
|
||||
from documents.regex import safe_regex_search
|
||||
from documents.utils import normalize_unicode
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from django.db.models import QuerySet
|
||||
@@ -312,12 +311,11 @@ def consumable_document_matches_workflow(
|
||||
trigger_matched = False
|
||||
|
||||
# Document filename vs trigger filename
|
||||
document_filename = normalize_unicode(document.original_file.name)
|
||||
if (
|
||||
trigger.filter_filename is not None
|
||||
and len(trigger.filter_filename) > 0
|
||||
and not fnmatch(
|
||||
document_filename.lower(),
|
||||
document.original_file.name.lower(),
|
||||
trigger.filter_filename.lower(),
|
||||
)
|
||||
):
|
||||
@@ -330,12 +328,10 @@ def consumable_document_matches_workflow(
|
||||
# Document path vs trigger path
|
||||
|
||||
# Use the original_path if set, else us the original_file
|
||||
match_against = normalize_unicode(
|
||||
str(
|
||||
document.original_path
|
||||
if document.original_path is not None
|
||||
else document.original_file,
|
||||
),
|
||||
match_against = (
|
||||
document.original_path
|
||||
if document.original_path is not None
|
||||
else document.original_file
|
||||
)
|
||||
|
||||
if (
|
||||
@@ -540,7 +536,7 @@ def existing_document_matches_workflow(
|
||||
and len(trigger.filter_filename) > 0
|
||||
and document.original_filename is not None
|
||||
and not fnmatch(
|
||||
normalize_unicode(document.original_filename).lower(),
|
||||
document.original_filename.lower(),
|
||||
trigger.filter_filename.lower(),
|
||||
)
|
||||
):
|
||||
|
||||
@@ -27,7 +27,6 @@ from django_softdelete.models import SoftDeleteModel
|
||||
|
||||
from documents.data_models import DocumentSource
|
||||
from documents.parsers import get_default_file_extension
|
||||
from documents.utils import normalize_unicode
|
||||
|
||||
|
||||
class ModelWithOwner(models.Model):
|
||||
@@ -468,7 +467,7 @@ class Document(SoftDeleteModel, ModelWithOwner): # type: ignore[django-manager-
|
||||
context_document = (
|
||||
self.root_document if self.root_document_id is not None else self
|
||||
)
|
||||
result = normalize_unicode(str(context_document))
|
||||
result = str(context_document)
|
||||
|
||||
if counter:
|
||||
result += f"_{counter:02}"
|
||||
|
||||
@@ -87,7 +87,6 @@ from documents.regex import validate_regex_pattern
|
||||
from documents.templating.filepath import validate_filepath_template_and_render
|
||||
from documents.templating.utils import convert_format_str_to_template_format
|
||||
from documents.templating.workflows import validate_workflow_template
|
||||
from documents.utils import normalize_unicode
|
||||
from documents.validators import uri_validator
|
||||
from documents.validators import url_validator
|
||||
from documents.versioning import sort_versions_newest_first
|
||||
@@ -675,9 +674,6 @@ class TagSerializer(MatchingModelSerializer, OwnedObjectSerializer):
|
||||
ordering = ordering or (Lower("name"),)
|
||||
children = children.order_by(*ordering)
|
||||
|
||||
if not children:
|
||||
return []
|
||||
|
||||
serializer = TagSerializer(
|
||||
children,
|
||||
many=True,
|
||||
@@ -3121,13 +3117,6 @@ class WorkflowTriggerSerializer(serializers.ModelSerializer[WorkflowTrigger]):
|
||||
):
|
||||
attrs["filter_path"] = None
|
||||
|
||||
# Normalize once at write time, since these are matched against many
|
||||
# documents but edited rarely
|
||||
if attrs.get("filter_filename") is not None:
|
||||
attrs["filter_filename"] = normalize_unicode(attrs["filter_filename"])
|
||||
if attrs.get("filter_path") is not None:
|
||||
attrs["filter_path"] = normalize_unicode(attrs["filter_path"])
|
||||
|
||||
if (
|
||||
"filter_custom_field_query" in attrs
|
||||
and attrs["filter_custom_field_query"] is not None
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import unicodedata
|
||||
from collections.abc import Iterable
|
||||
from pathlib import PurePath
|
||||
|
||||
@@ -25,7 +26,6 @@ from documents.templating.environment import _template_environment
|
||||
from documents.templating.filters import format_datetime
|
||||
from documents.templating.filters import get_cf_value
|
||||
from documents.templating.filters import localize_date
|
||||
from documents.utils import normalize_unicode
|
||||
|
||||
logger = logging.getLogger("paperless.templating")
|
||||
|
||||
@@ -42,7 +42,7 @@ class FilePathTemplate(Template):
|
||||
3. Removing extra spaces before and after forward slashes
|
||||
4. Preserving spaces in other parts of the path
|
||||
"""
|
||||
value = normalize_unicode(value)
|
||||
value = unicodedata.normalize("NFC", value)
|
||||
value = value.replace("\n", "").replace("\r", "")
|
||||
value = re.sub(r"\s*/\s*", "/", value)
|
||||
|
||||
@@ -184,17 +184,17 @@ def get_basic_metadata_context(
|
||||
"""
|
||||
return {
|
||||
"title": pathvalidate.sanitize_filename(
|
||||
normalize_unicode(document.title),
|
||||
unicodedata.normalize("NFC", document.title),
|
||||
replacement_text="-",
|
||||
),
|
||||
"correspondent": pathvalidate.sanitize_filename(
|
||||
normalize_unicode(document.correspondent.name),
|
||||
unicodedata.normalize("NFC", document.correspondent.name),
|
||||
replacement_text="-",
|
||||
)
|
||||
if document.correspondent
|
||||
else no_value_default,
|
||||
"document_type": pathvalidate.sanitize_filename(
|
||||
normalize_unicode(document.document_type.name),
|
||||
unicodedata.normalize("NFC", document.document_type.name),
|
||||
replacement_text="-",
|
||||
)
|
||||
if document.document_type
|
||||
@@ -205,7 +205,8 @@ def get_basic_metadata_context(
|
||||
"owner_username": document.owner.username
|
||||
if document.owner
|
||||
else no_value_default,
|
||||
"original_name": normalize_unicode(
|
||||
"original_name": unicodedata.normalize(
|
||||
"NFC",
|
||||
PurePath(document.original_filename).with_suffix("").name,
|
||||
)
|
||||
if document.original_filename
|
||||
@@ -274,12 +275,12 @@ def get_tags_context(tags: Iterable[Tag]) -> dict[str, str | list[str]]:
|
||||
return {
|
||||
"tag_list": pathvalidate.sanitize_filename(
|
||||
",".join(
|
||||
sorted(normalize_unicode(tag.name) for tag in tags),
|
||||
sorted(unicodedata.normalize("NFC", tag.name) for tag in tags),
|
||||
),
|
||||
replacement_text="-",
|
||||
),
|
||||
# Assumed to be ordered, but a template could loop through to find what they want
|
||||
"tag_name_list": [normalize_unicode(x.name) for x in tags],
|
||||
"tag_name_list": [unicodedata.normalize("NFC", x.name) for x in tags],
|
||||
}
|
||||
|
||||
|
||||
@@ -306,7 +307,7 @@ def get_custom_fields_context(
|
||||
CustomField.FieldDataType.LONG_TEXT,
|
||||
}:
|
||||
value = pathvalidate.sanitize_filename(
|
||||
normalize_unicode(field_instance.value),
|
||||
unicodedata.normalize("NFC", field_instance.value),
|
||||
replacement_text="-",
|
||||
)
|
||||
elif (
|
||||
@@ -315,7 +316,8 @@ def get_custom_fields_context(
|
||||
):
|
||||
options = field_instance.field.extra_data["select_options"]
|
||||
value = pathvalidate.sanitize_filename(
|
||||
normalize_unicode(
|
||||
unicodedata.normalize(
|
||||
"NFC",
|
||||
next(
|
||||
option["label"]
|
||||
for option in options
|
||||
@@ -328,7 +330,7 @@ def get_custom_fields_context(
|
||||
value = field_instance.value
|
||||
field_data["custom_fields"][
|
||||
pathvalidate.sanitize_filename(
|
||||
normalize_unicode(field_instance.field.name),
|
||||
unicodedata.normalize("NFC", field_instance.field.name),
|
||||
replacement_text="-",
|
||||
)
|
||||
] = {
|
||||
|
||||
@@ -35,8 +35,7 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
|
||||
THEN:
|
||||
- Existing config
|
||||
"""
|
||||
with patch.dict("os.environ", {}, clear=True):
|
||||
response = self.client.get(self.ENDPOINT, format="json")
|
||||
response = self.client.get(self.ENDPOINT, format="json")
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
|
||||
@@ -46,7 +45,6 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
|
||||
response.data[0],
|
||||
{
|
||||
"id": 1,
|
||||
"externally_configured_variables": [],
|
||||
"output_type": None,
|
||||
"pages": None,
|
||||
"language": None,
|
||||
@@ -93,31 +91,6 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
|
||||
},
|
||||
)
|
||||
|
||||
def test_api_get_config_reports_external_configuration_without_values(self) -> None:
|
||||
with patch.dict(
|
||||
"os.environ",
|
||||
{
|
||||
"PAPERLESS_OCR_LANGUAGE": "eng",
|
||||
"PAPERLESS_REMOTE_OCR_API_KEY": "secret-value",
|
||||
"PAPERLESS_FUTURE_SETTING": "future-value",
|
||||
"UNRELATED_SETTING": "unrelated-value",
|
||||
},
|
||||
clear=True,
|
||||
):
|
||||
response = self.client.get(self.ENDPOINT, format="json")
|
||||
|
||||
self.assertCountEqual(
|
||||
response.data[0]["externally_configured_variables"],
|
||||
[
|
||||
"PAPERLESS_FUTURE_SETTING",
|
||||
"PAPERLESS_OCR_LANGUAGE",
|
||||
"PAPERLESS_REMOTE_OCR_API_KEY",
|
||||
],
|
||||
)
|
||||
self.assertNotContains(response, "secret-value")
|
||||
self.assertNotContains(response, "future-value")
|
||||
self.assertNotContains(response, "UNRELATED_SETTING")
|
||||
|
||||
def test_api_get_ui_settings_with_config(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
|
||||
@@ -2,12 +2,14 @@ from __future__ import annotations
|
||||
|
||||
import datetime
|
||||
from typing import TYPE_CHECKING
|
||||
from unittest import TestCase
|
||||
from unittest import mock
|
||||
|
||||
from auditlog.models import LogEntry # type: ignore[import-untyped]
|
||||
from django.contrib.auth.models import Permission
|
||||
from django.contrib.auth.models import User
|
||||
from django.contrib.contenttypes.models import ContentType
|
||||
from django.core.exceptions import FieldError
|
||||
from django.core.files.uploadedfile import SimpleUploadedFile
|
||||
from django.test import TestCase as DjangoTestCase
|
||||
from django.utils import timezone
|
||||
@@ -20,7 +22,6 @@ from documents.filters import TitleContentFilter
|
||||
from documents.models import Document
|
||||
from documents.tests.utils import DirectoriesMixin
|
||||
from documents.tests.utils import read_streaming_response
|
||||
from documents.versioning import annotate_effective_content
|
||||
from documents.views import DocumentSelectionMixin
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -891,104 +892,32 @@ class TestDocumentVersioningApi(DirectoriesMixin, APITestCase):
|
||||
)
|
||||
|
||||
|
||||
class TestVersionAwareFilters(DjangoTestCase):
|
||||
"""
|
||||
The filters annotate effective_content themselves rather than relying on
|
||||
the caller's queryset carrying it, so they stay version-aware on a plain
|
||||
Document queryset (e.g. the bulk-edit "select all matching" path).
|
||||
"""
|
||||
class TestVersionAwareFilters(TestCase):
|
||||
def test_title_content_filter_falls_back_to_content(self) -> None:
|
||||
queryset = mock.Mock()
|
||||
fallback_queryset = mock.Mock()
|
||||
queryset.filter.side_effect = [FieldError("missing field"), fallback_queryset]
|
||||
|
||||
def setUp(self) -> None:
|
||||
super().setUp()
|
||||
self.root = Document.objects.create(
|
||||
title="root",
|
||||
checksum="root",
|
||||
mime_type="application/pdf",
|
||||
content="superseded-content",
|
||||
)
|
||||
Document.objects.create(
|
||||
title="version",
|
||||
checksum="version",
|
||||
mime_type="application/pdf",
|
||||
root_document=self.root,
|
||||
version_index=1,
|
||||
content="latest-content",
|
||||
)
|
||||
self.unversioned = Document.objects.create(
|
||||
title="unversioned",
|
||||
checksum="unversioned",
|
||||
mime_type="application/pdf",
|
||||
content="latest-content",
|
||||
)
|
||||
result = TitleContentFilter().filter(queryset, " latest ")
|
||||
|
||||
def test_title_content_filter_matches_latest_version_content(self) -> None:
|
||||
result = TitleContentFilter().filter(
|
||||
Document.objects.filter(root_document__isnull=True),
|
||||
self.assertIs(result, fallback_queryset)
|
||||
self.assertEqual(queryset.filter.call_count, 2)
|
||||
|
||||
def test_effective_content_filter_falls_back_to_content_lookup(self) -> None:
|
||||
queryset = mock.Mock()
|
||||
fallback_queryset = mock.Mock()
|
||||
queryset.filter.side_effect = [FieldError("missing field"), fallback_queryset]
|
||||
|
||||
result = EffectiveContentFilter(lookup_expr="icontains").filter(
|
||||
queryset,
|
||||
" latest ",
|
||||
)
|
||||
|
||||
self.assertCountEqual(
|
||||
[doc.id for doc in result],
|
||||
[self.root.id, self.unversioned.id],
|
||||
)
|
||||
|
||||
def test_effective_content_filter_matches_latest_version_content(self) -> None:
|
||||
result = EffectiveContentFilter(lookup_expr="icontains").filter(
|
||||
Document.objects.filter(root_document__isnull=True),
|
||||
" latest ",
|
||||
)
|
||||
|
||||
self.assertCountEqual(
|
||||
[doc.id for doc in result],
|
||||
[self.root.id, self.unversioned.id],
|
||||
)
|
||||
|
||||
def test_effective_content_filter_ignores_superseded_content(self) -> None:
|
||||
result = EffectiveContentFilter(lookup_expr="icontains").filter(
|
||||
Document.objects.filter(root_document__isnull=True),
|
||||
"superseded",
|
||||
)
|
||||
|
||||
self.assertEqual(list(result), [])
|
||||
|
||||
def test_filters_reuse_an_existing_annotation(self) -> None:
|
||||
"""
|
||||
Annotating twice under the same alias is an error, so an already
|
||||
annotated queryset (the search path) has to be left alone.
|
||||
"""
|
||||
annotated = annotate_effective_content(
|
||||
Document.objects.filter(root_document__isnull=True),
|
||||
)
|
||||
self.assertIs(annotate_effective_content(annotated), annotated)
|
||||
|
||||
result = EffectiveContentFilter(lookup_expr="icontains").filter(
|
||||
annotated,
|
||||
"latest",
|
||||
)
|
||||
|
||||
self.assertCountEqual(
|
||||
[doc.id for doc in result],
|
||||
[self.root.id, self.unversioned.id],
|
||||
)
|
||||
|
||||
def test_bulk_selection_does_not_match_superseded_content(self) -> None:
|
||||
"""
|
||||
Bulk edit's "select all matching" builds its own queryset, so before
|
||||
the filters annotated for themselves it matched the root document's
|
||||
superseded content -- selecting documents the list view, filtered by
|
||||
the same term, does not show.
|
||||
"""
|
||||
user = User.objects.create_superuser(username="bulk_selection")
|
||||
|
||||
selected = DocumentSelectionMixin()._resolve_document_ids(
|
||||
user=user,
|
||||
validated_data={
|
||||
"all": True,
|
||||
"filters": {"content__icontains": "superseded"},
|
||||
},
|
||||
)
|
||||
|
||||
self.assertEqual(selected, [])
|
||||
self.assertIs(result, fallback_queryset)
|
||||
first_kwargs = queryset.filter.call_args_list[0].kwargs
|
||||
second_kwargs = queryset.filter.call_args_list[1].kwargs
|
||||
self.assertEqual(first_kwargs, {"effective_content__icontains": "latest"})
|
||||
self.assertEqual(second_kwargs, {"content__icontains": "latest"})
|
||||
|
||||
def test_effective_content_filter_returns_input_for_empty_values(self) -> None:
|
||||
queryset = mock.Mock()
|
||||
|
||||
@@ -1947,29 +1947,6 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
|
||||
self.assertEqual(len(response.data["documents"]), 1)
|
||||
self.assertEqual(response.data["documents"][0]["id"], title_match.id)
|
||||
|
||||
def test_global_search_returns_latest_version_content(self) -> None:
|
||||
root = Document.objects.create(
|
||||
title="bank statement",
|
||||
content="superseded content",
|
||||
checksum="GSV1",
|
||||
pk=23,
|
||||
)
|
||||
Document.objects.create(
|
||||
title="bank statement v2",
|
||||
content="latest content",
|
||||
checksum="GSV2",
|
||||
pk=24,
|
||||
root_document=root,
|
||||
version_index=1,
|
||||
)
|
||||
|
||||
self.client.force_authenticate(self.user)
|
||||
|
||||
response = self.client.get("/api/search/?query=bank&db_only=true")
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
returned = {doc["id"]: doc["content"] for doc in response.data["documents"]}
|
||||
self.assertEqual(returned.get(root.id), "latest content")
|
||||
|
||||
def test_global_search_filters_owned_mail_objects(self) -> None:
|
||||
user1 = User.objects.create_user("mail-search-user")
|
||||
user2 = User.objects.create_user("other-mail-search-user")
|
||||
|
||||
@@ -1,69 +0,0 @@
|
||||
import unicodedata
|
||||
from typing import TYPE_CHECKING
|
||||
from unittest import mock
|
||||
|
||||
import celery.result
|
||||
import pytest
|
||||
from django.core.files.uploadedfile import SimpleUploadedFile
|
||||
|
||||
from documents.models import Document
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from documents.data_models import ConsumableDocument
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def consume_file_mock():
|
||||
with mock.patch("documents.tasks.consume_file.apply_async") as m:
|
||||
m.return_value = celery.result.AsyncResult(id="test-task-id")
|
||||
yield m
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def directories(tmp_path, settings, _media_settings):
|
||||
scratch = tmp_path / "scratch"
|
||||
scratch.mkdir()
|
||||
settings.SCRATCH_DIR = scratch
|
||||
return scratch
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestUpdateVersionNFCNormalization:
|
||||
def test_nfd_filename_normalized_to_nfc(
|
||||
self,
|
||||
admin_client,
|
||||
consume_file_mock: mock.MagicMock,
|
||||
directories,
|
||||
):
|
||||
"""Uploaded new-version file with NFD filename must have its temp name stored as NFC."""
|
||||
document = Document.objects.create(
|
||||
title="Test",
|
||||
content="content",
|
||||
checksum="checksum",
|
||||
mime_type="application/pdf",
|
||||
)
|
||||
|
||||
nfd = unicodedata.normalize("NFD", "Rechnung März.pdf")
|
||||
nfc = unicodedata.normalize("NFC", "Rechnung März.pdf")
|
||||
|
||||
assert nfd != nfc
|
||||
|
||||
uploaded = SimpleUploadedFile(
|
||||
nfd,
|
||||
b"%PDF-1.4 test",
|
||||
content_type="application/pdf",
|
||||
)
|
||||
response = admin_client.post(
|
||||
f"/api/documents/{document.pk}/update_version/",
|
||||
{"document": uploaded},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
|
||||
task_kwargs = consume_file_mock.call_args.kwargs["kwargs"]
|
||||
input_doc: ConsumableDocument = task_kwargs["input_doc"]
|
||||
|
||||
assert input_doc.original_file.name == nfc, (
|
||||
f"Expected NFC filename {nfc!r}, got {input_doc.original_file.name!r}"
|
||||
)
|
||||
assert unicodedata.is_normalized("NFC", input_doc.original_file.name)
|
||||
@@ -1,48 +0,0 @@
|
||||
import unicodedata
|
||||
from datetime import date
|
||||
|
||||
import pytest
|
||||
|
||||
from documents.models import Correspondent
|
||||
from documents.models import Document
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestGetPublicFilenameNfc:
|
||||
def test_normalizes_nfd_title_to_nfc(self) -> None:
|
||||
nfd_title = unicodedata.normalize("NFD", "Gehaltserhöhung")
|
||||
assert not unicodedata.is_normalized("NFC", nfd_title)
|
||||
|
||||
doc = Document(
|
||||
mime_type="application/pdf",
|
||||
title=nfd_title,
|
||||
created=date(2025, 10, 17),
|
||||
)
|
||||
|
||||
result = doc.get_public_filename()
|
||||
|
||||
assert unicodedata.is_normalized("NFC", result)
|
||||
assert (
|
||||
result
|
||||
== "2025-10-17 "
|
||||
+ unicodedata.normalize(
|
||||
"NFC",
|
||||
nfd_title,
|
||||
)
|
||||
+ ".pdf"
|
||||
)
|
||||
|
||||
def test_normalizes_nfd_correspondent_name_to_nfc(self) -> None:
|
||||
nfd_name = unicodedata.normalize("NFD", "Müller GmbH")
|
||||
correspondent = Correspondent.objects.create(name=nfd_name)
|
||||
|
||||
doc = Document.objects.create(
|
||||
mime_type="application/pdf",
|
||||
title="Rechnung",
|
||||
created=date(2025, 10, 17),
|
||||
correspondent=correspondent,
|
||||
)
|
||||
|
||||
result = doc.get_public_filename()
|
||||
|
||||
assert unicodedata.is_normalized("NFC", result)
|
||||
@@ -1,80 +0,0 @@
|
||||
import unicodedata
|
||||
|
||||
import pytest
|
||||
|
||||
from documents.data_models import ConsumableDocument
|
||||
from documents.data_models import DocumentSource
|
||||
from documents.matching import consumable_document_matches_workflow
|
||||
from documents.matching import existing_document_matches_workflow
|
||||
from documents.models import Document
|
||||
from documents.models import Workflow
|
||||
from documents.models import WorkflowTrigger
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestMatchingNfcNormalization:
|
||||
def test_consumable_document_filename_nfd_matches_nfc_pattern(
|
||||
self,
|
||||
tmp_path,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A file on disk whose name is NFD-normalized
|
||||
- A workflow trigger filename filter typed as NFC
|
||||
WHEN:
|
||||
- The consumable document is checked against the trigger
|
||||
THEN:
|
||||
- It matches, because both sides are normalized before comparing
|
||||
"""
|
||||
nfd_name = unicodedata.normalize("NFD", "Gehaltserhöhung.pdf")
|
||||
nfc_pattern = unicodedata.normalize("NFC", "*Gehaltserhöhung*")
|
||||
assert nfd_name != unicodedata.normalize("NFC", nfd_name)
|
||||
|
||||
file_path = tmp_path / nfd_name
|
||||
file_path.write_bytes(b"%PDF-1.4 test")
|
||||
|
||||
document = ConsumableDocument(
|
||||
source=DocumentSource.ConsumeFolder,
|
||||
original_file=file_path,
|
||||
)
|
||||
trigger = WorkflowTrigger(
|
||||
type=WorkflowTrigger.WorkflowTriggerType.CONSUMPTION,
|
||||
filter_filename=nfc_pattern,
|
||||
sources=[],
|
||||
)
|
||||
|
||||
matched, reason = consumable_document_matches_workflow(document, trigger)
|
||||
|
||||
assert matched, reason
|
||||
|
||||
def test_existing_document_filename_nfd_matches_nfc_pattern(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A Document whose original_filename is NFD-normalized (e.g. from
|
||||
before normalization was applied at consumption time)
|
||||
- A workflow trigger filename filter typed as NFC
|
||||
WHEN:
|
||||
- The document is checked against the trigger
|
||||
THEN:
|
||||
- It matches, because both sides are normalized before comparing
|
||||
"""
|
||||
nfd_name = unicodedata.normalize("NFD", "Gehaltserhöhung.pdf")
|
||||
nfc_pattern = unicodedata.normalize("NFC", "*Gehaltserhöhung*")
|
||||
|
||||
document = Document.objects.create(
|
||||
title="Test",
|
||||
content="content",
|
||||
checksum="checksum",
|
||||
mime_type="application/pdf",
|
||||
original_filename=nfd_name,
|
||||
)
|
||||
workflow = Workflow.objects.create(name="Test workflow", order=0)
|
||||
trigger = WorkflowTrigger.objects.create(
|
||||
type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED,
|
||||
filter_filename=nfc_pattern,
|
||||
)
|
||||
workflow.triggers.add(trigger)
|
||||
|
||||
matched, reason = existing_document_matches_workflow(document, trigger)
|
||||
|
||||
assert matched, reason
|
||||
@@ -1,6 +0,0 @@
|
||||
from documents.utils import normalize_unicode
|
||||
|
||||
|
||||
class TestNormalizeUnicode:
|
||||
def test_none_passes_through(self) -> None:
|
||||
assert normalize_unicode(None) is None
|
||||
@@ -32,6 +32,7 @@ from documents.signals.handlers import update_llm_suggestions_cache
|
||||
from documents.tests.utils import DirectoriesMixin
|
||||
from documents.tests.utils import read_streaming_response
|
||||
from paperless.models import ApplicationConfiguration
|
||||
from paperless_ai.exceptions import LLMProviderError
|
||||
from paperless_ai.exceptions import LLMTimeoutError
|
||||
|
||||
|
||||
@@ -737,6 +738,38 @@ class TestAISuggestions(DirectoriesMixin, TestCase):
|
||||
get_llm_suggestion_cache(self.document.pk, backend="openai-like"),
|
||||
)
|
||||
|
||||
@patch("documents.views.get_ai_document_classification")
|
||||
@override_settings(
|
||||
AI_ENABLED=True,
|
||||
LLM_BACKEND="openai-like",
|
||||
)
|
||||
def test_ai_suggestions_with_llm_provider_error(
|
||||
self,
|
||||
mock_get_ai_classification,
|
||||
) -> None:
|
||||
mock_get_ai_classification.side_effect = LLMProviderError(
|
||||
"confidential provider response",
|
||||
)
|
||||
|
||||
self.client.force_login(user=self.user)
|
||||
response = self.client.get(
|
||||
f"/api/documents/{self.document.pk}/ai_suggestions/",
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_502_BAD_GATEWAY)
|
||||
self.assertEqual(
|
||||
response.json(),
|
||||
{
|
||||
"ai": [
|
||||
"AI backend rejected the request. Check logs for details.",
|
||||
],
|
||||
},
|
||||
)
|
||||
self.assertNotIn("confidential provider response", response.content.decode())
|
||||
self.assertIsNone(
|
||||
get_llm_suggestion_cache(self.document.pk, backend="openai-like"),
|
||||
)
|
||||
|
||||
@patch("documents.views.get_ai_document_classification")
|
||||
@override_settings(
|
||||
AI_ENABLED=True,
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import hashlib
|
||||
import logging
|
||||
import shutil
|
||||
import unicodedata
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Iterable
|
||||
from collections.abc import Iterator
|
||||
@@ -32,25 +31,6 @@ def identity(iterable: Iterable[_T]) -> Iterable[_T]:
|
||||
return iterable
|
||||
|
||||
|
||||
def normalize_unicode(value: str | None) -> str | None:
|
||||
"""
|
||||
Normalize a string to Unicode NFC form, or return None unchanged.
|
||||
|
||||
This is the single normalization pass for any user- or filesystem-supplied
|
||||
text that ends up in a filename, path, or is compared/matched against one
|
||||
(titles, correspondent/tag/type names, uploaded filenames, workflow and
|
||||
mail rule filename/path filters). Composed (NFC) and decomposed (NFD)
|
||||
forms of the same visible text are different byte sequences, which breaks
|
||||
exact comparisons and filesystem lookups even though the text looks
|
||||
identical. Always normalize through this function rather than calling
|
||||
unicodedata.normalize() directly, so every call site agrees on the same
|
||||
form.
|
||||
"""
|
||||
if value is None:
|
||||
return None
|
||||
return unicodedata.normalize("NFC", value)
|
||||
|
||||
|
||||
class QuerySetStream(Generic[_M]):
|
||||
"""Stream a QuerySet via .iterator(chunk_size=...) instead of
|
||||
materializing it (plus any prefetch caches) all at once, while still
|
||||
|
||||
@@ -27,13 +27,10 @@ def versions_newest_first(documents: QuerySet[Document]) -> QuerySet[Document]:
|
||||
|
||||
def annotate_effective_content(documents: QuerySet[Document]) -> QuerySet[Document]:
|
||||
"""
|
||||
Annotates documents with the content of their newest version unless the
|
||||
queryset already carries the annotation, falling back to their own, so
|
||||
get_effective_content() can answer from the row rather than querying for
|
||||
the versions of each document.
|
||||
Annotates documents with the content of their newest version, falling back
|
||||
to their own, so get_effective_content() can answer from the row rather
|
||||
than querying for the versions of each document
|
||||
"""
|
||||
if "effective_content" in documents.query.annotations:
|
||||
return documents
|
||||
return documents.annotate(
|
||||
effective_content=Coalesce(
|
||||
Subquery(
|
||||
|
||||
+20
-11
@@ -231,9 +231,7 @@ from documents.tasks import sanity_check
|
||||
from documents.tasks import train_classifier
|
||||
from documents.tasks import update_document_parent_tags
|
||||
from documents.utils import get_boolean
|
||||
from documents.utils import normalize_unicode
|
||||
from documents.versioning import VersionResolutionError
|
||||
from documents.versioning import annotate_effective_content
|
||||
from documents.versioning import get_latest_version_for_root
|
||||
from documents.versioning import get_request_version_param
|
||||
from documents.versioning import get_root_document
|
||||
@@ -253,6 +251,7 @@ from paperless.views import StandardPagination
|
||||
from paperless_ai.ai_classifier import get_ai_document_classification
|
||||
from paperless_ai.ai_classifier import get_llm_output_language
|
||||
from paperless_ai.chat import stream_chat_with_documents
|
||||
from paperless_ai.exceptions import LLMProviderError
|
||||
from paperless_ai.exceptions import LLMTimeoutError
|
||||
from paperless_ai.matching import extract_unmatched_names
|
||||
from paperless_ai.matching import match_correspondents_by_name
|
||||
@@ -1604,6 +1603,22 @@ class DocumentViewSet(
|
||||
{"ai": [_("AI backend request timed out.")]},
|
||||
status=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
)
|
||||
except LLMProviderError:
|
||||
logger.exception(
|
||||
"AI backend rejected the request for document %s",
|
||||
doc.pk,
|
||||
)
|
||||
return Response(
|
||||
{
|
||||
"ai": [
|
||||
_(
|
||||
"AI backend rejected the request. "
|
||||
"Check logs for details.",
|
||||
),
|
||||
],
|
||||
},
|
||||
status=status.HTTP_502_BAD_GATEWAY,
|
||||
)
|
||||
set_llm_suggestions_cache(
|
||||
doc.pk,
|
||||
llm_suggestions,
|
||||
@@ -2069,7 +2084,6 @@ class DocumentViewSet(
|
||||
|
||||
try:
|
||||
doc_name, doc_data = serializer.validated_data.get("document")
|
||||
doc_name = normalize_unicode(doc_name)
|
||||
version_label = serializer.validated_data.get("version_label")
|
||||
|
||||
t = int(mktime(datetime.now().timetuple()))
|
||||
@@ -3336,7 +3350,7 @@ class PostDocumentView(GenericAPIView[Any]):
|
||||
serializer.is_valid(raise_exception=True)
|
||||
|
||||
doc_name, doc_data = serializer.validated_data.get("document")
|
||||
doc_name = normalize_unicode(doc_name)
|
||||
doc_name = normalize("NFC", doc_name)
|
||||
correspondent_id = serializer.validated_data.get("correspondent")
|
||||
document_type_id = serializer.validated_data.get("document_type")
|
||||
storage_path_id = serializer.validated_data.get("storage_path")
|
||||
@@ -3635,13 +3649,8 @@ class GlobalSearchView(PassUserMixin):
|
||||
OBJECT_LIMIT = 3
|
||||
docs = []
|
||||
if request.user.has_perm("documents.view_document"):
|
||||
# Never more than OBJECT_LIMIT rows come back here, so annotating
|
||||
# is cheap -- and without it these results show the root
|
||||
# document's superseded content.
|
||||
all_docs = annotate_effective_content(
|
||||
Document.objects.filter(
|
||||
id__in=permitted_document_ids(request.user),
|
||||
),
|
||||
all_docs = Document.objects.filter(
|
||||
id__in=permitted_document_ids(request.user),
|
||||
)
|
||||
if db_only:
|
||||
docs = all_docs.filter(title__icontains=query)[:OBJECT_LIMIT]
|
||||
|
||||
@@ -2,7 +2,7 @@ msgid ""
|
||||
msgstr ""
|
||||
"Project-Id-Version: paperless-ngx\n"
|
||||
"Report-Msgid-Bugs-To: \n"
|
||||
"POT-Creation-Date: 2026-09-08 15:56+0000\n"
|
||||
"POT-Creation-Date: 2026-09-07 20:47+0000\n"
|
||||
"PO-Revision-Date: 2022-02-17 04:17\n"
|
||||
"Last-Translator: \n"
|
||||
"Language-Team: English\n"
|
||||
@@ -21,39 +21,39 @@ msgstr ""
|
||||
msgid "Documents"
|
||||
msgstr ""
|
||||
|
||||
#: documents/filters.py:463
|
||||
#: documents/filters.py:473
|
||||
msgid "Value must be valid JSON."
|
||||
msgstr ""
|
||||
|
||||
#: documents/filters.py:482
|
||||
#: documents/filters.py:492
|
||||
msgid "Invalid custom field query expression"
|
||||
msgstr ""
|
||||
|
||||
#: documents/filters.py:492
|
||||
#: documents/filters.py:502
|
||||
msgid "Invalid expression list. Must be nonempty."
|
||||
msgstr ""
|
||||
|
||||
#: documents/filters.py:513
|
||||
#: documents/filters.py:523
|
||||
msgid "Invalid logical operator {op!r}"
|
||||
msgstr ""
|
||||
|
||||
#: documents/filters.py:527
|
||||
#: documents/filters.py:537
|
||||
msgid "Maximum number of query conditions exceeded."
|
||||
msgstr ""
|
||||
|
||||
#: documents/filters.py:591
|
||||
#: documents/filters.py:601
|
||||
msgid "{name!r} is not a valid custom field."
|
||||
msgstr ""
|
||||
|
||||
#: documents/filters.py:628
|
||||
#: documents/filters.py:638
|
||||
msgid "{data_type} does not support query expr {expr!r}."
|
||||
msgstr ""
|
||||
|
||||
#: documents/filters.py:747 documents/models.py:136
|
||||
#: documents/filters.py:757 documents/models.py:136
|
||||
msgid "Maximum nesting depth exceeded."
|
||||
msgstr ""
|
||||
|
||||
#: documents/filters.py:1109
|
||||
#: documents/filters.py:1119
|
||||
msgid "Custom field not found"
|
||||
msgstr ""
|
||||
|
||||
@@ -1631,49 +1631,49 @@ msgstr ""
|
||||
msgid "workflow runs"
|
||||
msgstr ""
|
||||
|
||||
#: documents/serialisers.py:524 documents/serialisers.py:881
|
||||
#: documents/serialisers.py:2841 documents/views.py:315 documents/views.py:2625
|
||||
#: documents/serialisers.py:524 documents/serialisers.py:878
|
||||
#: documents/serialisers.py:2838 documents/views.py:314 documents/views.py:2624
|
||||
#: paperless_mail/serialisers.py:156
|
||||
msgid "Insufficient permissions."
|
||||
msgstr ""
|
||||
|
||||
#: documents/serialisers.py:717
|
||||
#: documents/serialisers.py:714
|
||||
msgid "Invalid color."
|
||||
msgstr ""
|
||||
|
||||
#: documents/serialisers.py:2318
|
||||
#: documents/serialisers.py:2315
|
||||
#, python-format
|
||||
msgid "File type %(type)s not supported"
|
||||
msgstr ""
|
||||
|
||||
#: documents/serialisers.py:2362
|
||||
#: documents/serialisers.py:2359
|
||||
#, python-format
|
||||
msgid "Custom field id must be an integer: %(id)s"
|
||||
msgstr ""
|
||||
|
||||
#: documents/serialisers.py:2369
|
||||
#: documents/serialisers.py:2366
|
||||
#, python-format
|
||||
msgid "Custom field with id %(id)s does not exist"
|
||||
msgstr ""
|
||||
|
||||
#: documents/serialisers.py:2386 documents/serialisers.py:2396
|
||||
#: documents/serialisers.py:2383 documents/serialisers.py:2393
|
||||
msgid ""
|
||||
"Custom fields must be a list of integers or an object mapping ids to values."
|
||||
msgstr ""
|
||||
|
||||
#: documents/serialisers.py:2391
|
||||
#: documents/serialisers.py:2388
|
||||
msgid "Some custom fields don't exist or were specified twice."
|
||||
msgstr ""
|
||||
|
||||
#: documents/serialisers.py:2538
|
||||
#: documents/serialisers.py:2535
|
||||
msgid "Invalid variable detected."
|
||||
msgstr ""
|
||||
|
||||
#: documents/serialisers.py:2897
|
||||
#: documents/serialisers.py:2894
|
||||
msgid "Duplicate document identifiers are not allowed."
|
||||
msgstr ""
|
||||
|
||||
#: documents/serialisers.py:2927 documents/views.py:4632
|
||||
#: documents/serialisers.py:2924 documents/views.py:4626
|
||||
#, python-format
|
||||
msgid "Documents not found: %(ids)s"
|
||||
msgstr ""
|
||||
@@ -1941,36 +1941,36 @@ msgstr ""
|
||||
msgid "Unable to parse URI {value}"
|
||||
msgstr ""
|
||||
|
||||
#: documents/views.py:308 documents/views.py:2622
|
||||
#: documents/views.py:307 documents/views.py:2621
|
||||
msgid "Invalid more_like_id"
|
||||
msgstr ""
|
||||
|
||||
#: documents/views.py:1592
|
||||
#: documents/views.py:1591
|
||||
msgid "Invalid AI configuration."
|
||||
msgstr ""
|
||||
|
||||
#: documents/views.py:1603
|
||||
#: documents/views.py:1602
|
||||
msgid "AI backend request timed out."
|
||||
msgstr ""
|
||||
|
||||
#: documents/views.py:2447 documents/views.py:2768
|
||||
#: documents/views.py:2446 documents/views.py:2767
|
||||
msgid "Specify only one of text, title_search, query, or more_like_id."
|
||||
msgstr ""
|
||||
|
||||
#: documents/views.py:4645
|
||||
#: documents/views.py:4639
|
||||
#, python-format
|
||||
msgid "Insufficient permissions to share document %(id)s."
|
||||
msgstr ""
|
||||
|
||||
#: documents/views.py:4691
|
||||
#: documents/views.py:4685
|
||||
msgid "Bundle is already being processed."
|
||||
msgstr ""
|
||||
|
||||
#: documents/views.py:4755
|
||||
#: documents/views.py:4749
|
||||
msgid "The share link bundle is still being prepared. Please try again later."
|
||||
msgstr ""
|
||||
|
||||
#: documents/views.py:4769
|
||||
#: documents/views.py:4763
|
||||
msgid "The share link bundle is unavailable."
|
||||
msgstr ""
|
||||
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import logging
|
||||
import os
|
||||
from io import BytesIO
|
||||
|
||||
import magic
|
||||
@@ -213,7 +212,6 @@ class ProfileSerializer(PasswordValidationMixin, serializers.ModelSerializer[Use
|
||||
class ApplicationConfigurationSerializer(
|
||||
serializers.ModelSerializer[ApplicationConfiguration],
|
||||
):
|
||||
externally_configured_variables = serializers.SerializerMethodField()
|
||||
user_args = serializers.JSONField(binary=True, allow_null=True)
|
||||
barcode_tag_mapping = serializers.JSONField(binary=True, allow_null=True)
|
||||
llm_api_key = ObfuscatedPasswordField(
|
||||
@@ -229,12 +227,6 @@ class ApplicationConfigurationSerializer(
|
||||
|
||||
OBFUSCATED_FIELDS = ("llm_api_key", "remote_ocr_api_key")
|
||||
|
||||
def get_externally_configured_variables(
|
||||
self,
|
||||
instance: ApplicationConfiguration,
|
||||
) -> list[str]:
|
||||
return sorted(name for name in os.environ if name.startswith("PAPERLESS_"))
|
||||
|
||||
def run_validation(self, data):
|
||||
# Empty strings treated as None to avoid unexpected behavior
|
||||
if "user_args" in data and data["user_args"] == "":
|
||||
|
||||
@@ -22,6 +22,7 @@ from paperless.network import validate_outbound_http_url
|
||||
from paperless_ai.base_model import ClassificationSuggestions
|
||||
from paperless_ai.base_model import DocumentClassifierSchema
|
||||
from paperless_ai.base_model import model_to_classification_suggestions
|
||||
from paperless_ai.exceptions import LLMProviderError
|
||||
from paperless_ai.exceptions import LLMTimeoutError
|
||||
|
||||
logger = logging.getLogger("paperless_ai.client")
|
||||
@@ -132,7 +133,7 @@ class AIClient:
|
||||
from llama_index.core.llms import ChatMessage
|
||||
|
||||
if self.settings.llm_backend == LLMBackend.OLLAMA:
|
||||
with self._normalize_timeouts():
|
||||
with self._normalize_errors():
|
||||
result = self.llm.chat(
|
||||
[ChatMessage(role="user", content=prompt)],
|
||||
format=DocumentClassifierSchema.model_json_schema(),
|
||||
@@ -153,7 +154,7 @@ class AIClient:
|
||||
content=f"{prompt}\n\n"
|
||||
f"Answer by calling the {tool.metadata.name} tool. Do not write the answer as text.",
|
||||
)
|
||||
with self._normalize_timeouts():
|
||||
with self._normalize_errors():
|
||||
result = self.llm.chat_with_tools(
|
||||
tools=[tool],
|
||||
user_msg=user_msg,
|
||||
@@ -173,7 +174,7 @@ class AIClient:
|
||||
)
|
||||
|
||||
@contextmanager
|
||||
def _normalize_timeouts(self) -> Iterator[None]:
|
||||
def _normalize_errors(self) -> Iterator[None]:
|
||||
try:
|
||||
yield
|
||||
except httpx.TimeoutException as exc:
|
||||
@@ -181,8 +182,23 @@ class AIClient:
|
||||
except Exception as exc:
|
||||
if self._is_openai_timeout(exc):
|
||||
raise LLMTimeoutError from exc
|
||||
if self._is_provider_error(exc):
|
||||
raise LLMProviderError from exc
|
||||
raise
|
||||
|
||||
def _is_provider_error(self, exc: Exception) -> bool:
|
||||
if self.settings.llm_backend == LLMBackend.OLLAMA:
|
||||
from ollama import ResponseError
|
||||
|
||||
return isinstance(exc, ResponseError)
|
||||
|
||||
if self.settings.llm_backend == LLMBackend.OPENAI_LIKE:
|
||||
from openai import APIStatusError
|
||||
|
||||
return isinstance(exc, APIStatusError)
|
||||
|
||||
return False
|
||||
|
||||
def _is_openai_timeout(self, exc: Exception) -> bool:
|
||||
if self.settings.llm_backend != LLMBackend.OPENAI_LIKE:
|
||||
return False
|
||||
|
||||
@@ -1,2 +1,6 @@
|
||||
class LLMTimeoutError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class LLMProviderError(Exception):
|
||||
"""The LLM backend rejected the request."""
|
||||
|
||||
@@ -4,6 +4,7 @@ from unittest.mock import MagicMock
|
||||
from unittest.mock import patch
|
||||
|
||||
import httpx
|
||||
import ollama
|
||||
import openai
|
||||
import pytest
|
||||
from llama_index.core.llms.llm import ToolSelection
|
||||
@@ -11,6 +12,7 @@ from llama_index.core.llms.llm import ToolSelection
|
||||
from paperless_ai.client import LLM_SYSTEM_PROMPT
|
||||
from paperless_ai.client import PLACEHOLDER_API_KEY
|
||||
from paperless_ai.client import AIClient
|
||||
from paperless_ai.exceptions import LLMProviderError
|
||||
from paperless_ai.exceptions import LLMTimeoutError
|
||||
|
||||
|
||||
@@ -214,6 +216,52 @@ def test_run_llm_query_openai_timeout_raises_local_error(
|
||||
client.run_llm_query("test_prompt")
|
||||
|
||||
|
||||
def test_run_llm_query_openai_status_error_raises_provider_error(
|
||||
mock_ai_config,
|
||||
mock_openai_llm,
|
||||
):
|
||||
mock_ai_config.llm_backend = "openai-like"
|
||||
mock_ai_config.llm_model = "test_model"
|
||||
mock_ai_config.llm_endpoint = "http://test-url"
|
||||
|
||||
request = httpx.Request("POST", "http://test-url/v1/chat/completions")
|
||||
body = {"error": {"message": "Thinking mode does not support this tool_choice"}}
|
||||
mock_openai_llm.return_value.chat_with_tools.side_effect = openai.BadRequestError(
|
||||
"Error code: 400",
|
||||
response=httpx.Response(400, request=request, json=body),
|
||||
body=body,
|
||||
)
|
||||
|
||||
client = AIClient()
|
||||
|
||||
with pytest.raises(LLMProviderError) as exc_info:
|
||||
client.run_llm_query("test_prompt")
|
||||
assert str(exc_info.value) == ""
|
||||
assert isinstance(exc_info.value.__cause__, openai.BadRequestError)
|
||||
|
||||
|
||||
def test_run_llm_query_ollama_response_error_raises_provider_error(
|
||||
mock_ai_config,
|
||||
mock_ollama_llm,
|
||||
):
|
||||
mock_ai_config.llm_backend = "ollama"
|
||||
mock_ai_config.llm_model = "test_model"
|
||||
mock_ai_config.llm_endpoint = "http://test-url"
|
||||
|
||||
response_error = ollama.ResponseError(
|
||||
"confidential provider response",
|
||||
status_code=400,
|
||||
)
|
||||
mock_ollama_llm.return_value.chat.side_effect = response_error
|
||||
|
||||
client = AIClient()
|
||||
|
||||
with pytest.raises(LLMProviderError) as exc_info:
|
||||
client.run_llm_query("test_prompt")
|
||||
assert str(exc_info.value) == ""
|
||||
assert exc_info.value.__cause__ is response_error
|
||||
|
||||
|
||||
def test_run_llm_query_httpx_timeout_raises_local_error(
|
||||
mock_ai_config,
|
||||
mock_ollama_llm,
|
||||
|
||||
@@ -6,6 +6,7 @@ import socket
|
||||
import ssl
|
||||
import tempfile
|
||||
import traceback
|
||||
import unicodedata
|
||||
from datetime import date
|
||||
from datetime import timedelta
|
||||
from fnmatch import fnmatch
|
||||
@@ -44,7 +45,6 @@ from documents.models import Correspondent
|
||||
from documents.models import PaperlessTask
|
||||
from documents.parsers import is_mime_type_supported
|
||||
from documents.tasks import consume_file
|
||||
from documents.utils import normalize_unicode
|
||||
from paperless.network import is_public_ip
|
||||
from paperless.network import resolve_hostname_ips
|
||||
from paperless_mail.models import MailAccount
|
||||
@@ -617,10 +617,10 @@ class MailAccountHandler(LoggingMixin):
|
||||
rule: MailRule,
|
||||
) -> str | None:
|
||||
if rule.assign_title_from == MailRule.TitleSource.FROM_SUBJECT:
|
||||
return normalize_unicode(message.subject)
|
||||
return unicodedata.normalize("NFC", message.subject)
|
||||
|
||||
elif rule.assign_title_from == MailRule.TitleSource.FROM_FILENAME:
|
||||
return normalize_unicode(Path(att.filename).stem)
|
||||
return unicodedata.normalize("NFC", Path(att.filename).stem)
|
||||
|
||||
elif rule.assign_title_from == MailRule.TitleSource.NONE:
|
||||
return None
|
||||
@@ -1004,8 +1004,6 @@ class MailAccountHandler(LoggingMixin):
|
||||
consume_tasks = []
|
||||
|
||||
for att in message.attachments:
|
||||
attachment_filename = normalize_unicode(att.filename)
|
||||
|
||||
if (
|
||||
att.content_disposition != "attachment"
|
||||
and rule.attachment_type
|
||||
@@ -1020,7 +1018,7 @@ class MailAccountHandler(LoggingMixin):
|
||||
|
||||
if not self.filename_inclusion_matches(
|
||||
rule.filter_attachment_filename_include,
|
||||
attachment_filename,
|
||||
att.filename,
|
||||
):
|
||||
# Force the filename and pattern to the lowercase
|
||||
# as this is system dependent otherwise
|
||||
@@ -1032,7 +1030,7 @@ class MailAccountHandler(LoggingMixin):
|
||||
continue
|
||||
elif self.filename_exclusion_matches(
|
||||
rule.filter_attachment_filename_exclude,
|
||||
attachment_filename,
|
||||
att.filename,
|
||||
):
|
||||
self.log.debug(
|
||||
f"Rule {rule}: "
|
||||
@@ -1066,7 +1064,7 @@ class MailAccountHandler(LoggingMixin):
|
||||
)
|
||||
|
||||
attachment_name = pathvalidate.sanitize_filename(
|
||||
attachment_filename,
|
||||
unicodedata.normalize("NFC", att.filename),
|
||||
)
|
||||
if attachment_name:
|
||||
temp_filename = temp_dir / attachment_name
|
||||
@@ -1177,7 +1175,7 @@ class MailAccountHandler(LoggingMixin):
|
||||
doc_overrides = DocumentMetadataOverrides(
|
||||
title=message.subject,
|
||||
filename=pathvalidate.sanitize_filename(
|
||||
normalize_unicode(f"{message.subject}.eml"),
|
||||
unicodedata.normalize("NFC", f"{message.subject}.eml"),
|
||||
),
|
||||
correspondent_id=correspondent.id if correspondent else None,
|
||||
document_type_id=doc_type.id if doc_type else None,
|
||||
|
||||
@@ -8,7 +8,6 @@ from documents.serialisers import CorrespondentField
|
||||
from documents.serialisers import DocumentTypeField
|
||||
from documents.serialisers import OwnedObjectSerializer
|
||||
from documents.serialisers import TagsField
|
||||
from documents.utils import normalize_unicode
|
||||
from paperless_mail.models import MailAccount
|
||||
from paperless_mail.models import MailRule
|
||||
from paperless_mail.models import ProcessedMail
|
||||
@@ -162,12 +161,6 @@ class MailRuleSerializer(OwnedObjectSerializer):
|
||||
raise serializers.ValidationError("Maximum mail age is unreasonably large.")
|
||||
return value
|
||||
|
||||
def validate_filter_attachment_filename_include(self, value):
|
||||
return normalize_unicode(value)
|
||||
|
||||
def validate_filter_attachment_filename_exclude(self, value):
|
||||
return normalize_unicode(value)
|
||||
|
||||
|
||||
class ProcessedMailSerializer(OwnedObjectSerializer):
|
||||
class Meta:
|
||||
|
||||
Reference in New Issue
Block a user