Compare commits

...
Author SHA1 Message Date
shamoon bdd9132fa5 Dont mark entire test file for db, use a mock for empty engine settings 2026-08-19 10:01:20 -07:00
shamoonandGitHub 72f4cd4348 Actually we cant use this any more, in case settings are in app config 2026-08-18 09:45:25 -07:00
shamoonandGitHub d73492e9e6 Update test_tesseract_parser.py 2026-08-18 09:45:25 -07:00
shamoonandGitHub 005dae49ec Docs 2026-08-18 09:45:25 -07:00
shamoonandGitHub 75135f8ee5 Frontend stuff, with sections 2026-08-18 09:45:25 -07:00
shamoonandGitHub 6758b70b7e Backend tests 2026-08-18 09:45:25 -07:00
shamoonandGitHub 718e9b9c73 Backend changes and migration for remote OCR Config 2026-08-18 09:45:25 -07:00
shamoon 4cf027de40 Tweak: space out the side menu sub-nav a bit 2026-08-18 09:44:03 -07:00
shamoon 41953c7846 Chore: harden ImageMagick policy 2026-08-18 09:38:54 -07:00
dependabot[bot]GitHubdependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
e5abe5cf32 Chore(deps): Bump the uv group across 1 directory with 2 updates (#13709)
Bumps the uv group with 2 updates in the / directory: [pymdown-extensions](https://github.com/facelessuser/pymdown-extensions) and [sqlparse](https://github.com/andialbrecht/sqlparse).


Updates `pymdown-extensions` from 11.0 to 11.0.1
- [Release notes](https://github.com/facelessuser/pymdown-extensions/releases)
- [Commits](https://github.com/facelessuser/pymdown-extensions/compare/11.0...11.0.1)

Updates `sqlparse` from 0.5.5 to 0.6.0
- [Changelog](https://github.com/andialbrecht/sqlparse/blob/master/CHANGELOG)
- [Commits](https://github.com/andialbrecht/sqlparse/compare/0.5.5...0.6.0)

---
updated-dependencies:
- dependency-name: pymdown-extensions
  dependency-version: 11.0.1
  dependency-type: indirect
- dependency-name: sqlparse
  dependency-version: 0.6.0
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-08-18 15:11:24 +00:00
22 changed files with 991 additions and 658 deletions
@@ -68,7 +68,14 @@
<!-- <policy domain="resource" name="thread" value="4"/> --> <!-- <policy domain="resource" name="thread" value="4"/> -->
<!-- <policy domain="resource" name="throttle" value="0"/> --> <!-- <policy domain="resource" name="throttle" value="0"/> -->
<!-- <policy domain="resource" name="time" value="3600"/> --> <!-- <policy domain="resource" name="time" value="3600"/> -->
<!-- <policy domain="coder" rights="none" pattern="MVG" /> --> <!-- Paperless does not process SVG or ImageMagick scripting formats. -->
<policy domain="coder" rights="none" pattern="SVG" />
<policy domain="coder" rights="none" pattern="SVGZ" />
<policy domain="coder" rights="none" pattern="MSVG" />
<policy domain="coder" rights="none" pattern="RSVG" />
<policy domain="coder" rights="none" pattern="MSL" />
<policy domain="coder" rights="none" pattern="MVG" />
<policy domain="coder" rights="none" pattern="EPHEMERAL" />
<!-- <policy domain="module" rights="none" pattern="{PS,PDF,XPS}" /> --> <!-- <policy domain="module" rights="none" pattern="{PS,PDF,XPS}" /> -->
<!-- <policy domain="delegate" rights="none" pattern="HTTPS" /> --> <!-- <policy domain="delegate" rights="none" pattern="HTTPS" /> -->
<!-- <policy domain="path" rights="none" pattern="@*" /> --> <!-- <policy domain="path" rights="none" pattern="@*" /> -->
@@ -78,8 +85,6 @@
<!-- <policy domain="system" name="pixel-cache-memory" value="anonymous"/> --> <!-- <policy domain="system" name="pixel-cache-memory" value="anonymous"/> -->
<!-- <policy domain="system" name="shred" value="2"/> --> <!-- <policy domain="system" name="shred" value="2"/> -->
<!-- <policy domain="system" name="precision" value="6"/> --> <!-- <policy domain="system" name="precision" value="6"/> -->
<!-- not needed due to the need to use explicitly by mvg: -->
<!-- <policy domain="delegate" rights="none" pattern="MVG" /> -->
<!-- use curl --> <!-- use curl -->
<policy domain="delegate" rights="none" pattern="URL" /> <policy domain="delegate" rights="none" pattern="URL" />
<policy domain="delegate" rights="none" pattern="HTTPS" /> <policy domain="delegate" rights="none" pattern="HTTPS" />
+8 -15
View File
@@ -416,22 +416,15 @@ to a positive number to enable polling and disable native filesystem notificatio
You may need to change the path in the files. Example: You may need to change the path in the files. Example:
`ExecStart=/opt/paperless/.local/bin/celery --app paperless worker --loglevel INFO` `ExecStart=/opt/paperless/.local/bin/celery --app paperless worker --loglevel INFO`
12. Configure ImageMagick to allow processing of PDF documents. Most 12. Configure ImageMagick to allow processing of PDF documents and disable
distributions have this disabled by default, since PDF documents can formats that Paperless-ngx does not use. Most distributions disable PDF
contain malware. If you don't do this, Paperless-ngx will fall back to processing by default, since PDF documents can contain malware. If you
Ghostscript for certain steps such as thumbnail generation. don't enable it, Paperless-ngx will fall back to Ghostscript for certain
steps such as thumbnail generation.
Edit `/etc/ImageMagick-6/policy.xml` and adjust Configure the active ImageMagick policy file (commonly
`/etc/ImageMagick-6/policy.xml` or `/etc/ImageMagick-7/policy.xml`) and
``` adjust similar to [the docker policy file](https://raw.githubusercontent.com/paperless-ngx/paperless-ngx/refs/heads/main/docker/rootfs/etc/ImageMagick-6/paperless-policy.xml). You should also include restrictions as noted there.
<policy domain="coder" rights="none" pattern="PDF" />
```
to
```
<policy domain="coder" rights="read|write" pattern="PDF" />
```
**Optional: Install the [jbig2enc](https://ocrmypdf.readthedocs.io/en/latest/jbig2.html) encoder.** **Optional: Install the [jbig2enc](https://ocrmypdf.readthedocs.io/en/latest/jbig2.html) encoder.**
This will reduce the size of generated PDF documents. You'll most likely need to compile this yourself, because this This will reduce the size of generated PDF documents. You'll most likely need to compile this yourself, because this
+2 -1
View File
@@ -1086,7 +1086,8 @@ Paperless-ngx supports performing OCR on documents using remote services. At the
[Microsoft's Azure "Document Intelligence" service](https://azure.microsoft.com/en-us/products/ai-services/ai-document-intelligence). [Microsoft's Azure "Document Intelligence" service](https://azure.microsoft.com/en-us/products/ai-services/ai-document-intelligence).
This is of course a paid service (with a free tier) which requires an Azure account and subscription. Azure AI is not affiliated with This is of course a paid service (with a free tier) which requires an Azure account and subscription. Azure AI is not affiliated with
Paperless-ngx in any way. When enabled, Paperless-ngx will automatically send appropriate documents to Azure for OCR processing, bypassing Paperless-ngx in any way. When enabled, Paperless-ngx will automatically send appropriate documents to Azure for OCR processing, bypassing
the local OCR engine. See the [configuration](configuration.md#PAPERLESS_REMOTE_OCR_ENGINE) options for more details. the local OCR engine. See the [configuration](configuration.md#PAPERLESS_REMOTE_OCR_ENGINE) options for more details. These
settings can be supplied as environment variables or via **Application Configuration**.
Additionally, when using a commercial service with this feature, consider both potential costs as well as any associated file size Additionally, when using a commercial service with this feature, consider both potential costs as well as any associated file size
or page limitations (e.g. with a free tier). or page limitations (e.g. with a free tier).
@@ -14,43 +14,48 @@
<a ngbNavLink>{{category}}</a> <a ngbNavLink>{{category}}</a>
<ng-template ngbNavContent> <ng-template ngbNavContent>
<div class="p-3"> <div class="p-3">
<div class="row row-cols-1 row-cols-md-2 row-cols-lg-3 g-2"> @for (section of getCategorySections(category); track section) {
@for (option of getCategoryOptions(category); track option.key) { @if (section) {
<div class="col"> <h5 class="mt-4 mb-3">{{section}}</h5>
<div class="card bg-light"> }
<div class="card-body"> <div class="row row-cols-1 row-cols-md-2 row-cols-lg-3 g-2">
<div class="card-title d-flex align-items-center"> @for (option of getCategoryOptions(category, section); track option.key) {
<h6 class="mb-0"> <div class="col">
{{option.title}} <div class="card bg-light">
</h6> <div class="card-body">
<a class="btn btn-sm btn-link" title="Read the documentation about this setting" i18n-title [href]="getDocsUrl(option.config_key)" target="_blank" referrerpolicy="no-referrer"> <div class="card-title d-flex align-items-center">
<i-bs name="info-circle"></i-bs> <h6 class="mb-0">
</a> {{option.title}}
@if (isSet(option.key)) { </h6>
<button type="button" class="btn btn-sm btn-link text-danger ms-auto pe-0" title="Reset" i18n-title (click)="resetOption(option.key)"> <a class="btn btn-sm btn-link" title="Read the documentation about this setting" i18n-title [href]="getDocsUrl(option.config_key)" target="_blank" referrerpolicy="no-referrer">
<i-bs class="me-1" name="x"></i-bs><ng-container i18n>Reset</ng-container> <i-bs name="info-circle"></i-bs>
</button> </a>
@if (isSet(option.key)) {
<button type="button" class="btn btn-sm btn-link text-danger ms-auto pe-0" title="Reset" i18n-title (click)="resetOption(option.key)">
<i-bs class="me-1" name="x"></i-bs><ng-container i18n>Reset</ng-container>
</button>
}
</div>
<div class="mb-n3">
@switch (option.type) {
@case (ConfigOptionType.Select) { <pngx-input-select [formControlName]="option.key" [error]="errors[option.key]" [items]="option.choices" [allowNull]="true"></pngx-input-select> }
@case (ConfigOptionType.Number) { <pngx-input-number [formControlName]="option.key" [error]="errors[option.key]" [showAdd]="false"></pngx-input-number> }
@case (ConfigOptionType.Boolean) { <pngx-input-switch [formControlName]="option.key" [error]="errors[option.key]" [showUnsetNote]="true" [horizontal]="true" title="Enable" i18n-title></pngx-input-switch> }
@case (ConfigOptionType.String) { <pngx-input-text [formControlName]="option.key" [error]="errors[option.key]"></pngx-input-text> }
@case (ConfigOptionType.JSON) { <pngx-input-text [formControlName]="option.key" [error]="errors[option.key]"></pngx-input-text> }
@case (ConfigOptionType.File) { <pngx-input-file [formControlName]="option.key" (upload)="uploadFile($event, option.key)" [error]="errors[option.key]"></pngx-input-file> }
@case (ConfigOptionType.Password) { <pngx-input-password [formControlName]="option.key" [error]="errors[option.key]"></pngx-input-password> }
}
</div>
@if (option.note) {
<div class="form-text fst-italic">{{option.note}}</div>
} }
</div> </div>
<div class="mb-n3">
@switch (option.type) {
@case (ConfigOptionType.Select) { <pngx-input-select [formControlName]="option.key" [error]="errors[option.key]" [items]="option.choices" [allowNull]="true"></pngx-input-select> }
@case (ConfigOptionType.Number) { <pngx-input-number [formControlName]="option.key" [error]="errors[option.key]" [showAdd]="false"></pngx-input-number> }
@case (ConfigOptionType.Boolean) { <pngx-input-switch [formControlName]="option.key" [error]="errors[option.key]" [showUnsetNote]="true" [horizontal]="true" title="Enable" i18n-title></pngx-input-switch> }
@case (ConfigOptionType.String) { <pngx-input-text [formControlName]="option.key" [error]="errors[option.key]"></pngx-input-text> }
@case (ConfigOptionType.JSON) { <pngx-input-text [formControlName]="option.key" [error]="errors[option.key]"></pngx-input-text> }
@case (ConfigOptionType.File) { <pngx-input-file [formControlName]="option.key" (upload)="uploadFile($event, option.key)" [error]="errors[option.key]"></pngx-input-file> }
@case (ConfigOptionType.Password) { <pngx-input-password [formControlName]="option.key" [error]="errors[option.key]"></pngx-input-password> }
}
</div>
@if (option.note) {
<div class="form-text fst-italic">{{option.note}}</div>
}
</div> </div>
</div> </div>
</div> }
} </div>
</div> }
</div> </div>
</ng-template> </ng-template>
</li> </li>
@@ -8,7 +8,11 @@ import { NgbModule } from '@ng-bootstrap/ng-bootstrap'
import { NgSelectModule } from '@ng-select/ng-select' import { NgSelectModule } from '@ng-select/ng-select'
import { NgxBootstrapIconsModule, allIcons } from 'ngx-bootstrap-icons' import { NgxBootstrapIconsModule, allIcons } from 'ngx-bootstrap-icons'
import { of, throwError } from 'rxjs' import { of, throwError } from 'rxjs'
import { OutputTypeConfig } from 'src/app/data/paperless-config' import {
ConfigCategory,
ConfigSection,
OutputTypeConfig,
} from 'src/app/data/paperless-config'
import { ConfigService } from 'src/app/services/config.service' import { ConfigService } from 'src/app/services/config.service'
import { SettingsService } from 'src/app/services/settings.service' import { SettingsService } from 'src/app/services/settings.service'
import { ToastService } from 'src/app/services/toast.service' import { ToastService } from 'src/app/services/toast.service'
@@ -158,4 +162,23 @@ describe('ConfigComponent', () => {
component.resetOption('barcodes_enabled') component.resetOption('barcodes_enabled')
expect(component.configForm.get('barcodes_enabled').value).toBeNull() expect(component.configForm.get('barcodes_enabled').value).toBeNull()
}) })
it('should group options into sections within a category, or not', () => {
const sections = component.getCategorySections(ConfigCategory.OCR)
expect(sections).toEqual([null, ConfigSection.RemoteOCR])
expect(
component
.getCategoryOptions(ConfigCategory.OCR)
.map((option) => option.key)
).toContain('output_type')
expect(
component
.getCategoryOptions(ConfigCategory.OCR, ConfigSection.RemoteOCR)
.map((option) => option.key)
).toEqual([
'remote_ocr_engine',
'remote_ocr_api_key',
'remote_ocr_endpoint',
])
})
}) })
@@ -74,8 +74,20 @@ export class ConfigComponent
return Object.values(ConfigCategory) return Object.values(ConfigCategory)
} }
getCategoryOptions(category: string): ConfigOption[] { getCategorySections(category: string): string[] {
return PaperlessConfigOptions.filter((o) => o.category === category) return [
...new Set(
PaperlessConfigOptions.filter((o) => o.category === category).map(
(o) => o.section ?? null // null means no section
)
),
]
}
getCategoryOptions(category: string, section: string = null): ConfigOption[] {
return PaperlessConfigOptions.filter(
(o) => o.category === category && (o.section ?? null) === section
)
} }
initialConfig: PaperlessConfig initialConfig: PaperlessConfig
@@ -195,32 +195,32 @@
} }
</div> </div>
<div <div
class="attributes-submenu ms-2" class="attributes-submenu ms-3"
[ngbCollapse]="slimSidebarEnabled || attributesSectionsCollapsed" [ngbCollapse]="slimSidebarEnabled || attributesSectionsCollapsed"
> >
<ul class="nav flex-column"> <ul class="nav flex-column">
<li class="nav-item app-link" *pngxIfPermissions="{ action: PermissionAction.View, type: PermissionType.Tag }"> <li class="nav-item app-link" *pngxIfPermissions="{ action: PermissionAction.View, type: PermissionType.Tag }">
<a class="nav-link py-1" routerLink="attributes/tags" routerLinkActive="active" (click)="closeMenu()"> <a class="nav-link" routerLink="attributes/tags" routerLinkActive="active" (click)="closeMenu()">
<i-bs class="me-2" name="tags"></i-bs><span><ng-container i18n>Tags</ng-container></span> <i-bs class="me-2" name="tags"></i-bs><span><ng-container i18n>Tags</ng-container></span>
</a> </a>
</li> </li>
<li class="nav-item app-link" *pngxIfPermissions="{ action: PermissionAction.View, type: PermissionType.Correspondent }"> <li class="nav-item app-link" *pngxIfPermissions="{ action: PermissionAction.View, type: PermissionType.Correspondent }">
<a class="nav-link py-1" routerLink="attributes/correspondents" routerLinkActive="active" (click)="closeMenu()"> <a class="nav-link" routerLink="attributes/correspondents" routerLinkActive="active" (click)="closeMenu()">
<i-bs class="me-2" name="person"></i-bs><span><ng-container i18n>Correspondents</ng-container></span> <i-bs class="me-2" name="person"></i-bs><span><ng-container i18n>Correspondents</ng-container></span>
</a> </a>
</li> </li>
<li class="nav-item app-link" *pngxIfPermissions="{ action: PermissionAction.View, type: PermissionType.DocumentType }"> <li class="nav-item app-link" *pngxIfPermissions="{ action: PermissionAction.View, type: PermissionType.DocumentType }">
<a class="nav-link py-1" routerLink="attributes/documenttypes" routerLinkActive="active" (click)="closeMenu()"> <a class="nav-link" routerLink="attributes/documenttypes" routerLinkActive="active" (click)="closeMenu()">
<i-bs class="me-2" name="hash"></i-bs><span><ng-container i18n>Document types</ng-container></span> <i-bs class="me-2" name="hash"></i-bs><span><ng-container i18n>Document types</ng-container></span>
</a> </a>
</li> </li>
<li class="nav-item app-link" *pngxIfPermissions="{ action: PermissionAction.View, type: PermissionType.StoragePath }"> <li class="nav-item app-link" *pngxIfPermissions="{ action: PermissionAction.View, type: PermissionType.StoragePath }">
<a class="nav-link py-1" routerLink="attributes/storagepaths" routerLinkActive="active" (click)="closeMenu()"> <a class="nav-link" routerLink="attributes/storagepaths" routerLinkActive="active" (click)="closeMenu()">
<i-bs class="me-2" name="folder"></i-bs><span><ng-container i18n>Storage paths</ng-container></span> <i-bs class="me-2" name="folder"></i-bs><span><ng-container i18n>Storage paths</ng-container></span>
</a> </a>
</li> </li>
<li class="nav-item app-link" *pngxIfPermissions="{ action: PermissionAction.View, type: PermissionType.CustomField }"> <li class="nav-item app-link" *pngxIfPermissions="{ action: PermissionAction.View, type: PermissionType.CustomField }">
<a class="nav-link py-1" routerLink="attributes/customfields" routerLinkActive="active" (click)="closeMenu()"> <a class="nav-link" routerLink="attributes/customfields" routerLinkActive="active" (click)="closeMenu()">
<i-bs class="me-2" name="ui-radios"></i-bs><span><ng-container i18n>Custom fields</ng-container></span> <i-bs class="me-2" name="ui-radios"></i-bs><span><ng-container i18n>Custom fields</ng-container></span>
</a> </a>
</li> </li>
@@ -216,8 +216,8 @@ main {
// bring sub-menu markers back out to L edge // bring sub-menu markers back out to L edge
.attributes-submenu .nav-link { .attributes-submenu .nav-link {
margin-left: -0.5rem; margin-left: -1rem;
padding-left: calc(var(--bs-nav-link-padding-x) + 0.5rem); padding-left: calc(var(--bs-nav-link-padding-x) + 1rem);
} }
.attributes-row .attributes-expand-btn { .attributes-row .attributes-expand-btn {
+39
View File
@@ -54,6 +54,10 @@ export const ConfigCategory = {
AI: $localize`AI Settings`, AI: $localize`AI Settings`,
} }
export const ConfigSection = {
RemoteOCR: $localize`Remote OCR`,
}
export const LLMEmbeddingBackendConfig = { export const LLMEmbeddingBackendConfig = {
OPENAI_LIKE: 'openai-like', OPENAI_LIKE: 'openai-like',
HUGGINGFACE: 'huggingface', HUGGINGFACE: 'huggingface',
@@ -65,6 +69,10 @@ export const LLMBackendConfig = {
OLLAMA: 'ollama', OLLAMA: 'ollama',
} }
export const RemoteOCREngineConfig = {
AZURE_AI: 'azureai',
}
export interface ConfigOption { export interface ConfigOption {
key: string key: string
title: string title: string
@@ -72,6 +80,7 @@ export interface ConfigOption {
choices?: Array<{ id: string; name: string }> choices?: Array<{ id: string; name: string }>
config_key?: string config_key?: string
category: string category: string
section?: string
note?: string note?: string
} }
@@ -181,6 +190,33 @@ export const PaperlessConfigOptions: ConfigOption[] = [
config_key: 'PAPERLESS_OCR_USER_ARGS', config_key: 'PAPERLESS_OCR_USER_ARGS',
category: ConfigCategory.OCR, category: ConfigCategory.OCR,
}, },
{
key: 'remote_ocr_engine',
title: $localize`Remote OCR Engine`,
type: ConfigOptionType.Select,
choices: mapToItems(RemoteOCREngineConfig),
config_key: 'PAPERLESS_REMOTE_OCR_ENGINE',
category: ConfigCategory.OCR,
section: ConfigSection.RemoteOCR,
note: $localize`Enabling remote OCR sends documents to a third-party service for processing. Consider the privacy implications as well as potential costs before enabling.`,
},
{
key: 'remote_ocr_api_key',
title: $localize`Remote OCR API Key`,
type: ConfigOptionType.Password,
config_key: 'PAPERLESS_REMOTE_OCR_API_KEY',
category: ConfigCategory.OCR,
section: ConfigSection.RemoteOCR,
},
{
key: 'remote_ocr_endpoint',
title: $localize`Remote OCR Endpoint`,
type: ConfigOptionType.String,
config_key: 'PAPERLESS_REMOTE_OCR_ENDPOINT',
category: ConfigCategory.OCR,
section: ConfigSection.RemoteOCR,
note: $localize`Required when using the Azure AI engine.`,
},
{ {
key: 'app_logo', key: 'app_logo',
title: $localize`Application Logo`, title: $localize`Application Logo`,
@@ -398,6 +434,9 @@ export interface PaperlessConfig extends ObjectWithId {
barcode_enable_tag: boolean barcode_enable_tag: boolean
barcode_tag_mapping: object barcode_tag_mapping: object
barcode_tag_split: boolean barcode_tag_split: boolean
remote_ocr_engine: string
remote_ocr_api_key: string
remote_ocr_endpoint: string
ai_enabled: boolean ai_enabled: boolean
llm_embedding_backend: string llm_embedding_backend: string
llm_embedding_model: string llm_embedding_model: string
@@ -72,6 +72,9 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
"barcode_enable_tag": None, "barcode_enable_tag": None,
"barcode_tag_mapping": None, "barcode_tag_mapping": None,
"barcode_tag_split": None, "barcode_tag_split": None,
"remote_ocr_engine": None,
"remote_ocr_api_key": None,
"remote_ocr_endpoint": None,
"ai_enabled": False, "ai_enabled": False,
"llm_embedding_backend": None, "llm_embedding_backend": None,
"llm_embedding_model": None, "llm_embedding_model": None,
@@ -870,6 +873,49 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
config.refresh_from_db() config.refresh_from_db()
self.assertEqual(config.llm_api_key, None) self.assertEqual(config.llm_api_key, None)
def test_update_remote_ocr_api_key(self) -> None:
"""
GIVEN:
- Existing config with remote_ocr_api_key specified
WHEN:
- API to update remote_ocr_api_key is called with all *s
- API to update remote_ocr_api_key is called with empty string
THEN:
- remote_ocr_api_key is unchanged
- remote_ocr_api_key is set to None
"""
config = ApplicationConfiguration.objects.first()
assert config is not None
config.remote_ocr_api_key = "1234567890"
config.save()
# Test with all *
response = self.client.patch(
f"{self.ENDPOINT}1/",
json.dumps(
{
"remote_ocr_api_key": "*" * 32,
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
config.refresh_from_db()
self.assertEqual(config.remote_ocr_api_key, "1234567890")
# Test with empty string
response = self.client.patch(
f"{self.ENDPOINT}1/",
json.dumps(
{
"remote_ocr_api_key": "",
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
config.refresh_from_db()
self.assertEqual(config.remote_ocr_api_key, None)
def test_enable_ai_index_triggers_update(self) -> None: def test_enable_ai_index_triggers_update(self) -> None:
""" """
GIVEN: GIVEN:
-14
View File
@@ -337,20 +337,6 @@ def check_deprecated_v2_ocr_env_vars(
return warnings return warnings
@register()
def check_remote_parser_configured(app_configs: Any, **kwargs: Any) -> list[Error]:
if settings.REMOTE_OCR_ENGINE == "azureai" and not (
settings.REMOTE_OCR_ENDPOINT and settings.REMOTE_OCR_API_KEY
):
return [
Error(
"Azure AI remote parser requires endpoint and API key to be configured.",
),
]
return []
def get_tesseract_langs(): def get_tesseract_langs():
proc = subprocess.run( proc = subprocess.run(
[shutil.which("tesseract"), "--list-langs"], [shutil.which("tesseract"), "--list-langs"],
+24
View File
@@ -185,6 +185,30 @@ class GeneralConfig(BaseConfig):
self.app_logo = app_config.app_logo.url if app_config.app_logo else None self.app_logo = app_config.app_logo.url if app_config.app_logo else None
@dataclasses.dataclass
class RemoteOCRConfig(BaseConfig):
"""
Settings for the remote (cloud) OCR parser
"""
remote_ocr_engine: str | None = dataclasses.field(init=False)
remote_ocr_api_key: str | None = dataclasses.field(init=False)
remote_ocr_endpoint: str | None = dataclasses.field(init=False)
def __post_init__(self) -> None:
app_config = self._get_config_instance()
self.remote_ocr_engine = (
app_config.remote_ocr_engine or settings.REMOTE_OCR_ENGINE
)
self.remote_ocr_api_key = (
app_config.remote_ocr_api_key or settings.REMOTE_OCR_API_KEY
)
self.remote_ocr_endpoint = (
app_config.remote_ocr_endpoint or settings.REMOTE_OCR_ENDPOINT
)
@dataclasses.dataclass @dataclasses.dataclass
class AIConfig(BaseConfig): class AIConfig(BaseConfig):
""" """
@@ -0,0 +1,44 @@
# Generated by Django 5.2.16 on 2026-08-10 14:37
from django.db import migrations
from django.db import models
class Migration(migrations.Migration):
dependencies = [
("paperless", "0013_applicationconfiguration_llm_request_timeout"),
]
operations = [
migrations.AddField(
model_name="applicationconfiguration",
name="remote_ocr_api_key",
field=models.CharField(
blank=True,
max_length=1024,
null=True,
verbose_name="Sets the remote OCR API key",
),
),
migrations.AddField(
model_name="applicationconfiguration",
name="remote_ocr_endpoint",
field=models.CharField(
blank=True,
max_length=256,
null=True,
verbose_name="Sets the remote OCR endpoint",
),
),
migrations.AddField(
model_name="applicationconfiguration",
name="remote_ocr_engine",
field=models.CharField(
blank=True,
choices=[("azureai", "Azure AI Document Intelligence")],
max_length=32,
null=True,
verbose_name="Sets the remote OCR engine",
),
),
]
+37
View File
@@ -74,6 +74,14 @@ class ColorConvertChoices(models.TextChoices):
CMYK = ("CMYK", _("CMYK")) CMYK = ("CMYK", _("CMYK"))
class RemoteOCREngine(models.TextChoices):
"""
Matches to PAPERLESS_REMOTE_OCR_ENGINE
"""
AZURE_AI = ("azureai", _("Azure AI Document Intelligence"))
class LLMEmbeddingBackend(models.TextChoices): class LLMEmbeddingBackend(models.TextChoices):
OPENAI_LIKE = ("openai-like", _("OpenAI-compatible")) OPENAI_LIKE = ("openai-like", _("OpenAI-compatible"))
HUGGINGFACE = ("huggingface", _("Huggingface")) HUGGINGFACE = ("huggingface", _("Huggingface"))
@@ -286,6 +294,35 @@ class ApplicationConfiguration(AbstractSingletonModel):
null=True, null=True,
) )
"""
Settings for the remote OCR parser
"""
# PAPERLESS_REMOTE_OCR_ENGINE
remote_ocr_engine = models.CharField(
verbose_name=_("Sets the remote OCR engine"),
blank=True,
null=True,
max_length=32,
choices=RemoteOCREngine.choices,
)
# PAPERLESS_REMOTE_OCR_API_KEY
remote_ocr_api_key = models.CharField(
verbose_name=_("Sets the remote OCR API key"),
blank=True,
null=True,
max_length=1024,
)
# PAPERLESS_REMOTE_OCR_ENDPOINT
remote_ocr_endpoint = models.CharField(
verbose_name=_("Sets the remote OCR endpoint"),
blank=True,
null=True,
max_length=256,
)
""" """
AI related settings AI related settings
""" """
+14 -10
View File
@@ -61,6 +61,18 @@ class RemoteEngineConfig:
self.api_key = api_key self.api_key = api_key
self.endpoint = endpoint self.endpoint = endpoint
@classmethod
def from_app_config(cls) -> Self:
"""Build the config from the app config, falling back to the env."""
from paperless.config import RemoteOCRConfig
app_config = RemoteOCRConfig()
return cls(
engine=app_config.remote_ocr_engine,
api_key=app_config.remote_ocr_api_key,
endpoint=app_config.remote_ocr_endpoint,
)
def engine_is_valid(self) -> bool: def engine_is_valid(self) -> bool:
"""Return True when the engine is known and fully configured.""" """Return True when the engine is known and fully configured."""
return ( return (
@@ -145,11 +157,7 @@ class RemoteDocumentParser:
20 when the remote engine is configured and the MIME type is 20 when the remote engine is configured and the MIME type is
supported, otherwise None. supported, otherwise None.
""" """
config = RemoteEngineConfig( config = RemoteEngineConfig.from_app_config()
engine=settings.REMOTE_OCR_ENGINE,
api_key=settings.REMOTE_OCR_API_KEY,
endpoint=settings.REMOTE_OCR_ENDPOINT,
)
if not config.engine_is_valid(): if not config.engine_is_valid():
return None return None
if mime_type not in _SUPPORTED_MIME_TYPES: if mime_type not in _SUPPORTED_MIME_TYPES:
@@ -244,11 +252,7 @@ class RemoteDocumentParser:
Whether an archive copy is wanted. For PDFs, False skips the Whether an archive copy is wanted. For PDFs, False skips the
remote engine and uses locally-extracted text instead. remote engine and uses locally-extracted text instead.
""" """
config = RemoteEngineConfig( config = RemoteEngineConfig.from_app_config()
engine=settings.REMOTE_OCR_ENGINE,
api_key=settings.REMOTE_OCR_API_KEY,
endpoint=settings.REMOTE_OCR_ENDPOINT,
)
if not config.engine_is_valid(): if not config.engine_is_valid():
logger.warning( logger.warning(
+14 -5
View File
@@ -219,6 +219,13 @@ class ApplicationConfigurationSerializer(
allow_null=True, allow_null=True,
max_length=1024, max_length=1024,
) )
remote_ocr_api_key = ObfuscatedPasswordField(
required=False,
allow_null=True,
max_length=1024,
)
OBFUSCATED_FIELDS = ("llm_api_key", "remote_ocr_api_key")
def run_validation(self, data): def run_validation(self, data):
# Empty strings treated as None to avoid unexpected behavior # Empty strings treated as None to avoid unexpected behavior
@@ -230,11 +237,13 @@ class ApplicationConfigurationSerializer(
data["language"] = None data["language"] = None
if "llm_output_language" in data and data["llm_output_language"] == "": if "llm_output_language" in data and data["llm_output_language"] == "":
data["llm_output_language"] = None data["llm_output_language"] = None
if "llm_api_key" in data and data["llm_api_key"] is not None: for field in self.OBFUSCATED_FIELDS:
if data["llm_api_key"] == "": if field in data and data[field] is not None:
data["llm_api_key"] = None if data[field] == "":
elif len(data["llm_api_key"].replace("*", "")) == 0: data[field] = None
del data["llm_api_key"] # Not a real value, don't overwrite the stored one
elif len(data[field].replace("*", "")) == 0:
del data[field]
return super().run_validation(data) return super().run_validation(data)
def update(self, instance, validated_data): def update(self, instance, validated_data):
+24 -2
View File
@@ -114,7 +114,26 @@ def remote_parser() -> Generator[RemoteDocumentParser, None, None]:
@pytest.fixture() @pytest.fixture()
def azure_settings(settings: SettingsWrapper) -> SettingsWrapper: def empty_remote_ocr_app_config(mocker: MockerFixture) -> MagicMock:
# empty app config without accessing db
app_config = mocker.MagicMock(
remote_ocr_engine=None,
remote_ocr_api_key=None,
remote_ocr_endpoint=None,
remote_ocr_mode=None,
)
mocker.patch(
"paperless.config.BaseConfig._get_config_instance",
return_value=app_config,
)
return app_config
@pytest.fixture()
def azure_settings(
settings: SettingsWrapper,
empty_remote_ocr_app_config: MagicMock,
) -> SettingsWrapper:
"""Configure Django settings for a valid Azure AI OCR engine. """Configure Django settings for a valid Azure AI OCR engine.
Sets ``REMOTE_OCR_ENGINE``, ``REMOTE_OCR_API_KEY``, and Sets ``REMOTE_OCR_ENGINE``, ``REMOTE_OCR_API_KEY``, and
@@ -133,7 +152,10 @@ def azure_settings(settings: SettingsWrapper) -> SettingsWrapper:
@pytest.fixture() @pytest.fixture()
def no_engine_settings(settings: SettingsWrapper) -> SettingsWrapper: def no_engine_settings(
settings: SettingsWrapper,
empty_remote_ocr_app_config: MagicMock,
) -> SettingsWrapper:
"""Configure Django settings with no remote engine configured. """Configure Django settings with no remote engine configured.
Returns Returns
@@ -21,6 +21,7 @@ from unittest.mock import Mock
import pytest import pytest
from documents.parsers import ParseError from documents.parsers import ParseError
from paperless.models import ApplicationConfiguration
from paperless.parsers import ParserContext from paperless.parsers import ParserContext
from paperless.parsers import ParserProtocol from paperless.parsers import ParserProtocol
from paperless.parsers.remote import RemoteDocumentParser from paperless.parsers.remote import RemoteDocumentParser
@@ -198,21 +199,21 @@ class TestRemoteParserScore:
def test_score_returns_none_when_api_key_missing( def test_score_returns_none_when_api_key_missing(
self, self,
settings: SettingsWrapper, no_engine_settings: SettingsWrapper,
) -> None: ) -> None:
settings.REMOTE_OCR_ENGINE = "azureai" no_engine_settings.REMOTE_OCR_ENGINE = "azureai"
settings.REMOTE_OCR_API_KEY = None no_engine_settings.REMOTE_OCR_ENDPOINT = (
settings.REMOTE_OCR_ENDPOINT = "https://test.cognitiveservices.azure.com" "https://test.cognitiveservices.azure.com"
)
result = RemoteDocumentParser.score("application/pdf", "doc.pdf") result = RemoteDocumentParser.score("application/pdf", "doc.pdf")
assert result is None assert result is None
def test_score_returns_none_when_endpoint_missing( def test_score_returns_none_when_endpoint_missing(
self, self,
settings: SettingsWrapper, no_engine_settings: SettingsWrapper,
) -> None: ) -> None:
settings.REMOTE_OCR_ENGINE = "azureai" no_engine_settings.REMOTE_OCR_ENGINE = "azureai"
settings.REMOTE_OCR_API_KEY = "key" no_engine_settings.REMOTE_OCR_API_KEY = "key"
settings.REMOTE_OCR_ENDPOINT = None
result = RemoteDocumentParser.score("application/pdf", "doc.pdf") result = RemoteDocumentParser.score("application/pdf", "doc.pdf")
assert result is None assert result is None
@@ -227,6 +228,24 @@ class TestRemoteParserScore:
score = RemoteDocumentParser.score("application/pdf", "doc.pdf") score = RemoteDocumentParser.score("application/pdf", "doc.pdf")
assert score is not None and score > 10 assert score is not None and score > 10
@pytest.mark.django_db
def test_score_uses_app_config_when_env_unset(
self,
settings: SettingsWrapper,
) -> None:
"""The app config alone is enough to activate the parser."""
settings.REMOTE_OCR_ENGINE = None
settings.REMOTE_OCR_API_KEY = None
settings.REMOTE_OCR_ENDPOINT = None
config = ApplicationConfiguration.objects.first()
assert config is not None
config.remote_ocr_engine = "azureai"
config.remote_ocr_api_key = "app-config-key"
config.remote_ocr_endpoint = "https://config.cognitiveservices.azure.com"
config.save()
assert RemoteDocumentParser.score("application/pdf", "doc.pdf") == 20
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# Properties # Properties
@@ -1277,6 +1277,8 @@ class TestParserFileTypes:
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# Remote ocr config from ApplicationConfiguration needs DB access
@pytest.mark.django_db
class TestRasterisedDocumentParserRegistry: class TestRasterisedDocumentParserRegistry:
def test_registered_in_defaults(self) -> None: def test_registered_in_defaults(self) -> None:
from paperless.parsers.registry import ParserRegistry from paperless.parsers.registry import ParserRegistry
-26
View File
@@ -15,7 +15,6 @@ from paperless.checks import audit_log_check
from paperless.checks import binaries_check from paperless.checks import binaries_check
from paperless.checks import check_default_language_available from paperless.checks import check_default_language_available
from paperless.checks import check_deprecated_db_settings from paperless.checks import check_deprecated_db_settings
from paperless.checks import check_remote_parser_configured
from paperless.checks import check_v3_minimum_upgrade_version from paperless.checks import check_v3_minimum_upgrade_version
from paperless.checks import debug_mode_check from paperless.checks import debug_mode_check
from paperless.checks import paths_check from paperless.checks import paths_check
@@ -631,31 +630,6 @@ class TestV3MinimumUpgradeVersionCheck:
assert check_v3_minimum_upgrade_version(None) == [] assert check_v3_minimum_upgrade_version(None) == []
class TestRemoteParserChecks:
def test_no_engine(self, settings: SettingsWrapper) -> None:
settings.REMOTE_OCR_ENGINE = None
msgs = check_remote_parser_configured(None)
assert len(msgs) == 0
def test_azure_no_endpoint(self, settings: SettingsWrapper) -> None:
settings.REMOTE_OCR_ENGINE = "azureai"
settings.REMOTE_OCR_API_KEY = "somekey"
settings.REMOTE_OCR_ENDPOINT = None
msgs = check_remote_parser_configured(None)
assert len(msgs) == 1
msg = msgs[0]
assert (
"Azure AI remote parser requires endpoint and API key to be configured."
in msg.msg
)
class TestTesseractChecks: class TestTesseractChecks:
def test_default_language(self) -> None: def test_default_language(self) -> None:
check_default_language_available(None) check_default_language_available(None)
@@ -0,0 +1,88 @@
"""Tests for RemoteOCRConfig precedence between app config and Django settings."""
from __future__ import annotations
from typing import TYPE_CHECKING
import pytest
from django.test import override_settings
from paperless.config import RemoteOCRConfig
if TYPE_CHECKING:
from unittest.mock import MagicMock
@pytest.fixture()
def null_app_config(mocker) -> MagicMock:
"""Mock ApplicationConfiguration with all fields None → falls back to Django settings."""
return mocker.MagicMock(
remote_ocr_engine=None,
remote_ocr_api_key=None,
remote_ocr_endpoint=None,
)
@pytest.fixture()
def make_remote_ocr_config(mocker):
def _make(app_config, **django_settings_overrides):
mocker.patch(
"paperless.config.BaseConfig._get_config_instance",
return_value=app_config,
)
with override_settings(**django_settings_overrides):
return RemoteOCRConfig()
return _make
class TestRemoteOCRConfig:
def test_falls_back_to_settings(
self,
make_remote_ocr_config,
null_app_config,
) -> None:
cfg = make_remote_ocr_config(
null_app_config,
REMOTE_OCR_ENGINE="azureai",
REMOTE_OCR_API_KEY="env-key",
REMOTE_OCR_ENDPOINT="https://env.cognitiveservices.azure.com",
)
assert cfg.remote_ocr_engine == "azureai"
assert cfg.remote_ocr_api_key == "env-key"
assert cfg.remote_ocr_endpoint == "https://env.cognitiveservices.azure.com"
def test_app_config_takes_precedence(
self,
make_remote_ocr_config,
mocker,
) -> None:
app_config = mocker.MagicMock(
remote_ocr_engine="azureai",
remote_ocr_api_key="db-key",
remote_ocr_endpoint="https://db.cognitiveservices.azure.com",
)
cfg = make_remote_ocr_config(
app_config,
REMOTE_OCR_ENGINE=None,
REMOTE_OCR_API_KEY="env-key",
REMOTE_OCR_ENDPOINT="https://env.cognitiveservices.azure.com",
)
assert cfg.remote_ocr_engine == "azureai"
assert cfg.remote_ocr_api_key == "db-key"
assert cfg.remote_ocr_endpoint == "https://db.cognitiveservices.azure.com"
def test_unset_everywhere(
self,
make_remote_ocr_config,
null_app_config,
) -> None:
cfg = make_remote_ocr_config(
null_app_config,
REMOTE_OCR_ENGINE=None,
REMOTE_OCR_API_KEY=None,
REMOTE_OCR_ENDPOINT=None,
)
assert cfg.remote_ocr_engine is None
assert cfg.remote_ocr_api_key is None
assert cfg.remote_ocr_endpoint is None
Generated
+530 -530
View File
File diff suppressed because it is too large Load Diff