mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-08-20 17:53:20 +00:00
Compare commits
18
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
166ee77173 | ||
|
|
9160204bdb | ||
|
|
62e03fcf0c | ||
|
|
ba2c2e1e14 | ||
|
|
ee9e7a7358 | ||
|
|
26c5112d0f | ||
|
|
bd7b816341 | ||
|
|
f21b4892b2 | ||
|
|
655275d5f5 | ||
|
|
a6137487cf | ||
|
|
41b69a826e | ||
|
|
39a868f97d | ||
|
|
c7471608c0 | ||
|
|
965f627929 | ||
|
|
5ee14593fd | ||
|
|
0dfdfdaca9 | ||
|
|
39fb5504d5 | ||
|
|
16c384d331 |
@@ -2066,6 +2066,18 @@ password. All of these options come from their similarly-named [Django settings]
|
|||||||
|
|
||||||
Defaults to None.
|
Defaults to None.
|
||||||
|
|
||||||
|
#### [`PAPERLESS_REMOTE_OCR_MODE=<str>`](#PAPERLESS_REMOTE_OCR_MODE) {#PAPERLESS_REMOTE_OCR_MODE}
|
||||||
|
|
||||||
|
: Which documents are sent to the remote OCR engine.
|
||||||
|
|
||||||
|
- `always`: every document of a supported file type is sent to the remote
|
||||||
|
engine, bypassing the local OCR engine.
|
||||||
|
- `workflow_only`: documents are processed locally unless a workflow
|
||||||
|
explicitly enables remote OCR for them, letting you use the remote engine
|
||||||
|
selectively.
|
||||||
|
|
||||||
|
Defaults to "always".
|
||||||
|
|
||||||
## AI {#ai}
|
## AI {#ai}
|
||||||
|
|
||||||
#### [`PAPERLESS_AI_ENABLED=<bool>`](#PAPERLESS_AI_ENABLED) {#PAPERLESS_AI_ENABLED}
|
#### [`PAPERLESS_AI_ENABLED=<bool>`](#PAPERLESS_AI_ENABLED) {#PAPERLESS_AI_ENABLED}
|
||||||
|
|||||||
@@ -456,6 +456,20 @@ def score(
|
|||||||
return 10
|
return 10
|
||||||
```
|
```
|
||||||
|
|
||||||
|
**Remote services**
|
||||||
|
|
||||||
|
If your parser sends document content to a remote service, declare it:
|
||||||
|
|
||||||
|
```python
|
||||||
|
class MyCustomParser:
|
||||||
|
uses_remote_service = True
|
||||||
|
```
|
||||||
|
|
||||||
|
Paperless-ngx excludes such parsers when the document being consumed has not
|
||||||
|
been marked for remote processing, so users can keep remote OCR off by default
|
||||||
|
and enable it selectively with a workflow. Parsers that do not declare the
|
||||||
|
attribute are treated as fully local and are always considered.
|
||||||
|
|
||||||
**Archive and rendition flags**
|
**Archive and rendition flags**
|
||||||
|
|
||||||
```python
|
```python
|
||||||
|
|||||||
+6
-1
@@ -1090,11 +1090,16 @@ Paperless-ngx supports performing OCR on documents using remote services. At the
|
|||||||
[Microsoft's Azure "Document Intelligence" service](https://azure.microsoft.com/en-us/products/ai-services/ai-document-intelligence).
|
[Microsoft's Azure "Document Intelligence" service](https://azure.microsoft.com/en-us/products/ai-services/ai-document-intelligence).
|
||||||
This is of course a paid service (with a free tier) which requires an Azure account and subscription. Azure AI is not affiliated with
|
This is of course a paid service (with a free tier) which requires an Azure account and subscription. Azure AI is not affiliated with
|
||||||
Paperless-ngx in any way. When enabled, Paperless-ngx will automatically send appropriate documents to Azure for OCR processing, bypassing
|
Paperless-ngx in any way. When enabled, Paperless-ngx will automatically send appropriate documents to Azure for OCR processing, bypassing
|
||||||
the local OCR engine. See the [configuration](configuration.md#PAPERLESS_REMOTE_OCR_ENGINE) options for more details.
|
the local OCR engine. See the [configuration](configuration.md#PAPERLESS_REMOTE_OCR_ENGINE) options for more details. These
|
||||||
|
settings can be supplied as environment variables or via **Application Configuration**.
|
||||||
|
|
||||||
Additionally, when using a commercial service with this feature, consider both potential costs as well as any associated file size
|
Additionally, when using a commercial service with this feature, consider both potential costs as well as any associated file size
|
||||||
or page limitations (e.g. with a free tier).
|
or page limitations (e.g. with a free tier).
|
||||||
|
|
||||||
|
By default, every document of a supported file type is sent to the remote engine. To use it more selectively, set the
|
||||||
|
[remote OCR mode](configuration.md#PAPERLESS_REMOTE_OCR_MODE) to `workflow_only`. Documents are then processed locally
|
||||||
|
unless a workflow explicitly enables remote OCR for them, so you can limit the remote engine to particular documents.
|
||||||
|
|
||||||
## Architecture
|
## Architecture
|
||||||
|
|
||||||
Paperless-ngx consists of the following components:
|
Paperless-ngx consists of the following components:
|
||||||
|
|||||||
+10
-10
@@ -8488,67 +8488,67 @@
|
|||||||
<source>Delete version</source>
|
<source>Delete version</source>
|
||||||
<context-group purpose="location">
|
<context-group purpose="location">
|
||||||
<context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.html</context>
|
<context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.html</context>
|
||||||
<context context-type="linenumber">146,147</context>
|
<context context-type="linenumber">145,146</context>
|
||||||
</context-group>
|
</context-group>
|
||||||
</trans-unit>
|
</trans-unit>
|
||||||
<trans-unit id="9111284310237278846" datatype="html">
|
<trans-unit id="9111284310237278846" datatype="html">
|
||||||
<source>Error deleting version</source>
|
<source>Error deleting version</source>
|
||||||
<context-group purpose="location">
|
<context-group purpose="location">
|
||||||
<context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.ts</context>
|
<context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.ts</context>
|
||||||
<context context-type="linenumber">164</context>
|
<context context-type="linenumber">158</context>
|
||||||
</context-group>
|
</context-group>
|
||||||
</trans-unit>
|
</trans-unit>
|
||||||
<trans-unit id="1450293275912542759" datatype="html">
|
<trans-unit id="1450293275912542759" datatype="html">
|
||||||
<source>Error updating version label</source>
|
<source>Error updating version label</source>
|
||||||
<context-group purpose="location">
|
<context-group purpose="location">
|
||||||
<context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.ts</context>
|
<context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.ts</context>
|
||||||
<context context-type="linenumber">197</context>
|
<context context-type="linenumber">191</context>
|
||||||
</context-group>
|
</context-group>
|
||||||
</trans-unit>
|
</trans-unit>
|
||||||
<trans-unit id="3567158019134759286" datatype="html">
|
<trans-unit id="3567158019134759286" datatype="html">
|
||||||
<source>Uploading new version. Processing will happen in the background.</source>
|
<source>Uploading new version. Processing will happen in the background.</source>
|
||||||
<context-group purpose="location">
|
<context-group purpose="location">
|
||||||
<context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.ts</context>
|
<context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.ts</context>
|
||||||
<context context-type="linenumber">219</context>
|
<context context-type="linenumber">213</context>
|
||||||
</context-group>
|
</context-group>
|
||||||
</trans-unit>
|
</trans-unit>
|
||||||
<trans-unit id="1549684689040001385" datatype="html">
|
<trans-unit id="1549684689040001385" datatype="html">
|
||||||
<source>Missing task ID.</source>
|
<source>Missing task ID.</source>
|
||||||
<context-group purpose="location">
|
<context-group purpose="location">
|
||||||
<context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.ts</context>
|
<context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.ts</context>
|
||||||
<context context-type="linenumber">232</context>
|
<context context-type="linenumber">226</context>
|
||||||
</context-group>
|
</context-group>
|
||||||
</trans-unit>
|
</trans-unit>
|
||||||
<trans-unit id="205445381296663832" datatype="html">
|
<trans-unit id="205445381296663832" datatype="html">
|
||||||
<source>Upload failed.</source>
|
<source>Upload failed.</source>
|
||||||
<context-group purpose="location">
|
<context-group purpose="location">
|
||||||
<context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.ts</context>
|
<context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.ts</context>
|
||||||
<context context-type="linenumber">254</context>
|
<context context-type="linenumber">248</context>
|
||||||
</context-group>
|
</context-group>
|
||||||
<context-group purpose="location">
|
<context-group purpose="location">
|
||||||
<context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.ts</context>
|
<context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.ts</context>
|
||||||
<context context-type="linenumber">278</context>
|
<context context-type="linenumber">272</context>
|
||||||
</context-group>
|
</context-group>
|
||||||
</trans-unit>
|
</trans-unit>
|
||||||
<trans-unit id="8350861980330256599" datatype="html">
|
<trans-unit id="8350861980330256599" datatype="html">
|
||||||
<source>Error uploading new version</source>
|
<source>Error uploading new version</source>
|
||||||
<context-group purpose="location">
|
<context-group purpose="location">
|
||||||
<context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.ts</context>
|
<context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.ts</context>
|
||||||
<context context-type="linenumber">281</context>
|
<context context-type="linenumber">275</context>
|
||||||
</context-group>
|
</context-group>
|
||||||
</trans-unit>
|
</trans-unit>
|
||||||
<trans-unit id="4050545381573552340" datatype="html">
|
<trans-unit id="4050545381573552340" datatype="html">
|
||||||
<source>Existing document added as a version.</source>
|
<source>Existing document added as a version.</source>
|
||||||
<context-group purpose="location">
|
<context-group purpose="location">
|
||||||
<context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.ts</context>
|
<context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.ts</context>
|
||||||
<context context-type="linenumber">324</context>
|
<context context-type="linenumber">318</context>
|
||||||
</context-group>
|
</context-group>
|
||||||
</trans-unit>
|
</trans-unit>
|
||||||
<trans-unit id="8552490241798412439" datatype="html">
|
<trans-unit id="8552490241798412439" datatype="html">
|
||||||
<source>Error adding existing document as a version</source>
|
<source>Error adding existing document as a version</source>
|
||||||
<context-group purpose="location">
|
<context-group purpose="location">
|
||||||
<context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.ts</context>
|
<context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.ts</context>
|
||||||
<context context-type="linenumber">329</context>
|
<context context-type="linenumber">323</context>
|
||||||
</context-group>
|
</context-group>
|
||||||
</trans-unit>
|
</trans-unit>
|
||||||
<trans-unit id="6299008920007331381" datatype="html">
|
<trans-unit id="6299008920007331381" datatype="html">
|
||||||
|
|||||||
@@ -14,43 +14,48 @@
|
|||||||
<a ngbNavLink>{{category}}</a>
|
<a ngbNavLink>{{category}}</a>
|
||||||
<ng-template ngbNavContent>
|
<ng-template ngbNavContent>
|
||||||
<div class="p-3">
|
<div class="p-3">
|
||||||
<div class="row row-cols-1 row-cols-md-2 row-cols-lg-3 g-2">
|
@for (section of getCategorySections(category); track section) {
|
||||||
@for (option of getCategoryOptions(category); track option.key) {
|
@if (section) {
|
||||||
<div class="col">
|
<h5 class="mt-4 mb-3">{{section}}</h5>
|
||||||
<div class="card bg-light">
|
}
|
||||||
<div class="card-body">
|
<div class="row row-cols-1 row-cols-md-2 row-cols-lg-3 g-2">
|
||||||
<div class="card-title d-flex align-items-center">
|
@for (option of getCategoryOptions(category, section); track option.key) {
|
||||||
<h6 class="mb-0">
|
<div class="col">
|
||||||
{{option.title}}
|
<div class="card bg-light">
|
||||||
</h6>
|
<div class="card-body">
|
||||||
<a class="btn btn-sm btn-link" title="Read the documentation about this setting" i18n-title [href]="getDocsUrl(option.config_key)" target="_blank" referrerpolicy="no-referrer">
|
<div class="card-title d-flex align-items-center">
|
||||||
<i-bs name="info-circle"></i-bs>
|
<h6 class="mb-0">
|
||||||
</a>
|
{{option.title}}
|
||||||
@if (isSet(option.key)) {
|
</h6>
|
||||||
<button type="button" class="btn btn-sm btn-link text-danger ms-auto pe-0" title="Reset" i18n-title (click)="resetOption(option.key)">
|
<a class="btn btn-sm btn-link" title="Read the documentation about this setting" i18n-title [href]="getDocsUrl(option.config_key)" target="_blank" referrerpolicy="no-referrer">
|
||||||
<i-bs class="me-1" name="x"></i-bs><ng-container i18n>Reset</ng-container>
|
<i-bs name="info-circle"></i-bs>
|
||||||
</button>
|
</a>
|
||||||
|
@if (isSet(option.key)) {
|
||||||
|
<button type="button" class="btn btn-sm btn-link text-danger ms-auto pe-0" title="Reset" i18n-title (click)="resetOption(option.key)">
|
||||||
|
<i-bs class="me-1" name="x"></i-bs><ng-container i18n>Reset</ng-container>
|
||||||
|
</button>
|
||||||
|
}
|
||||||
|
</div>
|
||||||
|
<div class="mb-n3">
|
||||||
|
@switch (option.type) {
|
||||||
|
@case (ConfigOptionType.Select) { <pngx-input-select [formControlName]="option.key" [error]="errors[option.key]" [items]="option.choices" [allowNull]="true"></pngx-input-select> }
|
||||||
|
@case (ConfigOptionType.Number) { <pngx-input-number [formControlName]="option.key" [error]="errors[option.key]" [showAdd]="false"></pngx-input-number> }
|
||||||
|
@case (ConfigOptionType.Boolean) { <pngx-input-switch [formControlName]="option.key" [error]="errors[option.key]" [showUnsetNote]="true" [horizontal]="true" title="Enable" i18n-title></pngx-input-switch> }
|
||||||
|
@case (ConfigOptionType.String) { <pngx-input-text [formControlName]="option.key" [error]="errors[option.key]"></pngx-input-text> }
|
||||||
|
@case (ConfigOptionType.JSON) { <pngx-input-text [formControlName]="option.key" [error]="errors[option.key]"></pngx-input-text> }
|
||||||
|
@case (ConfigOptionType.File) { <pngx-input-file [formControlName]="option.key" (upload)="uploadFile($event, option.key)" [error]="errors[option.key]"></pngx-input-file> }
|
||||||
|
@case (ConfigOptionType.Password) { <pngx-input-password [formControlName]="option.key" [error]="errors[option.key]"></pngx-input-password> }
|
||||||
|
}
|
||||||
|
</div>
|
||||||
|
@if (option.note) {
|
||||||
|
<div class="form-text fst-italic">{{option.note}}</div>
|
||||||
}
|
}
|
||||||
</div>
|
</div>
|
||||||
<div class="mb-n3">
|
|
||||||
@switch (option.type) {
|
|
||||||
@case (ConfigOptionType.Select) { <pngx-input-select [formControlName]="option.key" [error]="errors[option.key]" [items]="option.choices" [allowNull]="true"></pngx-input-select> }
|
|
||||||
@case (ConfigOptionType.Number) { <pngx-input-number [formControlName]="option.key" [error]="errors[option.key]" [showAdd]="false"></pngx-input-number> }
|
|
||||||
@case (ConfigOptionType.Boolean) { <pngx-input-switch [formControlName]="option.key" [error]="errors[option.key]" [showUnsetNote]="true" [horizontal]="true" title="Enable" i18n-title></pngx-input-switch> }
|
|
||||||
@case (ConfigOptionType.String) { <pngx-input-text [formControlName]="option.key" [error]="errors[option.key]"></pngx-input-text> }
|
|
||||||
@case (ConfigOptionType.JSON) { <pngx-input-text [formControlName]="option.key" [error]="errors[option.key]"></pngx-input-text> }
|
|
||||||
@case (ConfigOptionType.File) { <pngx-input-file [formControlName]="option.key" (upload)="uploadFile($event, option.key)" [error]="errors[option.key]"></pngx-input-file> }
|
|
||||||
@case (ConfigOptionType.Password) { <pngx-input-password [formControlName]="option.key" [error]="errors[option.key]"></pngx-input-password> }
|
|
||||||
}
|
|
||||||
</div>
|
|
||||||
@if (option.note) {
|
|
||||||
<div class="form-text fst-italic">{{option.note}}</div>
|
|
||||||
}
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
}
|
||||||
}
|
</div>
|
||||||
</div>
|
}
|
||||||
</div>
|
</div>
|
||||||
</ng-template>
|
</ng-template>
|
||||||
</li>
|
</li>
|
||||||
|
|||||||
@@ -8,7 +8,11 @@ import { NgbModule } from '@ng-bootstrap/ng-bootstrap'
|
|||||||
import { NgSelectModule } from '@ng-select/ng-select'
|
import { NgSelectModule } from '@ng-select/ng-select'
|
||||||
import { NgxBootstrapIconsModule, allIcons } from 'ngx-bootstrap-icons'
|
import { NgxBootstrapIconsModule, allIcons } from 'ngx-bootstrap-icons'
|
||||||
import { of, throwError } from 'rxjs'
|
import { of, throwError } from 'rxjs'
|
||||||
import { OutputTypeConfig } from 'src/app/data/paperless-config'
|
import {
|
||||||
|
ConfigCategory,
|
||||||
|
ConfigSection,
|
||||||
|
OutputTypeConfig,
|
||||||
|
} from 'src/app/data/paperless-config'
|
||||||
import { ConfigService } from 'src/app/services/config.service'
|
import { ConfigService } from 'src/app/services/config.service'
|
||||||
import { SettingsService } from 'src/app/services/settings.service'
|
import { SettingsService } from 'src/app/services/settings.service'
|
||||||
import { ToastService } from 'src/app/services/toast.service'
|
import { ToastService } from 'src/app/services/toast.service'
|
||||||
@@ -158,4 +162,24 @@ describe('ConfigComponent', () => {
|
|||||||
component.resetOption('barcodes_enabled')
|
component.resetOption('barcodes_enabled')
|
||||||
expect(component.configForm.get('barcodes_enabled').value).toBeNull()
|
expect(component.configForm.get('barcodes_enabled').value).toBeNull()
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it('should group options into sections within a category, or not', () => {
|
||||||
|
const sections = component.getCategorySections(ConfigCategory.OCR)
|
||||||
|
expect(sections).toEqual([null, ConfigSection.RemoteOCR])
|
||||||
|
expect(
|
||||||
|
component
|
||||||
|
.getCategoryOptions(ConfigCategory.OCR)
|
||||||
|
.map((option) => option.key)
|
||||||
|
).toContain('output_type')
|
||||||
|
expect(
|
||||||
|
component
|
||||||
|
.getCategoryOptions(ConfigCategory.OCR, ConfigSection.RemoteOCR)
|
||||||
|
.map((option) => option.key)
|
||||||
|
).toEqual([
|
||||||
|
'remote_ocr_engine',
|
||||||
|
'remote_ocr_api_key',
|
||||||
|
'remote_ocr_endpoint',
|
||||||
|
'remote_ocr_mode',
|
||||||
|
])
|
||||||
|
})
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -74,8 +74,20 @@ export class ConfigComponent
|
|||||||
return Object.values(ConfigCategory)
|
return Object.values(ConfigCategory)
|
||||||
}
|
}
|
||||||
|
|
||||||
getCategoryOptions(category: string): ConfigOption[] {
|
getCategorySections(category: string): string[] {
|
||||||
return PaperlessConfigOptions.filter((o) => o.category === category)
|
return [
|
||||||
|
...new Set(
|
||||||
|
PaperlessConfigOptions.filter((o) => o.category === category).map(
|
||||||
|
(o) => o.section ?? null // null means no section
|
||||||
|
)
|
||||||
|
),
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
getCategoryOptions(category: string, section: string = null): ConfigOption[] {
|
||||||
|
return PaperlessConfigOptions.filter(
|
||||||
|
(o) => o.category === category && (o.section ?? null) === section
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
initialConfig: PaperlessConfig
|
initialConfig: PaperlessConfig
|
||||||
|
|||||||
-1
@@ -141,7 +141,6 @@
|
|||||||
i18n-confirmMessage
|
i18n-confirmMessage
|
||||||
[disabled]="!userIsOwner || !userCanEdit"
|
[disabled]="!userIsOwner || !userCanEdit"
|
||||||
(confirm)="deleteVersion(version.id)"
|
(confirm)="deleteVersion(version.id)"
|
||||||
*pngxIfPermissions="{ action: PermissionAction.Delete, type: PermissionType.Document }"
|
|
||||||
>
|
>
|
||||||
<span class="visually-hidden" i18n>Delete version</span>
|
<span class="visually-hidden" i18n>Delete version</span>
|
||||||
</pngx-confirm-button>
|
</pngx-confirm-button>
|
||||||
|
|||||||
-41
@@ -1,16 +1,10 @@
|
|||||||
import { DatePipe } from '@angular/common'
|
import { DatePipe } from '@angular/common'
|
||||||
import { SimpleChange, signal } from '@angular/core'
|
import { SimpleChange, signal } from '@angular/core'
|
||||||
import { ComponentFixture, TestBed } from '@angular/core/testing'
|
import { ComponentFixture, TestBed } from '@angular/core/testing'
|
||||||
import { By } from '@angular/platform-browser'
|
|
||||||
import { NgbModal } from '@ng-bootstrap/ng-bootstrap'
|
import { NgbModal } from '@ng-bootstrap/ng-bootstrap'
|
||||||
import { NgxBootstrapIconsModule, allIcons } from 'ngx-bootstrap-icons'
|
import { NgxBootstrapIconsModule, allIcons } from 'ngx-bootstrap-icons'
|
||||||
import { Subject, of, throwError } from 'rxjs'
|
import { Subject, of, throwError } from 'rxjs'
|
||||||
import { DocumentVersionInfo } from 'src/app/data/document'
|
import { DocumentVersionInfo } from 'src/app/data/document'
|
||||||
import {
|
|
||||||
PermissionAction,
|
|
||||||
PermissionsService,
|
|
||||||
PermissionType,
|
|
||||||
} from 'src/app/services/permissions.service'
|
|
||||||
import { DocumentService } from 'src/app/services/rest/document.service'
|
import { DocumentService } from 'src/app/services/rest/document.service'
|
||||||
import { SettingsService } from 'src/app/services/settings.service'
|
import { SettingsService } from 'src/app/services/settings.service'
|
||||||
import { ToastService } from 'src/app/services/toast.service'
|
import { ToastService } from 'src/app/services/toast.service'
|
||||||
@@ -37,9 +31,6 @@ describe('DocumentVersionDropdownComponent', () => {
|
|||||||
let finished$: Subject<{ taskId: string }>
|
let finished$: Subject<{ taskId: string }>
|
||||||
let failed$: Subject<{ taskId: string; message?: string }>
|
let failed$: Subject<{ taskId: string; message?: string }>
|
||||||
let modalService: jest.Mocked<Pick<NgbModal, 'open'>>
|
let modalService: jest.Mocked<Pick<NgbModal, 'open'>>
|
||||||
let permissionsService: jest.Mocked<
|
|
||||||
Pick<PermissionsService, 'currentUserCan'>
|
|
||||||
>
|
|
||||||
|
|
||||||
beforeEach(async () => {
|
beforeEach(async () => {
|
||||||
finished$ = new Subject<{ taskId: string }>()
|
finished$ = new Subject<{ taskId: string }>()
|
||||||
@@ -56,9 +47,6 @@ describe('DocumentVersionDropdownComponent', () => {
|
|||||||
showError: jest.fn(),
|
showError: jest.fn(),
|
||||||
showInfo: jest.fn(),
|
showInfo: jest.fn(),
|
||||||
}
|
}
|
||||||
permissionsService = {
|
|
||||||
currentUserCan: jest.fn().mockReturnValue(true),
|
|
||||||
}
|
|
||||||
|
|
||||||
await TestBed.configureTestingModule({
|
await TestBed.configureTestingModule({
|
||||||
imports: [
|
imports: [
|
||||||
@@ -85,10 +73,6 @@ describe('DocumentVersionDropdownComponent', () => {
|
|||||||
provide: NgbModal,
|
provide: NgbModal,
|
||||||
useValue: modalService,
|
useValue: modalService,
|
||||||
},
|
},
|
||||||
{
|
|
||||||
provide: PermissionsService,
|
|
||||||
useValue: permissionsService,
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
provide: WebsocketStatusService,
|
provide: WebsocketStatusService,
|
||||||
useValue: {
|
useValue: {
|
||||||
@@ -159,31 +143,6 @@ describe('DocumentVersionDropdownComponent', () => {
|
|||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should not show version delete buttons without document delete permission', () => {
|
|
||||||
fixture.destroy()
|
|
||||||
permissionsService.currentUserCan.mockReturnValue(false)
|
|
||||||
fixture = TestBed.createComponent(DocumentVersionDropdownComponent)
|
|
||||||
component = fixture.componentInstance
|
|
||||||
component.documentId = 3
|
|
||||||
component.selectedVersionId = 3
|
|
||||||
component.userIsOwner = true
|
|
||||||
component.userCanEdit = true
|
|
||||||
component.versions = [
|
|
||||||
{ id: 3, is_root: true, checksum: 'aaaa' },
|
|
||||||
{ id: 10, is_root: false, checksum: 'bbbb' },
|
|
||||||
]
|
|
||||||
|
|
||||||
fixture.detectChanges()
|
|
||||||
|
|
||||||
expect(permissionsService.currentUserCan).toHaveBeenCalledWith(
|
|
||||||
PermissionAction.Delete,
|
|
||||||
PermissionType.Document
|
|
||||||
)
|
|
||||||
expect(
|
|
||||||
fixture.debugElement.queryAll(By.css('pngx-confirm-button'))
|
|
||||||
).toHaveLength(0)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('beginEditingVersion should set active row and draft label', () => {
|
it('beginEditingVersion should set active row and draft label', () => {
|
||||||
component.userCanEdit = true
|
component.userCanEdit = true
|
||||||
component.userIsOwner = true
|
component.userIsOwner = true
|
||||||
|
|||||||
+1
-7
@@ -25,7 +25,6 @@ import {
|
|||||||
tap,
|
tap,
|
||||||
} from 'rxjs/operators'
|
} from 'rxjs/operators'
|
||||||
import { DocumentVersionInfo } from 'src/app/data/document'
|
import { DocumentVersionInfo } from 'src/app/data/document'
|
||||||
import { IfPermissionsDirective } from 'src/app/directives/if-permissions.directive'
|
|
||||||
import { CustomDatePipe } from 'src/app/pipes/custom-date.pipe'
|
import { CustomDatePipe } from 'src/app/pipes/custom-date.pipe'
|
||||||
import { DocumentService } from 'src/app/services/rest/document.service'
|
import { DocumentService } from 'src/app/services/rest/document.service'
|
||||||
import { ToastService } from 'src/app/services/toast.service'
|
import { ToastService } from 'src/app/services/toast.service'
|
||||||
@@ -34,7 +33,6 @@ import {
|
|||||||
WebsocketStatusService,
|
WebsocketStatusService,
|
||||||
} from 'src/app/services/websocket-status.service'
|
} from 'src/app/services/websocket-status.service'
|
||||||
import { ConfirmButtonComponent } from '../../common/confirm-button/confirm-button.component'
|
import { ConfirmButtonComponent } from '../../common/confirm-button/confirm-button.component'
|
||||||
import { ComponentWithPermissions } from '../../with-permissions/with-permissions.component'
|
|
||||||
import { AddExistingDocumentVersionDialogComponent } from './add-existing-document-version-dialog/add-existing-document-version-dialog.component'
|
import { AddExistingDocumentVersionDialogComponent } from './add-existing-document-version-dialog/add-existing-document-version-dialog.component'
|
||||||
|
|
||||||
@Component({
|
@Component({
|
||||||
@@ -46,15 +44,11 @@ import { AddExistingDocumentVersionDialogComponent } from './add-existing-docume
|
|||||||
NgbDropdownModule,
|
NgbDropdownModule,
|
||||||
NgxBootstrapIconsModule,
|
NgxBootstrapIconsModule,
|
||||||
ConfirmButtonComponent,
|
ConfirmButtonComponent,
|
||||||
IfPermissionsDirective,
|
|
||||||
SlicePipe,
|
SlicePipe,
|
||||||
CustomDatePipe,
|
CustomDatePipe,
|
||||||
],
|
],
|
||||||
})
|
})
|
||||||
export class DocumentVersionDropdownComponent
|
export class DocumentVersionDropdownComponent implements OnChanges, OnDestroy {
|
||||||
extends ComponentWithPermissions
|
|
||||||
implements OnChanges, OnDestroy
|
|
||||||
{
|
|
||||||
UploadState = UploadState
|
UploadState = UploadState
|
||||||
|
|
||||||
@Input() documentId: number
|
@Input() documentId: number
|
||||||
|
|||||||
@@ -54,6 +54,10 @@ export const ConfigCategory = {
|
|||||||
AI: $localize`AI Settings`,
|
AI: $localize`AI Settings`,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export const ConfigSection = {
|
||||||
|
RemoteOCR: $localize`Remote OCR`,
|
||||||
|
}
|
||||||
|
|
||||||
export const LLMEmbeddingBackendConfig = {
|
export const LLMEmbeddingBackendConfig = {
|
||||||
OPENAI_LIKE: 'openai-like',
|
OPENAI_LIKE: 'openai-like',
|
||||||
HUGGINGFACE: 'huggingface',
|
HUGGINGFACE: 'huggingface',
|
||||||
@@ -65,6 +69,15 @@ export const LLMBackendConfig = {
|
|||||||
OLLAMA: 'ollama',
|
OLLAMA: 'ollama',
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export const RemoteOCREngineConfig = {
|
||||||
|
AZURE_AI: 'azureai',
|
||||||
|
}
|
||||||
|
|
||||||
|
export const RemoteOCRModeConfig = {
|
||||||
|
ALWAYS: 'always',
|
||||||
|
WORKFLOW_ONLY: 'workflow_only',
|
||||||
|
}
|
||||||
|
|
||||||
export interface ConfigOption {
|
export interface ConfigOption {
|
||||||
key: string
|
key: string
|
||||||
title: string
|
title: string
|
||||||
@@ -72,6 +85,7 @@ export interface ConfigOption {
|
|||||||
choices?: Array<{ id: string; name: string }>
|
choices?: Array<{ id: string; name: string }>
|
||||||
config_key?: string
|
config_key?: string
|
||||||
category: string
|
category: string
|
||||||
|
section?: string
|
||||||
note?: string
|
note?: string
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -181,6 +195,43 @@ export const PaperlessConfigOptions: ConfigOption[] = [
|
|||||||
config_key: 'PAPERLESS_OCR_USER_ARGS',
|
config_key: 'PAPERLESS_OCR_USER_ARGS',
|
||||||
category: ConfigCategory.OCR,
|
category: ConfigCategory.OCR,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
key: 'remote_ocr_engine',
|
||||||
|
title: $localize`Remote OCR Engine`,
|
||||||
|
type: ConfigOptionType.Select,
|
||||||
|
choices: mapToItems(RemoteOCREngineConfig),
|
||||||
|
config_key: 'PAPERLESS_REMOTE_OCR_ENGINE',
|
||||||
|
category: ConfigCategory.OCR,
|
||||||
|
section: ConfigSection.RemoteOCR,
|
||||||
|
note: $localize`Enabling remote OCR sends documents to a third-party service for processing. Consider the privacy implications as well as potential costs before enabling.`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
key: 'remote_ocr_api_key',
|
||||||
|
title: $localize`Remote OCR API Key`,
|
||||||
|
type: ConfigOptionType.Password,
|
||||||
|
config_key: 'PAPERLESS_REMOTE_OCR_API_KEY',
|
||||||
|
category: ConfigCategory.OCR,
|
||||||
|
section: ConfigSection.RemoteOCR,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
key: 'remote_ocr_endpoint',
|
||||||
|
title: $localize`Remote OCR Endpoint`,
|
||||||
|
type: ConfigOptionType.String,
|
||||||
|
config_key: 'PAPERLESS_REMOTE_OCR_ENDPOINT',
|
||||||
|
category: ConfigCategory.OCR,
|
||||||
|
section: ConfigSection.RemoteOCR,
|
||||||
|
note: $localize`Required when using the Azure AI engine.`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
key: 'remote_ocr_mode',
|
||||||
|
title: $localize`Remote OCR Mode`,
|
||||||
|
type: ConfigOptionType.Select,
|
||||||
|
choices: mapToItems(RemoteOCRModeConfig),
|
||||||
|
config_key: 'PAPERLESS_REMOTE_OCR_MODE',
|
||||||
|
category: ConfigCategory.OCR,
|
||||||
|
section: ConfigSection.RemoteOCR,
|
||||||
|
note: $localize`Which documents are sent to the remote engine. Use 'workflow_only' to keep remote OCR off unless a workflow enables it for a document.`,
|
||||||
|
},
|
||||||
{
|
{
|
||||||
key: 'app_logo',
|
key: 'app_logo',
|
||||||
title: $localize`Application Logo`,
|
title: $localize`Application Logo`,
|
||||||
@@ -398,6 +449,10 @@ export interface PaperlessConfig extends ObjectWithId {
|
|||||||
barcode_enable_tag: boolean
|
barcode_enable_tag: boolean
|
||||||
barcode_tag_mapping: object
|
barcode_tag_mapping: object
|
||||||
barcode_tag_split: boolean
|
barcode_tag_split: boolean
|
||||||
|
remote_ocr_engine: string
|
||||||
|
remote_ocr_api_key: string
|
||||||
|
remote_ocr_endpoint: string
|
||||||
|
remote_ocr_mode: string
|
||||||
ai_enabled: boolean
|
ai_enabled: boolean
|
||||||
llm_embedding_backend: string
|
llm_embedding_backend: string
|
||||||
llm_embedding_model: string
|
llm_embedding_model: string
|
||||||
|
|||||||
@@ -53,6 +53,7 @@ from documents.utils import copy_basic_file_stats
|
|||||||
from documents.utils import copy_file_with_basic_stats
|
from documents.utils import copy_file_with_basic_stats
|
||||||
from documents.utils import run_subprocess
|
from documents.utils import run_subprocess
|
||||||
from paperless.config import OcrConfig
|
from paperless.config import OcrConfig
|
||||||
|
from paperless.config import RemoteOCRConfig
|
||||||
from paperless.models import ArchiveFileGenerationChoices
|
from paperless.models import ArchiveFileGenerationChoices
|
||||||
from paperless.parsers import ParserContext
|
from paperless.parsers import ParserContext
|
||||||
from paperless.parsers import ParserProtocol
|
from paperless.parsers import ParserProtocol
|
||||||
@@ -451,12 +452,19 @@ class ConsumerPlugin(
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
self.log.error(f"Error attempting to clean PDF: {e}")
|
self.log.error(f"Error attempting to clean PDF: {e}")
|
||||||
|
|
||||||
|
# Workflows have already run at this point, so the metadata knows
|
||||||
|
# whether this document was singled out for remote OCR
|
||||||
|
allow_remote = (
|
||||||
|
self.metadata.remote_ocr or RemoteOCRConfig().remote_ocr_by_default
|
||||||
|
)
|
||||||
|
|
||||||
# Based on the mime type, get the parser for that type
|
# Based on the mime type, get the parser for that type
|
||||||
parser_class: type[ParserProtocol] | None = (
|
parser_class: type[ParserProtocol] | None = (
|
||||||
get_parser_registry().get_parser_for_file(
|
get_parser_registry().get_parser_for_file(
|
||||||
mime_type,
|
mime_type,
|
||||||
self.filename,
|
self.filename,
|
||||||
self.working_copy,
|
self.working_copy,
|
||||||
|
allow_remote=allow_remote,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
if not parser_class:
|
if not parser_class:
|
||||||
|
|||||||
@@ -34,6 +34,7 @@ class DocumentMetadataOverrides:
|
|||||||
skip_asn_if_exists: bool = False
|
skip_asn_if_exists: bool = False
|
||||||
version_label: str | None = None
|
version_label: str | None = None
|
||||||
actor_id: int | None = None
|
actor_id: int | None = None
|
||||||
|
remote_ocr: bool = False
|
||||||
|
|
||||||
def update(self, other: "DocumentMetadataOverrides") -> "DocumentMetadataOverrides":
|
def update(self, other: "DocumentMetadataOverrides") -> "DocumentMetadataOverrides":
|
||||||
"""
|
"""
|
||||||
@@ -57,6 +58,8 @@ class DocumentMetadataOverrides:
|
|||||||
self.actor_id = other.actor_id
|
self.actor_id = other.actor_id
|
||||||
if other.skip_asn_if_exists:
|
if other.skip_asn_if_exists:
|
||||||
self.skip_asn_if_exists = True
|
self.skip_asn_if_exists = True
|
||||||
|
if other.remote_ocr:
|
||||||
|
self.remote_ocr = True
|
||||||
if other.version_label is not None:
|
if other.version_label is not None:
|
||||||
self.version_label = other.version_label
|
self.version_label = other.version_label
|
||||||
|
|
||||||
|
|||||||
@@ -266,7 +266,11 @@ class WriteBatch:
|
|||||||
if self._lock is not None:
|
if self._lock is not None:
|
||||||
self._lock.release()
|
self._lock.release()
|
||||||
|
|
||||||
def add_or_update(self, document: Document) -> None:
|
def add_or_update(
|
||||||
|
self,
|
||||||
|
document: Document,
|
||||||
|
effective_content: str | None = None,
|
||||||
|
) -> None:
|
||||||
"""
|
"""
|
||||||
Add or update a document in the batch.
|
Add or update a document in the batch.
|
||||||
|
|
||||||
@@ -276,9 +280,11 @@ class WriteBatch:
|
|||||||
|
|
||||||
Args:
|
Args:
|
||||||
document: Django Document instance to index
|
document: Django Document instance to index
|
||||||
|
effective_content: Override document.content for indexing (used when
|
||||||
|
re-indexing with newer OCR text from document versions)
|
||||||
"""
|
"""
|
||||||
self.remove(document.pk)
|
self.remove(document.pk)
|
||||||
doc = self._backend._build_tantivy_doc(document)
|
doc = self._backend._build_tantivy_doc(document, effective_content)
|
||||||
self._writer.add_document(doc)
|
self._writer.add_document(doc)
|
||||||
|
|
||||||
def remove(self, doc_id: int) -> None:
|
def remove(self, doc_id: int) -> None:
|
||||||
@@ -419,17 +425,18 @@ class TantivyBackend:
|
|||||||
def _build_tantivy_doc(
|
def _build_tantivy_doc(
|
||||||
self,
|
self,
|
||||||
document: Document,
|
document: Document,
|
||||||
|
effective_content: str | None = None,
|
||||||
viewer_ids: list[int] | None = None,
|
viewer_ids: list[int] | None = None,
|
||||||
viewer_group_ids: list[int] | None = None,
|
viewer_group_ids: list[int] | None = None,
|
||||||
) -> tantivy.Document:
|
) -> tantivy.Document:
|
||||||
"""Build a tantivy Document from a Django Document instance.
|
"""Build a tantivy Document from a Django Document instance.
|
||||||
|
|
||||||
A root document is indexed with its effective content, i.e. the newest
|
``effective_content`` overrides ``document.content`` for indexing —
|
||||||
version's OCR text, so it is never indexed with its own outdated text.
|
used when re-indexing a root document with a newer version's OCR text.
|
||||||
Annotate the queryset with ``annotate_effective_content`` when indexing
|
|
||||||
more than a couple of documents, to resolve that without a query each.
|
|
||||||
"""
|
"""
|
||||||
content = document.get_effective_content() or ""
|
content = (
|
||||||
|
effective_content if effective_content is not None else document.content
|
||||||
|
)
|
||||||
|
|
||||||
doc = tantivy.Document()
|
doc = tantivy.Document()
|
||||||
|
|
||||||
@@ -577,7 +584,11 @@ class TantivyBackend:
|
|||||||
|
|
||||||
return doc
|
return doc
|
||||||
|
|
||||||
def add_or_update(self, document: Document) -> None:
|
def add_or_update(
|
||||||
|
self,
|
||||||
|
document: Document,
|
||||||
|
effective_content: str | None = None,
|
||||||
|
) -> None:
|
||||||
"""
|
"""
|
||||||
Add or update a single document with file locking.
|
Add or update a single document with file locking.
|
||||||
|
|
||||||
@@ -590,11 +601,12 @@ class TantivyBackend:
|
|||||||
|
|
||||||
Args:
|
Args:
|
||||||
document: Django Document instance to index
|
document: Django Document instance to index
|
||||||
|
effective_content: Override document.content for indexing
|
||||||
"""
|
"""
|
||||||
self._ensure_open()
|
self._ensure_open()
|
||||||
try:
|
try:
|
||||||
with self.batch_update(lock_timeout=_LOCK_TIMEOUT_SECONDS) as batch:
|
with self.batch_update(lock_timeout=_LOCK_TIMEOUT_SECONDS) as batch:
|
||||||
batch.add_or_update(document)
|
batch.add_or_update(document, effective_content)
|
||||||
except SearchIndexLockError:
|
except SearchIndexLockError:
|
||||||
logger.error(
|
logger.error(
|
||||||
"Search index lock exhausted for document %d after %d attempts; "
|
"Search index lock exhausted for document %d after %d attempts; "
|
||||||
@@ -1015,6 +1027,7 @@ class TantivyBackend:
|
|||||||
):
|
):
|
||||||
doc = self._build_tantivy_doc(
|
doc = self._build_tantivy_doc(
|
||||||
document,
|
document,
|
||||||
|
document.get_effective_content(),
|
||||||
viewer_ids=viewer_ids,
|
viewer_ids=viewer_ids,
|
||||||
viewer_group_ids=viewer_group_ids,
|
viewer_group_ids=viewer_group_ids,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -794,12 +794,10 @@ def cleanup_user_deletion(sender, instance: User | Group, **kwargs) -> None:
|
|||||||
def add_to_index(sender, document, **kwargs) -> None:
|
def add_to_index(sender, document, **kwargs) -> None:
|
||||||
from documents.search import get_backend
|
from documents.search import get_backend
|
||||||
|
|
||||||
# A newly consumed version is not searchable on its own, its content
|
get_backend().add_or_update(
|
||||||
# becomes the effective_content of the root document
|
document,
|
||||||
if document.root_document_id:
|
effective_content=document.get_effective_content(),
|
||||||
document = document.root_document
|
)
|
||||||
|
|
||||||
get_backend().add_or_update(document)
|
|
||||||
|
|
||||||
|
|
||||||
def run_workflows_added(
|
def run_workflows_added(
|
||||||
|
|||||||
+15
-7
@@ -64,9 +64,9 @@ from documents.signals.handlers import send_websocket_document_updated
|
|||||||
from documents.utils import IterWrapper
|
from documents.utils import IterWrapper
|
||||||
from documents.utils import compute_checksum
|
from documents.utils import compute_checksum
|
||||||
from documents.utils import identity
|
from documents.utils import identity
|
||||||
from documents.versioning import annotate_effective_content
|
|
||||||
from documents.workflows.utils import get_workflows_for_trigger
|
from documents.workflows.utils import get_workflows_for_trigger
|
||||||
from paperless.config import AIConfig
|
from paperless.config import AIConfig
|
||||||
|
from paperless.config import RemoteOCRConfig
|
||||||
from paperless.logging import consume_task_id
|
from paperless.logging import consume_task_id
|
||||||
from paperless.parsers import ParserContext
|
from paperless.parsers import ParserContext
|
||||||
from paperless.parsers.registry import get_parser_registry
|
from paperless.parsers.registry import get_parser_registry
|
||||||
@@ -115,7 +115,10 @@ def index_document(self, document_id: int) -> None:
|
|||||||
)
|
)
|
||||||
return
|
return
|
||||||
with get_backend().batch_update() as batch:
|
with get_backend().batch_update() as batch:
|
||||||
batch.add_or_update(document)
|
batch.add_or_update(
|
||||||
|
document,
|
||||||
|
effective_content=document.get_effective_content(),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@shared_task(
|
@shared_task(
|
||||||
@@ -310,10 +313,7 @@ def bulk_update_documents(document_ids) -> None:
|
|||||||
from documents.search import get_backend
|
from documents.search import get_backend
|
||||||
|
|
||||||
document_ids = list(document_ids)
|
document_ids = list(document_ids)
|
||||||
# Annotated so indexing below doesn't query the versions of each document
|
documents = Document.objects.filter(id__in=document_ids)
|
||||||
documents = annotate_effective_content(
|
|
||||||
Document.objects.filter(id__in=document_ids),
|
|
||||||
)
|
|
||||||
|
|
||||||
for doc in documents:
|
for doc in documents:
|
||||||
clear_document_caches(doc.pk)
|
clear_document_caches(doc.pk)
|
||||||
@@ -338,10 +338,17 @@ def bulk_update_documents(document_ids) -> None:
|
|||||||
|
|
||||||
|
|
||||||
@shared_task
|
@shared_task
|
||||||
def update_document_content_maybe_archive_file(document_id) -> None:
|
def update_document_content_maybe_archive_file(
|
||||||
|
document_id,
|
||||||
|
*,
|
||||||
|
remote_ocr: bool = False,
|
||||||
|
) -> None:
|
||||||
"""
|
"""
|
||||||
Re-creates OCR content and thumbnail for a document, and archive file if
|
Re-creates OCR content and thumbnail for a document, and archive file if
|
||||||
it exists.
|
it exists.
|
||||||
|
|
||||||
|
Remote OCR is used only when the engine is configured to handle everything
|
||||||
|
or if explicitly asked for via ``remote_ocr``.
|
||||||
"""
|
"""
|
||||||
document = Document.objects.get(id=document_id)
|
document = Document.objects.get(id=document_id)
|
||||||
|
|
||||||
@@ -351,6 +358,7 @@ def update_document_content_maybe_archive_file(document_id) -> None:
|
|||||||
mime_type,
|
mime_type,
|
||||||
document.original_filename or "",
|
document.original_filename or "",
|
||||||
document.source_path,
|
document.source_path,
|
||||||
|
allow_remote=remote_ocr or RemoteOCRConfig().remote_ocr_by_default,
|
||||||
)
|
)
|
||||||
|
|
||||||
if not parser_class:
|
if not parser_class:
|
||||||
|
|||||||
@@ -16,7 +16,6 @@ from documents.search._backend import TantivyBackend
|
|||||||
from documents.search._backend import WriteBatch
|
from documents.search._backend import WriteBatch
|
||||||
from documents.search._backend import get_backend
|
from documents.search._backend import get_backend
|
||||||
from documents.search._backend import reset_backend
|
from documents.search._backend import reset_backend
|
||||||
from documents.signals.handlers import add_to_index
|
|
||||||
from documents.tests.factories import CorrespondentFactory
|
from documents.tests.factories import CorrespondentFactory
|
||||||
from documents.tests.factories import DocumentFactory
|
from documents.tests.factories import DocumentFactory
|
||||||
from documents.tests.factories import DocumentTypeFactory
|
from documents.tests.factories import DocumentTypeFactory
|
||||||
@@ -1031,81 +1030,6 @@ class TestHighlightHits:
|
|||||||
assert len(hits) == 0
|
assert len(hits) == 0
|
||||||
|
|
||||||
|
|
||||||
class TestVersionIndexing:
|
|
||||||
"""
|
|
||||||
GIVEN:
|
|
||||||
- A root document whose new version has just been consumed, e.g. by
|
|
||||||
the password removal workflow action
|
|
||||||
WHEN:
|
|
||||||
- The consumption finished signal is handled
|
|
||||||
THEN:
|
|
||||||
- The root document is indexed with the new version's content, since
|
|
||||||
versions are not searchable on their own
|
|
||||||
"""
|
|
||||||
|
|
||||||
def test_consumed_version_updates_root_entry(
|
|
||||||
self,
|
|
||||||
backend: TantivyBackend,
|
|
||||||
mocker: MockerFixture,
|
|
||||||
) -> None:
|
|
||||||
root = Document.objects.create(
|
|
||||||
title="Statement",
|
|
||||||
content="",
|
|
||||||
checksum="VER1",
|
|
||||||
pk=90,
|
|
||||||
)
|
|
||||||
backend.add_or_update(root)
|
|
||||||
version = Document.objects.create(
|
|
||||||
title="Statement",
|
|
||||||
content="unprotected statement text",
|
|
||||||
checksum="VER2",
|
|
||||||
pk=91,
|
|
||||||
root_document=root,
|
|
||||||
version_index=1,
|
|
||||||
)
|
|
||||||
mocker.patch("documents.search.get_backend", return_value=backend)
|
|
||||||
|
|
||||||
add_to_index(sender=None, document=version)
|
|
||||||
|
|
||||||
assert backend.search_ids("unprotected", user=None) == [root.pk]
|
|
||||||
|
|
||||||
|
|
||||||
class TestEffectiveContentIndexing:
|
|
||||||
"""
|
|
||||||
GIVEN:
|
|
||||||
- A root document with a newer version
|
|
||||||
WHEN:
|
|
||||||
- The root document is indexed
|
|
||||||
THEN:
|
|
||||||
- The newest version's content is indexed, never the root's own
|
|
||||||
outdated text
|
|
||||||
"""
|
|
||||||
|
|
||||||
def test_root_is_indexed_with_latest_version_content(
|
|
||||||
self,
|
|
||||||
backend: TantivyBackend,
|
|
||||||
) -> None:
|
|
||||||
root = Document.objects.create(
|
|
||||||
title="Statement",
|
|
||||||
content="stale original text",
|
|
||||||
checksum="EFF1",
|
|
||||||
pk=95,
|
|
||||||
)
|
|
||||||
Document.objects.create(
|
|
||||||
title="Statement",
|
|
||||||
content="latest version text",
|
|
||||||
checksum="EFF2",
|
|
||||||
pk=96,
|
|
||||||
root_document=root,
|
|
||||||
version_index=1,
|
|
||||||
)
|
|
||||||
|
|
||||||
backend.add_or_update(root)
|
|
||||||
|
|
||||||
assert backend.search_ids("latest", user=None) == [root.pk]
|
|
||||||
assert backend.search_ids("stale", user=None) == []
|
|
||||||
|
|
||||||
|
|
||||||
class TestIndexDirectoryGarbageCollection:
|
class TestIndexDirectoryGarbageCollection:
|
||||||
"""Regression tests for Tantivy segment files leaking on disk when
|
"""Regression tests for Tantivy segment files leaking on disk when
|
||||||
multiple long-lived worker processes (Granian/Celery) take turns writing
|
multiple long-lived worker processes (Granian/Celery) take turns writing
|
||||||
|
|||||||
@@ -72,6 +72,10 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
|
|||||||
"barcode_enable_tag": None,
|
"barcode_enable_tag": None,
|
||||||
"barcode_tag_mapping": None,
|
"barcode_tag_mapping": None,
|
||||||
"barcode_tag_split": None,
|
"barcode_tag_split": None,
|
||||||
|
"remote_ocr_engine": None,
|
||||||
|
"remote_ocr_api_key": None,
|
||||||
|
"remote_ocr_endpoint": None,
|
||||||
|
"remote_ocr_mode": None,
|
||||||
"ai_enabled": False,
|
"ai_enabled": False,
|
||||||
"llm_embedding_backend": None,
|
"llm_embedding_backend": None,
|
||||||
"llm_embedding_model": None,
|
"llm_embedding_model": None,
|
||||||
@@ -870,6 +874,49 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
|
|||||||
config.refresh_from_db()
|
config.refresh_from_db()
|
||||||
self.assertEqual(config.llm_api_key, None)
|
self.assertEqual(config.llm_api_key, None)
|
||||||
|
|
||||||
|
def test_update_remote_ocr_api_key(self) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- Existing config with remote_ocr_api_key specified
|
||||||
|
WHEN:
|
||||||
|
- API to update remote_ocr_api_key is called with all *s
|
||||||
|
- API to update remote_ocr_api_key is called with empty string
|
||||||
|
THEN:
|
||||||
|
- remote_ocr_api_key is unchanged
|
||||||
|
- remote_ocr_api_key is set to None
|
||||||
|
"""
|
||||||
|
config = ApplicationConfiguration.objects.first()
|
||||||
|
assert config is not None
|
||||||
|
config.remote_ocr_api_key = "1234567890"
|
||||||
|
config.save()
|
||||||
|
|
||||||
|
# Test with all *
|
||||||
|
response = self.client.patch(
|
||||||
|
f"{self.ENDPOINT}1/",
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"remote_ocr_api_key": "*" * 32,
|
||||||
|
},
|
||||||
|
),
|
||||||
|
content_type="application/json",
|
||||||
|
)
|
||||||
|
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||||
|
config.refresh_from_db()
|
||||||
|
self.assertEqual(config.remote_ocr_api_key, "1234567890")
|
||||||
|
# Test with empty string
|
||||||
|
response = self.client.patch(
|
||||||
|
f"{self.ENDPOINT}1/",
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"remote_ocr_api_key": "",
|
||||||
|
},
|
||||||
|
),
|
||||||
|
content_type="application/json",
|
||||||
|
)
|
||||||
|
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||||
|
config.refresh_from_db()
|
||||||
|
self.assertEqual(config.remote_ocr_api_key, None)
|
||||||
|
|
||||||
def test_enable_ai_index_triggers_update(self) -> None:
|
def test_enable_ai_index_triggers_update(self) -> None:
|
||||||
"""
|
"""
|
||||||
GIVEN:
|
GIVEN:
|
||||||
|
|||||||
@@ -1559,6 +1559,72 @@ class PostConsumeTestCase(DirectoriesMixin, GetConsumerMixin, TestCase):
|
|||||||
consumer.run_post_consume_script(doc)
|
consumer.run_post_consume_script(doc)
|
||||||
|
|
||||||
|
|
||||||
|
class TestConsumerRemoteOCR(
|
||||||
|
DirectoriesMixin,
|
||||||
|
FileSystemAssertsMixin,
|
||||||
|
GetConsumerMixin,
|
||||||
|
TestCase,
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
The consumer resolves the remote OCR mode and the per-document request from
|
||||||
|
workflows into the allow_remote flag it hands to the parser registry.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def setUp(self) -> None:
|
||||||
|
super().setUp()
|
||||||
|
|
||||||
|
patcher = mock.patch("documents.consumer.get_parser_registry")
|
||||||
|
self.mock_registry = patcher.start()
|
||||||
|
self.mock_registry.return_value.get_parser_for_file.return_value = DummyParser
|
||||||
|
self.addCleanup(patcher.stop)
|
||||||
|
|
||||||
|
def _consume(self, *, overrides: DocumentMetadataOverrides | None = None) -> bool:
|
||||||
|
src = (
|
||||||
|
Path(__file__).parent
|
||||||
|
/ "samples"
|
||||||
|
/ "documents"
|
||||||
|
/ "originals"
|
||||||
|
/ "0000001.pdf"
|
||||||
|
)
|
||||||
|
dst = self.dirs.scratch_dir / "sample.pdf"
|
||||||
|
shutil.copy(src, dst)
|
||||||
|
|
||||||
|
with self.get_consumer(dst, overrides=overrides) as consumer:
|
||||||
|
consumer.run()
|
||||||
|
|
||||||
|
_, kwargs = self.mock_registry.return_value.get_parser_for_file.call_args
|
||||||
|
return kwargs["allow_remote"]
|
||||||
|
|
||||||
|
@override_settings(REMOTE_OCR_MODE="always")
|
||||||
|
def test_always_mode_allows_remote(self) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN: Remote OCR mode is 'always'.
|
||||||
|
WHEN: A document is consumed without any workflow asking for it.
|
||||||
|
THEN: The registry is allowed to pick the remote parser.
|
||||||
|
"""
|
||||||
|
self.assertTrue(self._consume())
|
||||||
|
|
||||||
|
@override_settings(REMOTE_OCR_MODE="workflow_only")
|
||||||
|
def test_workflow_only_mode_denies_remote_by_default(self) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN: Remote OCR mode is 'workflow_only'.
|
||||||
|
WHEN: A document is consumed and nothing asked for remote OCR.
|
||||||
|
THEN: The remote parser is excluded.
|
||||||
|
"""
|
||||||
|
self.assertFalse(self._consume())
|
||||||
|
|
||||||
|
@override_settings(REMOTE_OCR_MODE="workflow_only")
|
||||||
|
def test_workflow_only_mode_allows_remote_when_requested(self) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN: Remote OCR mode is 'workflow_only'.
|
||||||
|
WHEN: A workflow set remote_ocr on the metadata overrides.
|
||||||
|
THEN: The registry is allowed to pick the remote parser.
|
||||||
|
"""
|
||||||
|
self.assertTrue(
|
||||||
|
self._consume(overrides=DocumentMetadataOverrides(remote_ocr=True)),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class TestMetadataOverrides(TestCase):
|
class TestMetadataOverrides(TestCase):
|
||||||
def test_update_skip_asn_if_exists(self) -> None:
|
def test_update_skip_asn_if_exists(self) -> None:
|
||||||
base = DocumentMetadataOverrides()
|
base = DocumentMetadataOverrides()
|
||||||
@@ -1566,6 +1632,20 @@ class TestMetadataOverrides(TestCase):
|
|||||||
base.update(incoming)
|
base.update(incoming)
|
||||||
self.assertTrue(base.skip_asn_if_exists)
|
self.assertTrue(base.skip_asn_if_exists)
|
||||||
|
|
||||||
|
def test_update_remote_ocr(self) -> None:
|
||||||
|
base = DocumentMetadataOverrides()
|
||||||
|
base.update(DocumentMetadataOverrides(remote_ocr=True))
|
||||||
|
self.assertTrue(base.remote_ocr)
|
||||||
|
|
||||||
|
def test_update_remote_ocr_is_not_unset(self) -> None:
|
||||||
|
"""
|
||||||
|
A later workflow that says nothing must not undo an earlier one that
|
||||||
|
asked for remote OCR.
|
||||||
|
"""
|
||||||
|
base = DocumentMetadataOverrides(remote_ocr=True)
|
||||||
|
base.update(DocumentMetadataOverrides())
|
||||||
|
self.assertTrue(base.remote_ocr)
|
||||||
|
|
||||||
def test_update_actor_and_version_label(self) -> None:
|
def test_update_actor_and_version_label(self) -> None:
|
||||||
base = DocumentMetadataOverrides(
|
base = DocumentMetadataOverrides(
|
||||||
actor_id=1,
|
actor_id=1,
|
||||||
|
|||||||
@@ -287,6 +287,45 @@ class TestUpdateContent(DirectoriesMixin, TestCase):
|
|||||||
self.assertNotEqual(Document.objects.get(pk=doc.pk).content, "test")
|
self.assertNotEqual(Document.objects.get(pk=doc.pk).content, "test")
|
||||||
|
|
||||||
|
|
||||||
|
class TestUpdateContentRemoteOCR(DirectoriesMixin, TestCase):
|
||||||
|
"""
|
||||||
|
Consumption workflows do not run on reprocess, so the remote parser is
|
||||||
|
used only in 'always' mode or when the caller explicitly asks for it.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def setUp(self) -> None:
|
||||||
|
super().setUp()
|
||||||
|
|
||||||
|
patcher = mock.patch("documents.tasks.get_parser_registry")
|
||||||
|
self.mock_registry = patcher.start()
|
||||||
|
self.mock_registry.return_value.get_parser_for_file.return_value = None
|
||||||
|
self.addCleanup(patcher.stop)
|
||||||
|
|
||||||
|
self.doc = Document.objects.create(
|
||||||
|
title="test",
|
||||||
|
content="my document",
|
||||||
|
checksum="wow",
|
||||||
|
mime_type="application/pdf",
|
||||||
|
)
|
||||||
|
|
||||||
|
def _allow_remote(self, **kwargs) -> bool:
|
||||||
|
tasks.update_document_content_maybe_archive_file(self.doc.pk, **kwargs)
|
||||||
|
_, call_kwargs = self.mock_registry.return_value.get_parser_for_file.call_args
|
||||||
|
return call_kwargs["allow_remote"]
|
||||||
|
|
||||||
|
@override_settings(REMOTE_OCR_MODE="always")
|
||||||
|
def test_always_mode_allows_remote(self) -> None:
|
||||||
|
self.assertTrue(self._allow_remote())
|
||||||
|
|
||||||
|
@override_settings(REMOTE_OCR_MODE="workflow_only")
|
||||||
|
def test_workflow_only_mode_denies_remote_by_default(self) -> None:
|
||||||
|
self.assertFalse(self._allow_remote())
|
||||||
|
|
||||||
|
@override_settings(REMOTE_OCR_MODE="workflow_only")
|
||||||
|
def test_workflow_only_mode_allows_remote_when_requested(self) -> None:
|
||||||
|
self.assertTrue(self._allow_remote(remote_ocr=True))
|
||||||
|
|
||||||
|
|
||||||
class TestAIIndex(DirectoriesMixin, TestCase):
|
class TestAIIndex(DirectoriesMixin, TestCase):
|
||||||
@override_settings(
|
@override_settings(
|
||||||
AI_ENABLED=True,
|
AI_ENABLED=True,
|
||||||
|
|||||||
@@ -6,10 +6,7 @@ from typing import TYPE_CHECKING
|
|||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from django.db.models import F
|
from django.db.models import F
|
||||||
from django.db.models import OuterRef
|
|
||||||
from django.db.models import QuerySet
|
from django.db.models import QuerySet
|
||||||
from django.db.models import Subquery
|
|
||||||
from django.db.models.functions import Coalesce
|
|
||||||
|
|
||||||
from documents.models import Document
|
from documents.models import Document
|
||||||
|
|
||||||
@@ -25,24 +22,6 @@ def versions_newest_first(documents: QuerySet[Document]) -> QuerySet[Document]:
|
|||||||
return documents.order_by(F("version_index").desc(nulls_last=True), "-id")
|
return documents.order_by(F("version_index").desc(nulls_last=True), "-id")
|
||||||
|
|
||||||
|
|
||||||
def annotate_effective_content(documents: QuerySet[Document]) -> QuerySet[Document]:
|
|
||||||
"""
|
|
||||||
Annotates documents with the content of their newest version, falling back
|
|
||||||
to their own, so get_effective_content() can answer from the row rather
|
|
||||||
than querying for the versions of each document
|
|
||||||
"""
|
|
||||||
return documents.annotate(
|
|
||||||
effective_content=Coalesce(
|
|
||||||
Subquery(
|
|
||||||
versions_newest_first(
|
|
||||||
Document.objects.filter(root_document=OuterRef("pk")),
|
|
||||||
).values("content")[:1],
|
|
||||||
),
|
|
||||||
F("content"),
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def sort_versions_newest_first(documents: list[Document]) -> list[Document]:
|
def sort_versions_newest_first(documents: list[Document]) -> list[Document]:
|
||||||
"""
|
"""
|
||||||
Same sorting as versions_newest_first()
|
Same sorting as versions_newest_first()
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ msgid ""
|
|||||||
msgstr ""
|
msgstr ""
|
||||||
"Project-Id-Version: paperless-ngx\n"
|
"Project-Id-Version: paperless-ngx\n"
|
||||||
"Report-Msgid-Bugs-To: \n"
|
"Report-Msgid-Bugs-To: \n"
|
||||||
"POT-Creation-Date: 2026-08-19 18:18+0000\n"
|
"POT-Creation-Date: 2026-08-19 14:23+0000\n"
|
||||||
"PO-Revision-Date: 2022-02-17 04:17\n"
|
"PO-Revision-Date: 2022-02-17 04:17\n"
|
||||||
"Last-Translator: \n"
|
"Last-Translator: \n"
|
||||||
"Language-Team: English\n"
|
"Language-Team: English\n"
|
||||||
@@ -2170,151 +2170,151 @@ msgstr ""
|
|||||||
msgid "paperless application settings"
|
msgid "paperless application settings"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: paperless/settings/__init__.py:562
|
#: paperless/settings/__init__.py:556
|
||||||
msgid "English (US)"
|
msgid "English (US)"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: paperless/settings/__init__.py:563
|
#: paperless/settings/__init__.py:557
|
||||||
msgid "Arabic"
|
msgid "Arabic"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: paperless/settings/__init__.py:564
|
#: paperless/settings/__init__.py:558
|
||||||
msgid "Afrikaans"
|
msgid "Afrikaans"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: paperless/settings/__init__.py:565
|
#: paperless/settings/__init__.py:559
|
||||||
msgid "Belarusian"
|
msgid "Belarusian"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: paperless/settings/__init__.py:566
|
#: paperless/settings/__init__.py:560
|
||||||
msgid "Bulgarian"
|
msgid "Bulgarian"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: paperless/settings/__init__.py:567
|
#: paperless/settings/__init__.py:561
|
||||||
msgid "Catalan"
|
msgid "Catalan"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: paperless/settings/__init__.py:568
|
#: paperless/settings/__init__.py:562
|
||||||
msgid "Czech"
|
msgid "Czech"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: paperless/settings/__init__.py:569
|
#: paperless/settings/__init__.py:563
|
||||||
msgid "Danish"
|
msgid "Danish"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: paperless/settings/__init__.py:570
|
#: paperless/settings/__init__.py:564
|
||||||
msgid "German"
|
msgid "German"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: paperless/settings/__init__.py:571
|
#: paperless/settings/__init__.py:565
|
||||||
msgid "Greek"
|
msgid "Greek"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: paperless/settings/__init__.py:572
|
#: paperless/settings/__init__.py:566
|
||||||
msgid "English (GB)"
|
msgid "English (GB)"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: paperless/settings/__init__.py:573
|
#: paperless/settings/__init__.py:567
|
||||||
msgid "Spanish"
|
msgid "Spanish"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: paperless/settings/__init__.py:574
|
#: paperless/settings/__init__.py:568
|
||||||
msgid "Persian"
|
msgid "Persian"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: paperless/settings/__init__.py:575
|
#: paperless/settings/__init__.py:569
|
||||||
msgid "Finnish"
|
msgid "Finnish"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: paperless/settings/__init__.py:576
|
#: paperless/settings/__init__.py:570
|
||||||
msgid "French"
|
msgid "French"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: paperless/settings/__init__.py:577
|
#: paperless/settings/__init__.py:571
|
||||||
msgid "Hungarian"
|
msgid "Hungarian"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: paperless/settings/__init__.py:578
|
#: paperless/settings/__init__.py:572
|
||||||
msgid "Indonesian"
|
msgid "Indonesian"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: paperless/settings/__init__.py:579
|
#: paperless/settings/__init__.py:573
|
||||||
msgid "Italian"
|
msgid "Italian"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: paperless/settings/__init__.py:580
|
#: paperless/settings/__init__.py:574
|
||||||
msgid "Japanese"
|
msgid "Japanese"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: paperless/settings/__init__.py:581
|
#: paperless/settings/__init__.py:575
|
||||||
msgid "Korean"
|
msgid "Korean"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: paperless/settings/__init__.py:582
|
#: paperless/settings/__init__.py:576
|
||||||
msgid "Luxembourgish"
|
msgid "Luxembourgish"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: paperless/settings/__init__.py:583
|
#: paperless/settings/__init__.py:577
|
||||||
msgid "Norwegian"
|
msgid "Norwegian"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: paperless/settings/__init__.py:584
|
#: paperless/settings/__init__.py:578
|
||||||
msgid "Dutch"
|
msgid "Dutch"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: paperless/settings/__init__.py:585
|
#: paperless/settings/__init__.py:579
|
||||||
msgid "Polish"
|
msgid "Polish"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: paperless/settings/__init__.py:586
|
#: paperless/settings/__init__.py:580
|
||||||
msgid "Portuguese (Brazil)"
|
msgid "Portuguese (Brazil)"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: paperless/settings/__init__.py:587
|
#: paperless/settings/__init__.py:581
|
||||||
msgid "Portuguese"
|
msgid "Portuguese"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: paperless/settings/__init__.py:588
|
#: paperless/settings/__init__.py:582
|
||||||
msgid "Romanian"
|
msgid "Romanian"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: paperless/settings/__init__.py:589
|
#: paperless/settings/__init__.py:583
|
||||||
msgid "Russian"
|
msgid "Russian"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: paperless/settings/__init__.py:590
|
#: paperless/settings/__init__.py:584
|
||||||
msgid "Slovak"
|
msgid "Slovak"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: paperless/settings/__init__.py:591
|
#: paperless/settings/__init__.py:585
|
||||||
msgid "Slovenian"
|
msgid "Slovenian"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: paperless/settings/__init__.py:592
|
#: paperless/settings/__init__.py:586
|
||||||
msgid "Serbian"
|
msgid "Serbian"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: paperless/settings/__init__.py:593
|
#: paperless/settings/__init__.py:587
|
||||||
msgid "Swedish"
|
msgid "Swedish"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: paperless/settings/__init__.py:594
|
#: paperless/settings/__init__.py:588
|
||||||
msgid "Turkish"
|
msgid "Turkish"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: paperless/settings/__init__.py:595
|
#: paperless/settings/__init__.py:589
|
||||||
msgid "Ukrainian"
|
msgid "Ukrainian"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: paperless/settings/__init__.py:596
|
#: paperless/settings/__init__.py:590
|
||||||
msgid "Vietnamese"
|
msgid "Vietnamese"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: paperless/settings/__init__.py:597
|
#: paperless/settings/__init__.py:591
|
||||||
msgid "Chinese Simplified"
|
msgid "Chinese Simplified"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: paperless/settings/__init__.py:598
|
#: paperless/settings/__init__.py:592
|
||||||
msgid "Chinese Traditional"
|
msgid "Chinese Traditional"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
|
|||||||
@@ -337,20 +337,6 @@ def check_deprecated_v2_ocr_env_vars(
|
|||||||
return warnings
|
return warnings
|
||||||
|
|
||||||
|
|
||||||
@register()
|
|
||||||
def check_remote_parser_configured(app_configs: Any, **kwargs: Any) -> list[Error]:
|
|
||||||
if settings.REMOTE_OCR_ENGINE == "azureai" and not (
|
|
||||||
settings.REMOTE_OCR_ENDPOINT and settings.REMOTE_OCR_API_KEY
|
|
||||||
):
|
|
||||||
return [
|
|
||||||
Error(
|
|
||||||
"Azure AI remote parser requires endpoint and API key to be configured.",
|
|
||||||
),
|
|
||||||
]
|
|
||||||
|
|
||||||
return []
|
|
||||||
|
|
||||||
|
|
||||||
def get_tesseract_langs():
|
def get_tesseract_langs():
|
||||||
proc = subprocess.run(
|
proc = subprocess.run(
|
||||||
[shutil.which("tesseract"), "--list-langs"],
|
[shutil.which("tesseract"), "--list-langs"],
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ from paperless.models import CleanChoices
|
|||||||
from paperless.models import ColorConvertChoices
|
from paperless.models import ColorConvertChoices
|
||||||
from paperless.models import ModeChoices
|
from paperless.models import ModeChoices
|
||||||
from paperless.models import OutputTypeChoices
|
from paperless.models import OutputTypeChoices
|
||||||
|
from paperless.models import RemoteOCRMode
|
||||||
|
|
||||||
|
|
||||||
@dataclasses.dataclass
|
@dataclasses.dataclass
|
||||||
@@ -185,6 +186,45 @@ class GeneralConfig(BaseConfig):
|
|||||||
self.app_logo = app_config.app_logo.url if app_config.app_logo else None
|
self.app_logo = app_config.app_logo.url if app_config.app_logo else None
|
||||||
|
|
||||||
|
|
||||||
|
@dataclasses.dataclass
|
||||||
|
class RemoteOCRConfig(BaseConfig):
|
||||||
|
"""
|
||||||
|
Settings for the remote (cloud) OCR parser
|
||||||
|
"""
|
||||||
|
|
||||||
|
remote_ocr_engine: str | None = dataclasses.field(init=False)
|
||||||
|
remote_ocr_api_key: str | None = dataclasses.field(init=False)
|
||||||
|
remote_ocr_endpoint: str | None = dataclasses.field(init=False)
|
||||||
|
remote_ocr_mode: RemoteOCRMode = dataclasses.field(init=False)
|
||||||
|
|
||||||
|
def __post_init__(self) -> None:
|
||||||
|
app_config = self._get_config_instance()
|
||||||
|
|
||||||
|
self.remote_ocr_engine = (
|
||||||
|
app_config.remote_ocr_engine or settings.REMOTE_OCR_ENGINE
|
||||||
|
)
|
||||||
|
self.remote_ocr_api_key = (
|
||||||
|
app_config.remote_ocr_api_key or settings.REMOTE_OCR_API_KEY
|
||||||
|
)
|
||||||
|
self.remote_ocr_endpoint = (
|
||||||
|
app_config.remote_ocr_endpoint or settings.REMOTE_OCR_ENDPOINT
|
||||||
|
)
|
||||||
|
self.remote_ocr_mode = app_config.remote_ocr_mode or RemoteOCRMode(
|
||||||
|
settings.REMOTE_OCR_MODE,
|
||||||
|
)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def remote_ocr_by_default(self) -> bool:
|
||||||
|
"""
|
||||||
|
Whether every supported document goes to the remote engine.
|
||||||
|
|
||||||
|
When False the remote engine is used only for documents that
|
||||||
|
explicitly asked for it, i.e. a workflow matched during consumption or
|
||||||
|
the user ticked the box when reprocessing.
|
||||||
|
"""
|
||||||
|
return self.remote_ocr_mode == RemoteOCRMode.ALWAYS
|
||||||
|
|
||||||
|
|
||||||
@dataclasses.dataclass
|
@dataclasses.dataclass
|
||||||
class AIConfig(BaseConfig):
|
class AIConfig(BaseConfig):
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -0,0 +1,44 @@
|
|||||||
|
# Generated by Django 5.2.16 on 2026-08-10 14:37
|
||||||
|
|
||||||
|
from django.db import migrations
|
||||||
|
from django.db import models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
dependencies = [
|
||||||
|
("paperless", "0013_applicationconfiguration_llm_request_timeout"),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AddField(
|
||||||
|
model_name="applicationconfiguration",
|
||||||
|
name="remote_ocr_api_key",
|
||||||
|
field=models.CharField(
|
||||||
|
blank=True,
|
||||||
|
max_length=1024,
|
||||||
|
null=True,
|
||||||
|
verbose_name="Sets the remote OCR API key",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
migrations.AddField(
|
||||||
|
model_name="applicationconfiguration",
|
||||||
|
name="remote_ocr_endpoint",
|
||||||
|
field=models.CharField(
|
||||||
|
blank=True,
|
||||||
|
max_length=256,
|
||||||
|
null=True,
|
||||||
|
verbose_name="Sets the remote OCR endpoint",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
migrations.AddField(
|
||||||
|
model_name="applicationconfiguration",
|
||||||
|
name="remote_ocr_engine",
|
||||||
|
field=models.CharField(
|
||||||
|
blank=True,
|
||||||
|
choices=[("azureai", "Azure AI Document Intelligence")],
|
||||||
|
max_length=32,
|
||||||
|
null=True,
|
||||||
|
verbose_name="Sets the remote OCR engine",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
]
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
# Generated by Django 5.2.16 on 2026-08-10 15:43
|
||||||
|
|
||||||
|
from django.db import migrations
|
||||||
|
from django.db import models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
dependencies = [
|
||||||
|
("paperless", "0014_applicationconfiguration_remote_ocr_api_key_and_more"),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AddField(
|
||||||
|
model_name="applicationconfiguration",
|
||||||
|
name="remote_ocr_mode",
|
||||||
|
field=models.CharField(
|
||||||
|
blank=True,
|
||||||
|
choices=[
|
||||||
|
("always", "All supported documents"),
|
||||||
|
("workflow_only", "Only when a workflow enables it"),
|
||||||
|
],
|
||||||
|
max_length=32,
|
||||||
|
null=True,
|
||||||
|
verbose_name="Sets which documents are sent to the remote OCR engine",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
]
|
||||||
@@ -74,6 +74,23 @@ class ColorConvertChoices(models.TextChoices):
|
|||||||
CMYK = ("CMYK", _("CMYK"))
|
CMYK = ("CMYK", _("CMYK"))
|
||||||
|
|
||||||
|
|
||||||
|
class RemoteOCREngine(models.TextChoices):
|
||||||
|
"""
|
||||||
|
Matches to PAPERLESS_REMOTE_OCR_ENGINE
|
||||||
|
"""
|
||||||
|
|
||||||
|
AZURE_AI = ("azureai", _("Azure AI Document Intelligence"))
|
||||||
|
|
||||||
|
|
||||||
|
class RemoteOCRMode(models.TextChoices):
|
||||||
|
"""
|
||||||
|
Matches to PAPERLESS_REMOTE_OCR_MODE
|
||||||
|
"""
|
||||||
|
|
||||||
|
ALWAYS = ("always", _("All supported documents"))
|
||||||
|
WORKFLOW_ONLY = ("workflow_only", _("Only when a workflow enables it"))
|
||||||
|
|
||||||
|
|
||||||
class LLMEmbeddingBackend(models.TextChoices):
|
class LLMEmbeddingBackend(models.TextChoices):
|
||||||
OPENAI_LIKE = ("openai-like", _("OpenAI-compatible"))
|
OPENAI_LIKE = ("openai-like", _("OpenAI-compatible"))
|
||||||
HUGGINGFACE = ("huggingface", _("Huggingface"))
|
HUGGINGFACE = ("huggingface", _("Huggingface"))
|
||||||
@@ -286,6 +303,44 @@ class ApplicationConfiguration(AbstractSingletonModel):
|
|||||||
null=True,
|
null=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
"""
|
||||||
|
Settings for the remote OCR parser
|
||||||
|
"""
|
||||||
|
|
||||||
|
# PAPERLESS_REMOTE_OCR_ENGINE
|
||||||
|
remote_ocr_engine = models.CharField(
|
||||||
|
verbose_name=_("Sets the remote OCR engine"),
|
||||||
|
blank=True,
|
||||||
|
null=True,
|
||||||
|
max_length=32,
|
||||||
|
choices=RemoteOCREngine.choices,
|
||||||
|
)
|
||||||
|
|
||||||
|
# PAPERLESS_REMOTE_OCR_API_KEY
|
||||||
|
remote_ocr_api_key = models.CharField(
|
||||||
|
verbose_name=_("Sets the remote OCR API key"),
|
||||||
|
blank=True,
|
||||||
|
null=True,
|
||||||
|
max_length=1024,
|
||||||
|
)
|
||||||
|
|
||||||
|
# PAPERLESS_REMOTE_OCR_ENDPOINT
|
||||||
|
remote_ocr_endpoint = models.CharField(
|
||||||
|
verbose_name=_("Sets the remote OCR endpoint"),
|
||||||
|
blank=True,
|
||||||
|
null=True,
|
||||||
|
max_length=256,
|
||||||
|
)
|
||||||
|
|
||||||
|
# PAPERLESS_REMOTE_OCR_MODE
|
||||||
|
remote_ocr_mode = models.CharField(
|
||||||
|
verbose_name=_("Sets which documents are sent to the remote OCR engine"),
|
||||||
|
blank=True,
|
||||||
|
null=True,
|
||||||
|
max_length=32,
|
||||||
|
choices=RemoteOCRMode.choices,
|
||||||
|
)
|
||||||
|
|
||||||
"""
|
"""
|
||||||
AI related settings
|
AI related settings
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -134,6 +134,11 @@ class ParserProtocol(Protocol):
|
|||||||
Author or organisation name.
|
Author or organisation name.
|
||||||
url : str
|
url : str
|
||||||
URL for documentation, source code, or issue tracker.
|
URL for documentation, source code, or issue tracker.
|
||||||
|
|
||||||
|
Parsers that send document content to a remote service should additionally
|
||||||
|
set ``uses_remote_service = True`` so the registry can exclude them when
|
||||||
|
remote processing has not been requested for a document. The attribute is
|
||||||
|
optional so a parser that omits it is treated as fully local.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
@@ -145,6 +150,10 @@ class ParserProtocol(Protocol):
|
|||||||
author: str
|
author: str
|
||||||
url: str
|
url: str
|
||||||
|
|
||||||
|
# NOTE: uses_remote_service is not declared here, the registry reads it
|
||||||
|
# with getattr(cls, ..., False) for backwards-compatibility with existing
|
||||||
|
# parsers
|
||||||
|
|
||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
# Class methods
|
# Class methods
|
||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
|
|||||||
@@ -334,6 +334,8 @@ class ParserRegistry:
|
|||||||
mime_type: str,
|
mime_type: str,
|
||||||
filename: str,
|
filename: str,
|
||||||
path: Path | None = None,
|
path: Path | None = None,
|
||||||
|
*,
|
||||||
|
allow_remote: bool = True,
|
||||||
) -> type[ParserProtocol] | None:
|
) -> type[ParserProtocol] | None:
|
||||||
"""Return the best parser class for the given file, or None.
|
"""Return the best parser class for the given file, or None.
|
||||||
|
|
||||||
@@ -359,6 +361,11 @@ class ParserRegistry:
|
|||||||
path:
|
path:
|
||||||
Optional filesystem path to the file. Forwarded to each
|
Optional filesystem path to the file. Forwarded to each
|
||||||
parser's score method.
|
parser's score method.
|
||||||
|
allow_remote:
|
||||||
|
When False, parsers that declare ``uses_remote_service = True``
|
||||||
|
are excluded from consideration, so a document is never sent to
|
||||||
|
a remote service. Parsers that do not declare the attribute
|
||||||
|
are treated as local and are always considered.
|
||||||
|
|
||||||
Returns
|
Returns
|
||||||
-------
|
-------
|
||||||
@@ -374,6 +381,13 @@ class ParserRegistry:
|
|||||||
if mime_type not in parser_class.supported_mime_types():
|
if mime_type not in parser_class.supported_mime_types():
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
if not allow_remote and getattr(
|
||||||
|
parser_class,
|
||||||
|
"uses_remote_service",
|
||||||
|
False,
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
|
||||||
score = parser_class.score(mime_type, filename, path)
|
score = parser_class.score(mime_type, filename, path)
|
||||||
if score is None:
|
if score is None:
|
||||||
continue
|
continue
|
||||||
|
|||||||
@@ -61,6 +61,18 @@ class RemoteEngineConfig:
|
|||||||
self.api_key = api_key
|
self.api_key = api_key
|
||||||
self.endpoint = endpoint
|
self.endpoint = endpoint
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_app_config(cls) -> Self:
|
||||||
|
"""Build the config from the app config, falling back to the env."""
|
||||||
|
from paperless.config import RemoteOCRConfig
|
||||||
|
|
||||||
|
app_config = RemoteOCRConfig()
|
||||||
|
return cls(
|
||||||
|
engine=app_config.remote_ocr_engine,
|
||||||
|
api_key=app_config.remote_ocr_api_key,
|
||||||
|
endpoint=app_config.remote_ocr_endpoint,
|
||||||
|
)
|
||||||
|
|
||||||
def engine_is_valid(self) -> bool:
|
def engine_is_valid(self) -> bool:
|
||||||
"""Return True when the engine is known and fully configured."""
|
"""Return True when the engine is known and fully configured."""
|
||||||
return (
|
return (
|
||||||
@@ -90,6 +102,9 @@ class RemoteDocumentParser:
|
|||||||
Maintainer name.
|
Maintainer name.
|
||||||
url : str
|
url : str
|
||||||
Issue tracker / source URL.
|
Issue tracker / source URL.
|
||||||
|
uses_remote_service : bool
|
||||||
|
Content is sent to a remote service, True so that the registry
|
||||||
|
can skip this parser if remote processing was not requested.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
name: str = "Paperless-ngx Remote OCR Parser"
|
name: str = "Paperless-ngx Remote OCR Parser"
|
||||||
@@ -97,6 +112,8 @@ class RemoteDocumentParser:
|
|||||||
author: str = "Paperless-ngx Contributors"
|
author: str = "Paperless-ngx Contributors"
|
||||||
url: str = "https://github.com/paperless-ngx/paperless-ngx"
|
url: str = "https://github.com/paperless-ngx/paperless-ngx"
|
||||||
|
|
||||||
|
uses_remote_service: bool = True
|
||||||
|
|
||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
# Class methods
|
# Class methods
|
||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
@@ -145,11 +162,7 @@ class RemoteDocumentParser:
|
|||||||
20 when the remote engine is configured and the MIME type is
|
20 when the remote engine is configured and the MIME type is
|
||||||
supported, otherwise None.
|
supported, otherwise None.
|
||||||
"""
|
"""
|
||||||
config = RemoteEngineConfig(
|
config = RemoteEngineConfig.from_app_config()
|
||||||
engine=settings.REMOTE_OCR_ENGINE,
|
|
||||||
api_key=settings.REMOTE_OCR_API_KEY,
|
|
||||||
endpoint=settings.REMOTE_OCR_ENDPOINT,
|
|
||||||
)
|
|
||||||
if not config.engine_is_valid():
|
if not config.engine_is_valid():
|
||||||
return None
|
return None
|
||||||
if mime_type not in _SUPPORTED_MIME_TYPES:
|
if mime_type not in _SUPPORTED_MIME_TYPES:
|
||||||
@@ -244,11 +257,7 @@ class RemoteDocumentParser:
|
|||||||
Whether an archive copy is wanted. For PDFs, False skips the
|
Whether an archive copy is wanted. For PDFs, False skips the
|
||||||
remote engine and uses locally-extracted text instead.
|
remote engine and uses locally-extracted text instead.
|
||||||
"""
|
"""
|
||||||
config = RemoteEngineConfig(
|
config = RemoteEngineConfig.from_app_config()
|
||||||
engine=settings.REMOTE_OCR_ENGINE,
|
|
||||||
api_key=settings.REMOTE_OCR_API_KEY,
|
|
||||||
endpoint=settings.REMOTE_OCR_ENDPOINT,
|
|
||||||
)
|
|
||||||
|
|
||||||
if not config.engine_is_valid():
|
if not config.engine_is_valid():
|
||||||
logger.warning(
|
logger.warning(
|
||||||
|
|||||||
@@ -219,6 +219,13 @@ class ApplicationConfigurationSerializer(
|
|||||||
allow_null=True,
|
allow_null=True,
|
||||||
max_length=1024,
|
max_length=1024,
|
||||||
)
|
)
|
||||||
|
remote_ocr_api_key = ObfuscatedPasswordField(
|
||||||
|
required=False,
|
||||||
|
allow_null=True,
|
||||||
|
max_length=1024,
|
||||||
|
)
|
||||||
|
|
||||||
|
OBFUSCATED_FIELDS = ("llm_api_key", "remote_ocr_api_key")
|
||||||
|
|
||||||
def run_validation(self, data):
|
def run_validation(self, data):
|
||||||
# Empty strings treated as None to avoid unexpected behavior
|
# Empty strings treated as None to avoid unexpected behavior
|
||||||
@@ -230,11 +237,13 @@ class ApplicationConfigurationSerializer(
|
|||||||
data["language"] = None
|
data["language"] = None
|
||||||
if "llm_output_language" in data and data["llm_output_language"] == "":
|
if "llm_output_language" in data and data["llm_output_language"] == "":
|
||||||
data["llm_output_language"] = None
|
data["llm_output_language"] = None
|
||||||
if "llm_api_key" in data and data["llm_api_key"] is not None:
|
for field in self.OBFUSCATED_FIELDS:
|
||||||
if data["llm_api_key"] == "":
|
if field in data and data[field] is not None:
|
||||||
data["llm_api_key"] = None
|
if data[field] == "":
|
||||||
elif len(data["llm_api_key"].replace("*", "")) == 0:
|
data[field] = None
|
||||||
del data["llm_api_key"]
|
# Not a real value, don't overwrite the stored one
|
||||||
|
elif len(data[field].replace("*", "")) == 0:
|
||||||
|
del data[field]
|
||||||
return super().run_validation(data)
|
return super().run_validation(data)
|
||||||
|
|
||||||
def update(self, instance, validated_data):
|
def update(self, instance, validated_data):
|
||||||
|
|||||||
@@ -1203,6 +1203,11 @@ WEBHOOKS_ALLOW_INTERNAL_REQUESTS = get_bool_from_env(
|
|||||||
REMOTE_OCR_ENGINE = os.getenv("PAPERLESS_REMOTE_OCR_ENGINE")
|
REMOTE_OCR_ENGINE = os.getenv("PAPERLESS_REMOTE_OCR_ENGINE")
|
||||||
REMOTE_OCR_API_KEY = os.getenv("PAPERLESS_REMOTE_OCR_API_KEY")
|
REMOTE_OCR_API_KEY = os.getenv("PAPERLESS_REMOTE_OCR_API_KEY")
|
||||||
REMOTE_OCR_ENDPOINT = os.getenv("PAPERLESS_REMOTE_OCR_ENDPOINT")
|
REMOTE_OCR_ENDPOINT = os.getenv("PAPERLESS_REMOTE_OCR_ENDPOINT")
|
||||||
|
REMOTE_OCR_MODE = get_choice_from_env(
|
||||||
|
"PAPERLESS_REMOTE_OCR_MODE",
|
||||||
|
{"always", "workflow_only"},
|
||||||
|
default="always",
|
||||||
|
)
|
||||||
|
|
||||||
################################################################################
|
################################################################################
|
||||||
# AI Settings #
|
# AI Settings #
|
||||||
|
|||||||
@@ -114,7 +114,26 @@ def remote_parser() -> Generator[RemoteDocumentParser, None, None]:
|
|||||||
|
|
||||||
|
|
||||||
@pytest.fixture()
|
@pytest.fixture()
|
||||||
def azure_settings(settings: SettingsWrapper) -> SettingsWrapper:
|
def empty_remote_ocr_app_config(mocker: MockerFixture) -> MagicMock:
|
||||||
|
# empty app config without accessing db
|
||||||
|
app_config = mocker.MagicMock(
|
||||||
|
remote_ocr_engine=None,
|
||||||
|
remote_ocr_api_key=None,
|
||||||
|
remote_ocr_endpoint=None,
|
||||||
|
remote_ocr_mode=None,
|
||||||
|
)
|
||||||
|
mocker.patch(
|
||||||
|
"paperless.config.BaseConfig._get_config_instance",
|
||||||
|
return_value=app_config,
|
||||||
|
)
|
||||||
|
return app_config
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture()
|
||||||
|
def azure_settings(
|
||||||
|
settings: SettingsWrapper,
|
||||||
|
empty_remote_ocr_app_config: MagicMock,
|
||||||
|
) -> SettingsWrapper:
|
||||||
"""Configure Django settings for a valid Azure AI OCR engine.
|
"""Configure Django settings for a valid Azure AI OCR engine.
|
||||||
|
|
||||||
Sets ``REMOTE_OCR_ENGINE``, ``REMOTE_OCR_API_KEY``, and
|
Sets ``REMOTE_OCR_ENGINE``, ``REMOTE_OCR_API_KEY``, and
|
||||||
@@ -133,7 +152,10 @@ def azure_settings(settings: SettingsWrapper) -> SettingsWrapper:
|
|||||||
|
|
||||||
|
|
||||||
@pytest.fixture()
|
@pytest.fixture()
|
||||||
def no_engine_settings(settings: SettingsWrapper) -> SettingsWrapper:
|
def no_engine_settings(
|
||||||
|
settings: SettingsWrapper,
|
||||||
|
empty_remote_ocr_app_config: MagicMock,
|
||||||
|
) -> SettingsWrapper:
|
||||||
"""Configure Django settings with no remote engine configured.
|
"""Configure Django settings with no remote engine configured.
|
||||||
|
|
||||||
Returns
|
Returns
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ from unittest.mock import Mock
|
|||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from documents.parsers import ParseError
|
from documents.parsers import ParseError
|
||||||
|
from paperless.models import ApplicationConfiguration
|
||||||
from paperless.parsers import ParserContext
|
from paperless.parsers import ParserContext
|
||||||
from paperless.parsers import ParserProtocol
|
from paperless.parsers import ParserProtocol
|
||||||
from paperless.parsers.remote import RemoteDocumentParser
|
from paperless.parsers.remote import RemoteDocumentParser
|
||||||
@@ -198,21 +199,21 @@ class TestRemoteParserScore:
|
|||||||
|
|
||||||
def test_score_returns_none_when_api_key_missing(
|
def test_score_returns_none_when_api_key_missing(
|
||||||
self,
|
self,
|
||||||
settings: SettingsWrapper,
|
no_engine_settings: SettingsWrapper,
|
||||||
) -> None:
|
) -> None:
|
||||||
settings.REMOTE_OCR_ENGINE = "azureai"
|
no_engine_settings.REMOTE_OCR_ENGINE = "azureai"
|
||||||
settings.REMOTE_OCR_API_KEY = None
|
no_engine_settings.REMOTE_OCR_ENDPOINT = (
|
||||||
settings.REMOTE_OCR_ENDPOINT = "https://test.cognitiveservices.azure.com"
|
"https://test.cognitiveservices.azure.com"
|
||||||
|
)
|
||||||
result = RemoteDocumentParser.score("application/pdf", "doc.pdf")
|
result = RemoteDocumentParser.score("application/pdf", "doc.pdf")
|
||||||
assert result is None
|
assert result is None
|
||||||
|
|
||||||
def test_score_returns_none_when_endpoint_missing(
|
def test_score_returns_none_when_endpoint_missing(
|
||||||
self,
|
self,
|
||||||
settings: SettingsWrapper,
|
no_engine_settings: SettingsWrapper,
|
||||||
) -> None:
|
) -> None:
|
||||||
settings.REMOTE_OCR_ENGINE = "azureai"
|
no_engine_settings.REMOTE_OCR_ENGINE = "azureai"
|
||||||
settings.REMOTE_OCR_API_KEY = "key"
|
no_engine_settings.REMOTE_OCR_API_KEY = "key"
|
||||||
settings.REMOTE_OCR_ENDPOINT = None
|
|
||||||
result = RemoteDocumentParser.score("application/pdf", "doc.pdf")
|
result = RemoteDocumentParser.score("application/pdf", "doc.pdf")
|
||||||
assert result is None
|
assert result is None
|
||||||
|
|
||||||
@@ -227,6 +228,24 @@ class TestRemoteParserScore:
|
|||||||
score = RemoteDocumentParser.score("application/pdf", "doc.pdf")
|
score = RemoteDocumentParser.score("application/pdf", "doc.pdf")
|
||||||
assert score is not None and score > 10
|
assert score is not None and score > 10
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
def test_score_uses_app_config_when_env_unset(
|
||||||
|
self,
|
||||||
|
settings: SettingsWrapper,
|
||||||
|
) -> None:
|
||||||
|
"""The app config alone is enough to activate the parser."""
|
||||||
|
settings.REMOTE_OCR_ENGINE = None
|
||||||
|
settings.REMOTE_OCR_API_KEY = None
|
||||||
|
settings.REMOTE_OCR_ENDPOINT = None
|
||||||
|
config = ApplicationConfiguration.objects.first()
|
||||||
|
assert config is not None
|
||||||
|
config.remote_ocr_engine = "azureai"
|
||||||
|
config.remote_ocr_api_key = "app-config-key"
|
||||||
|
config.remote_ocr_endpoint = "https://config.cognitiveservices.azure.com"
|
||||||
|
config.save()
|
||||||
|
|
||||||
|
assert RemoteDocumentParser.score("application/pdf", "doc.pdf") == 20
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Properties
|
# Properties
|
||||||
|
|||||||
@@ -1277,6 +1277,8 @@ class TestParserFileTypes:
|
|||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
# Remote ocr config from ApplicationConfiguration needs DB access
|
||||||
|
@pytest.mark.django_db
|
||||||
class TestRasterisedDocumentParserRegistry:
|
class TestRasterisedDocumentParserRegistry:
|
||||||
def test_registered_in_defaults(self) -> None:
|
def test_registered_in_defaults(self) -> None:
|
||||||
from paperless.parsers.registry import ParserRegistry
|
from paperless.parsers.registry import ParserRegistry
|
||||||
|
|||||||
@@ -15,7 +15,6 @@ from paperless.checks import audit_log_check
|
|||||||
from paperless.checks import binaries_check
|
from paperless.checks import binaries_check
|
||||||
from paperless.checks import check_default_language_available
|
from paperless.checks import check_default_language_available
|
||||||
from paperless.checks import check_deprecated_db_settings
|
from paperless.checks import check_deprecated_db_settings
|
||||||
from paperless.checks import check_remote_parser_configured
|
|
||||||
from paperless.checks import check_v3_minimum_upgrade_version
|
from paperless.checks import check_v3_minimum_upgrade_version
|
||||||
from paperless.checks import debug_mode_check
|
from paperless.checks import debug_mode_check
|
||||||
from paperless.checks import paths_check
|
from paperless.checks import paths_check
|
||||||
@@ -631,31 +630,6 @@ class TestV3MinimumUpgradeVersionCheck:
|
|||||||
assert check_v3_minimum_upgrade_version(None) == []
|
assert check_v3_minimum_upgrade_version(None) == []
|
||||||
|
|
||||||
|
|
||||||
class TestRemoteParserChecks:
|
|
||||||
def test_no_engine(self, settings: SettingsWrapper) -> None:
|
|
||||||
settings.REMOTE_OCR_ENGINE = None
|
|
||||||
msgs = check_remote_parser_configured(None)
|
|
||||||
|
|
||||||
assert len(msgs) == 0
|
|
||||||
|
|
||||||
def test_azure_no_endpoint(self, settings: SettingsWrapper) -> None:
|
|
||||||
|
|
||||||
settings.REMOTE_OCR_ENGINE = "azureai"
|
|
||||||
settings.REMOTE_OCR_API_KEY = "somekey"
|
|
||||||
settings.REMOTE_OCR_ENDPOINT = None
|
|
||||||
|
|
||||||
msgs = check_remote_parser_configured(None)
|
|
||||||
|
|
||||||
assert len(msgs) == 1
|
|
||||||
|
|
||||||
msg = msgs[0]
|
|
||||||
|
|
||||||
assert (
|
|
||||||
"Azure AI remote parser requires endpoint and API key to be configured."
|
|
||||||
in msg.msg
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class TestTesseractChecks:
|
class TestTesseractChecks:
|
||||||
def test_default_language(self) -> None:
|
def test_default_language(self) -> None:
|
||||||
check_default_language_available(None)
|
check_default_language_available(None)
|
||||||
|
|||||||
@@ -468,6 +468,124 @@ class TestParserRegistryGetParserForFile:
|
|||||||
assert result is AcceptingBuiltin
|
assert result is AcceptingBuiltin
|
||||||
|
|
||||||
|
|
||||||
|
class TestParserRegistryRemoteParsers:
|
||||||
|
"""Verify the allow_remote filter in ParserRegistry.get_parser_for_file()."""
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _remote_parser_cls() -> type:
|
||||||
|
class RemoteParser:
|
||||||
|
name = "remote"
|
||||||
|
version = "1.0"
|
||||||
|
author = "A"
|
||||||
|
url = "https://example.com/remote"
|
||||||
|
uses_remote_service = True
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def supported_mime_types(cls):
|
||||||
|
return {"text/plain": ".txt"}
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def score(cls, mime_type, filename, path=None):
|
||||||
|
return 20
|
||||||
|
|
||||||
|
return RemoteParser
|
||||||
|
|
||||||
|
def test_remote_parser_wins_when_remote_allowed(
|
||||||
|
self,
|
||||||
|
dummy_parser_cls: type,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN: A remote parser scoring 20 and a local parser scoring 10.
|
||||||
|
WHEN: get_parser_for_file() is called with allow_remote=True.
|
||||||
|
THEN: The remote parser is returned.
|
||||||
|
"""
|
||||||
|
remote_parser_cls = self._remote_parser_cls()
|
||||||
|
registry = ParserRegistry()
|
||||||
|
registry.register_builtin(dummy_parser_cls)
|
||||||
|
registry.register_builtin(remote_parser_cls)
|
||||||
|
|
||||||
|
result = registry.get_parser_for_file(
|
||||||
|
"text/plain",
|
||||||
|
"readme.txt",
|
||||||
|
allow_remote=True,
|
||||||
|
)
|
||||||
|
assert result is remote_parser_cls
|
||||||
|
|
||||||
|
def test_remote_parser_skipped_when_remote_not_allowed(
|
||||||
|
self,
|
||||||
|
dummy_parser_cls: type,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN: A remote parser scoring 20 and a local parser scoring 10.
|
||||||
|
WHEN: get_parser_for_file() is called with allow_remote=False.
|
||||||
|
THEN: The local parser is returned despite its lower score.
|
||||||
|
"""
|
||||||
|
registry = ParserRegistry()
|
||||||
|
registry.register_builtin(dummy_parser_cls)
|
||||||
|
registry.register_builtin(self._remote_parser_cls())
|
||||||
|
|
||||||
|
result = registry.get_parser_for_file(
|
||||||
|
"text/plain",
|
||||||
|
"readme.txt",
|
||||||
|
allow_remote=False,
|
||||||
|
)
|
||||||
|
assert result is dummy_parser_cls
|
||||||
|
|
||||||
|
def test_no_parser_when_only_remote_available_and_not_allowed(self) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN: A registry whose only candidate declares uses_remote_service.
|
||||||
|
WHEN: get_parser_for_file() is called with allow_remote=False.
|
||||||
|
THEN: None is returned — the remote parser is never used as a
|
||||||
|
fallback when remote processing was not requested.
|
||||||
|
"""
|
||||||
|
registry = ParserRegistry()
|
||||||
|
registry.register_builtin(self._remote_parser_cls())
|
||||||
|
|
||||||
|
result = registry.get_parser_for_file(
|
||||||
|
"text/plain",
|
||||||
|
"readme.txt",
|
||||||
|
allow_remote=False,
|
||||||
|
)
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
def test_parser_without_attribute_treated_as_local(
|
||||||
|
self,
|
||||||
|
dummy_parser_cls: type,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN: A third-party parser predating uses_remote_service, so it does
|
||||||
|
not declare the attribute at all.
|
||||||
|
WHEN: get_parser_for_file() is called with allow_remote=False.
|
||||||
|
THEN: It is still considered, i.e. treated as fully local, rather
|
||||||
|
than raising AttributeError.
|
||||||
|
"""
|
||||||
|
assert not hasattr(dummy_parser_cls, "uses_remote_service")
|
||||||
|
|
||||||
|
registry = ParserRegistry()
|
||||||
|
registry.register_builtin(dummy_parser_cls)
|
||||||
|
|
||||||
|
result = registry.get_parser_for_file(
|
||||||
|
"text/plain",
|
||||||
|
"readme.txt",
|
||||||
|
allow_remote=False,
|
||||||
|
)
|
||||||
|
assert result is dummy_parser_cls
|
||||||
|
|
||||||
|
def test_remote_allowed_by_default(self) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN: A registry containing only a remote parser.
|
||||||
|
WHEN: get_parser_for_file() is called without allow_remote.
|
||||||
|
THEN: The remote parser is returned — callers that do not opt in to
|
||||||
|
the filter keep the previous behaviour.
|
||||||
|
"""
|
||||||
|
remote_parser_cls = self._remote_parser_cls()
|
||||||
|
registry = ParserRegistry()
|
||||||
|
registry.register_builtin(remote_parser_cls)
|
||||||
|
|
||||||
|
result = registry.get_parser_for_file("text/plain", "readme.txt")
|
||||||
|
assert result is remote_parser_cls
|
||||||
|
|
||||||
|
|
||||||
class TestDiscover:
|
class TestDiscover:
|
||||||
"""Verify entrypoint discovery in ParserRegistry.discover()."""
|
"""Verify entrypoint discovery in ParserRegistry.discover()."""
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,113 @@
|
|||||||
|
"""Tests for RemoteOCRConfig precedence between app config and Django settings."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from django.test import override_settings
|
||||||
|
|
||||||
|
from paperless.config import RemoteOCRConfig
|
||||||
|
from paperless.models import RemoteOCRMode
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture()
|
||||||
|
def null_app_config(mocker) -> MagicMock:
|
||||||
|
"""Mock ApplicationConfiguration with all fields None → falls back to Django settings."""
|
||||||
|
return mocker.MagicMock(
|
||||||
|
remote_ocr_engine=None,
|
||||||
|
remote_ocr_api_key=None,
|
||||||
|
remote_ocr_endpoint=None,
|
||||||
|
remote_ocr_mode=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture()
|
||||||
|
def make_remote_ocr_config(mocker):
|
||||||
|
def _make(app_config, **django_settings_overrides):
|
||||||
|
mocker.patch(
|
||||||
|
"paperless.config.BaseConfig._get_config_instance",
|
||||||
|
return_value=app_config,
|
||||||
|
)
|
||||||
|
with override_settings(**django_settings_overrides):
|
||||||
|
return RemoteOCRConfig()
|
||||||
|
|
||||||
|
return _make
|
||||||
|
|
||||||
|
|
||||||
|
class TestRemoteOCRConfig:
|
||||||
|
def test_falls_back_to_settings(
|
||||||
|
self,
|
||||||
|
make_remote_ocr_config,
|
||||||
|
null_app_config,
|
||||||
|
) -> None:
|
||||||
|
cfg = make_remote_ocr_config(
|
||||||
|
null_app_config,
|
||||||
|
REMOTE_OCR_ENGINE="azureai",
|
||||||
|
REMOTE_OCR_API_KEY="env-key",
|
||||||
|
REMOTE_OCR_ENDPOINT="https://env.cognitiveservices.azure.com",
|
||||||
|
REMOTE_OCR_MODE=RemoteOCRMode.WORKFLOW_ONLY,
|
||||||
|
)
|
||||||
|
assert cfg.remote_ocr_engine == "azureai"
|
||||||
|
assert cfg.remote_ocr_api_key == "env-key"
|
||||||
|
assert cfg.remote_ocr_endpoint == "https://env.cognitiveservices.azure.com"
|
||||||
|
assert cfg.remote_ocr_mode == RemoteOCRMode.WORKFLOW_ONLY
|
||||||
|
|
||||||
|
def test_app_config_takes_precedence(
|
||||||
|
self,
|
||||||
|
make_remote_ocr_config,
|
||||||
|
mocker,
|
||||||
|
) -> None:
|
||||||
|
app_config = mocker.MagicMock(
|
||||||
|
remote_ocr_engine="azureai",
|
||||||
|
remote_ocr_api_key="db-key",
|
||||||
|
remote_ocr_endpoint="https://db.cognitiveservices.azure.com",
|
||||||
|
remote_ocr_mode=RemoteOCRMode.WORKFLOW_ONLY,
|
||||||
|
)
|
||||||
|
cfg = make_remote_ocr_config(
|
||||||
|
app_config,
|
||||||
|
REMOTE_OCR_ENGINE=None,
|
||||||
|
REMOTE_OCR_API_KEY="env-key",
|
||||||
|
REMOTE_OCR_ENDPOINT="https://env.cognitiveservices.azure.com",
|
||||||
|
REMOTE_OCR_MODE=RemoteOCRMode.ALWAYS,
|
||||||
|
)
|
||||||
|
assert cfg.remote_ocr_engine == "azureai"
|
||||||
|
assert cfg.remote_ocr_api_key == "db-key"
|
||||||
|
assert cfg.remote_ocr_endpoint == "https://db.cognitiveservices.azure.com"
|
||||||
|
assert cfg.remote_ocr_mode == RemoteOCRMode.WORKFLOW_ONLY
|
||||||
|
|
||||||
|
def test_unset_everywhere(
|
||||||
|
self,
|
||||||
|
make_remote_ocr_config,
|
||||||
|
null_app_config,
|
||||||
|
) -> None:
|
||||||
|
cfg = make_remote_ocr_config(
|
||||||
|
null_app_config,
|
||||||
|
REMOTE_OCR_ENGINE=None,
|
||||||
|
REMOTE_OCR_API_KEY=None,
|
||||||
|
REMOTE_OCR_ENDPOINT=None,
|
||||||
|
)
|
||||||
|
assert cfg.remote_ocr_engine is None
|
||||||
|
assert cfg.remote_ocr_api_key is None
|
||||||
|
assert cfg.remote_ocr_endpoint is None
|
||||||
|
|
||||||
|
|
||||||
|
class TestRemoteOCRByDefault:
|
||||||
|
def test_always_mode(self, make_remote_ocr_config, null_app_config) -> None:
|
||||||
|
cfg = make_remote_ocr_config(
|
||||||
|
null_app_config,
|
||||||
|
REMOTE_OCR_MODE=RemoteOCRMode.ALWAYS,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert cfg.remote_ocr_by_default is True
|
||||||
|
|
||||||
|
def test_workflow_only_mode(self, make_remote_ocr_config, null_app_config) -> None:
|
||||||
|
cfg = make_remote_ocr_config(
|
||||||
|
null_app_config,
|
||||||
|
REMOTE_OCR_MODE=RemoteOCRMode.WORKFLOW_ONLY,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert cfg.remote_ocr_by_default is False
|
||||||
Reference in New Issue
Block a user