mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-08-11 05:13:18 +00:00
Compare commits
8
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
49a074945c | ||
|
|
222552f9fd | ||
|
|
9ba637f828 | ||
|
|
f348386a9d | ||
|
|
c6caeff58a | ||
|
|
fdb81087a3 | ||
|
|
a6695392c7 | ||
|
|
359c240814 |
@@ -129,8 +129,8 @@ jobs:
|
||||
~/.pnpm-store
|
||||
~/.cache
|
||||
key: ${{ runner.os }}-frontend-${{ hashFiles('src-ui/pnpm-lock.yaml') }}
|
||||
- name: Install dependencies
|
||||
run: cd src-ui && pnpm install --frozen-lockfile
|
||||
- name: Re-link Angular CLI
|
||||
run: cd src-ui && pnpm link @angular/cli
|
||||
- name: Run lint
|
||||
run: cd src-ui && pnpm run lint
|
||||
unit-tests:
|
||||
@@ -168,8 +168,8 @@ jobs:
|
||||
~/.pnpm-store
|
||||
~/.cache
|
||||
key: ${{ runner.os }}-frontend-${{ hashFiles('src-ui/pnpm-lock.yaml') }}
|
||||
- name: Install dependencies
|
||||
run: cd src-ui && pnpm install --frozen-lockfile
|
||||
- name: Re-link Angular CLI
|
||||
run: cd src-ui && pnpm link @angular/cli
|
||||
- name: Run Jest unit tests
|
||||
run: cd src-ui && pnpm run test --max-workers=2 --shard=${{ matrix.shard-index }}/${{ matrix.shard-count }}
|
||||
- name: Upload test results to Codecov
|
||||
@@ -223,15 +223,18 @@ jobs:
|
||||
~/.pnpm-store
|
||||
~/.cache
|
||||
key: ${{ runner.os }}-frontend-${{ hashFiles('src-ui/pnpm-lock.yaml') }}
|
||||
- name: Re-link Angular CLI
|
||||
run: cd src-ui && pnpm link @angular/cli
|
||||
- name: Install dependencies
|
||||
run: cd src-ui && pnpm install --frozen-lockfile
|
||||
run: cd src-ui && pnpm install --no-frozen-lockfile
|
||||
- name: Run Playwright E2E tests
|
||||
run: cd src-ui && pnpm exec playwright test --shard ${{ matrix.shard-index }}/${{ matrix.shard-count }}
|
||||
frontend-build:
|
||||
name: Frontend Build
|
||||
bundle-analysis:
|
||||
name: Bundle Analysis
|
||||
needs: [changes, unit-tests, e2e-tests]
|
||||
if: needs.changes.outputs.frontend_changed == 'true'
|
||||
runs-on: ubuntu-24.04
|
||||
environment: bundle-analysis
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
@@ -257,19 +260,21 @@ jobs:
|
||||
~/.pnpm-store
|
||||
~/.cache
|
||||
key: ${{ runner.os }}-frontend-${{ hashFiles('src-ui/pnpm-lock.yaml') }}
|
||||
- name: Install dependencies
|
||||
run: cd src-ui && pnpm install --frozen-lockfile
|
||||
- name: Build
|
||||
- name: Re-link Angular CLI
|
||||
run: cd src-ui && pnpm link @angular/cli
|
||||
- name: Build and analyze
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
run: cd src-ui && pnpm run build --configuration=production
|
||||
gate:
|
||||
name: Frontend CI Gate
|
||||
needs: [changes, install-dependencies, lint, unit-tests, e2e-tests, frontend-build]
|
||||
needs: [changes, install-dependencies, lint, unit-tests, e2e-tests, bundle-analysis]
|
||||
if: always()
|
||||
runs-on: ubuntu-slim
|
||||
steps:
|
||||
- name: Check gate
|
||||
env:
|
||||
BUILD_RESULT: ${{ needs['frontend-build'].result }}
|
||||
BUNDLE_ANALYSIS_RESULT: ${{ needs['bundle-analysis'].result }}
|
||||
E2E_RESULT: ${{ needs['e2e-tests'].result }}
|
||||
FRONTEND_CHANGED: ${{ needs.changes.outputs.frontend_changed }}
|
||||
INSTALL_RESULT: ${{ needs['install-dependencies'].result }}
|
||||
@@ -301,8 +306,8 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "${BUILD_RESULT}" != "success" ]]; then
|
||||
echo "::error::Frontend build job result: ${BUILD_RESULT}"
|
||||
if [[ "${BUNDLE_ANALYSIS_RESULT}" != "success" ]]; then
|
||||
echo "::error::Frontend bundle-analysis job result: ${BUNDLE_ANALYSIS_RESULT}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
@@ -61,7 +61,10 @@ jobs:
|
||||
~/.cache
|
||||
key: ${{ runner.os }}-frontenddeps-${{ hashFiles('src-ui/pnpm-lock.yaml') }}
|
||||
- name: Install frontend dependencies
|
||||
run: cd src-ui && pnpm install --frozen-lockfile
|
||||
if: steps.cache-frontend-deps.outputs.cache-hit != 'true'
|
||||
run: cd src-ui && pnpm install
|
||||
- name: Re-link Angular cli
|
||||
run: cd src-ui && pnpm link @angular/cli
|
||||
- name: Generate frontend translation strings
|
||||
run: |
|
||||
cd src-ui
|
||||
|
||||
+1
-2
@@ -301,8 +301,7 @@ The following methods are supported:
|
||||
- `delete`
|
||||
- No `parameters` required
|
||||
- `reprocess`
|
||||
- Optional `parameters`: `{ "remote_ocr": true }` to send the documents to the
|
||||
remote OCR engine, see [Remote OCR](usage.md#remote-ocr). Defaults to false.
|
||||
- No `parameters` required
|
||||
- `set_permissions`
|
||||
- Requires `parameters`:
|
||||
- `"set_permissions": PERMISSIONS_OBJ` (see format [above](#permissions)) and / or
|
||||
|
||||
@@ -2047,18 +2047,6 @@ password. All of these options come from their similarly-named [Django settings]
|
||||
|
||||
Defaults to None.
|
||||
|
||||
#### [`PAPERLESS_REMOTE_OCR_MODE=<str>`](#PAPERLESS_REMOTE_OCR_MODE) {#PAPERLESS_REMOTE_OCR_MODE}
|
||||
|
||||
: Which documents are sent to the remote OCR engine.
|
||||
|
||||
- `always`: every document of a supported file type is sent to the remote
|
||||
engine, bypassing the local OCR engine.
|
||||
- `workflow_only`: documents are processed locally unless a workflow
|
||||
explicitly enables remote OCR for them, letting you use the remote engine
|
||||
selectively.
|
||||
|
||||
Defaults to "always".
|
||||
|
||||
## AI {#ai}
|
||||
|
||||
#### [`PAPERLESS_AI_ENABLED=<bool>`](#PAPERLESS_AI_ENABLED) {#PAPERLESS_AI_ENABLED}
|
||||
|
||||
@@ -456,20 +456,6 @@ def score(
|
||||
return 10
|
||||
```
|
||||
|
||||
**Remote services**
|
||||
|
||||
If your parser sends document content to a remote service, declare it:
|
||||
|
||||
```python
|
||||
class MyCustomParser:
|
||||
uses_remote_service = True
|
||||
```
|
||||
|
||||
Paperless-ngx excludes such parsers when the document being consumed has not
|
||||
been marked for remote processing, so users can keep remote OCR off by default
|
||||
and enable it selectively with a workflow. Parsers that do not declare the
|
||||
attribute are treated as fully local and are always considered.
|
||||
|
||||
**Archive and rendition flags**
|
||||
|
||||
```python
|
||||
|
||||
+1
-51
@@ -648,48 +648,6 @@ happened while it was still encrypted, that original version will likewise be mi
|
||||
**Current limitation**: Passwords are stored as a simple list without descriptions. To handle
|
||||
multiple PDF types with different passwords, create separate workflows for each use case.
|
||||
|
||||
##### Remote OCR {#workflow-action-remote-ocr}
|
||||
|
||||
"Remote OCR" actions send the document to the configured remote OCR engine instead of processing it
|
||||
locally. To use remote OCR selectively, set the [remote OCR mode](configuration.md#PAPERLESS_REMOTE_OCR_MODE)
|
||||
to `workflow_only` then add this action to a workflow that matches only the documents you
|
||||
want sent to the remote engine. See [Remote OCR](#remote-ocr) for the engine setup. The action only works with
|
||||
a **Consumption Started** trigger.
|
||||
|
||||
The action takes no options, its presence is what enables remote OCR for a matching document.
|
||||
|
||||
If the remote engine is not configured, or does not support the document's file type, the document is
|
||||
processed locally instead and a warning is written to the log.
|
||||
|
||||
##### Apply AI Suggestions {#workflow-action-apply-ai-suggestions}
|
||||
|
||||
"Apply AI Suggestions" actions ask the configured AI service for title and metadata suggestions,
|
||||
the same as the AI suggestions shown on the document detail page, except applied automatically and in bulk.
|
||||
It requires [AI features](configuration.md#ai) to be enabled. You can specify:
|
||||
|
||||
- Which suggestions to apply: title, tags, correspondent, document type, storage path and / or created
|
||||
date. Suggestions for fields you did not select are discarded.
|
||||
- Whether to create missing items. By default only tags, correspondents and document types that
|
||||
already exist are assigned and any other suggestion is dropped. With this enabled, suggested items
|
||||
that do not exist are created. Storage paths are never created.
|
||||
- Whether to overwrite existing values. By default a field is only filled in if it is currently empty.
|
||||
Note that documents almost always already have a title and created date, so if you select those you
|
||||
will usually want to enable this too. Tags are an exception: suggested tags are always added and
|
||||
never replace the document's existing tags.
|
||||
|
||||
The action works with every trigger **except Consumption Started**, because suggestions are made from
|
||||
the document's text, which does not exist until after the document has been processed.
|
||||
|
||||
Because the query to the AI service is slow, the action is queued and runs in the background rather
|
||||
than as part of the workflow run itself. The document is updated once the suggestions come back.
|
||||
|
||||
!!! warning
|
||||
|
||||
Every matching document results in a query to the AI service, which may incur costs and have privacy
|
||||
implications. Queries can be slow, so a workflow matching a large number of documents can occupy the
|
||||
task queue, and delay consumption of new documents, etc. Consider narrowing the trigger filters,
|
||||
running in small batches and / or increasing workers.
|
||||
|
||||
#### Workflow placeholders
|
||||
|
||||
Titles and webhook payloads can be generated by workflows using [Jinja templates](https://jinja.palletsprojects.com/en/3.1.x/templates/).
|
||||
@@ -1126,19 +1084,11 @@ Paperless-ngx supports performing OCR on documents using remote services. At the
|
||||
[Microsoft's Azure "Document Intelligence" service](https://azure.microsoft.com/en-us/products/ai-services/ai-document-intelligence).
|
||||
This is of course a paid service (with a free tier) which requires an Azure account and subscription. Azure AI is not affiliated with
|
||||
Paperless-ngx in any way. When enabled, Paperless-ngx will automatically send appropriate documents to Azure for OCR processing, bypassing
|
||||
the local OCR engine. See the [configuration](configuration.md#PAPERLESS_REMOTE_OCR_ENGINE) options for more details. These
|
||||
settings can be supplied as environment variables or via **Application Configuration**.
|
||||
the local OCR engine. See the [configuration](configuration.md#PAPERLESS_REMOTE_OCR_ENGINE) options for more details.
|
||||
|
||||
Additionally, when using a commercial service with this feature, consider both potential costs as well as any associated file size
|
||||
or page limitations (e.g. with a free tier).
|
||||
|
||||
By default, every document of a supported file type is sent to the remote engine. To use it more selectively, set the
|
||||
[remote OCR mode](configuration.md#PAPERLESS_REMOTE_OCR_MODE) to `workflow_only`. Documents are then processed locally
|
||||
unless a [remote OCR workflow action](#workflow-action-remote-ocr) enables it for them, so you can limit the remote
|
||||
engine to particular documents.
|
||||
|
||||
Setting the mode to `workflow_only` also allows the **Reprocess** actions to selectively use remote OCR for individual documents.
|
||||
|
||||
## Architecture
|
||||
|
||||
Paperless-ngx consists of the following components:
|
||||
|
||||
@@ -38,6 +38,7 @@ dependencies = [
|
||||
"django-soft-delete~=1.0.18",
|
||||
"django-treenode>=0.24",
|
||||
"djangorestframework~=3.16",
|
||||
"djangorestframework-guardian~=0.4.0",
|
||||
"drf-spectacular~=0.30",
|
||||
"drf-spectacular-sidecar~=2026.7.1",
|
||||
"drf-writable-nested~=0.7.1",
|
||||
|
||||
+9
-12
@@ -56,13 +56,13 @@
|
||||
},
|
||||
"architect": {
|
||||
"build": {
|
||||
"builder": "@angular/build:application",
|
||||
"builder": "@angular-builders/custom-webpack:browser",
|
||||
"options": {
|
||||
"outputPath": {
|
||||
"base": "dist/paperless-ui",
|
||||
"browser": ""
|
||||
"customWebpackConfig": {
|
||||
"path": "./extra-webpack.config.ts"
|
||||
},
|
||||
"browser": "src/main.ts",
|
||||
"outputPath": "dist/paperless-ui",
|
||||
"main": "src/main.ts",
|
||||
"outputHashing": "none",
|
||||
"index": "src/index.html",
|
||||
"polyfills": [
|
||||
@@ -97,7 +97,6 @@
|
||||
"scripts": [],
|
||||
"allowedCommonJsDependencies": [
|
||||
"file-saver",
|
||||
"mime-names",
|
||||
"utif"
|
||||
],
|
||||
"extractLicenses": false,
|
||||
@@ -118,13 +117,11 @@
|
||||
"with": "src/environments/environment.prod.ts"
|
||||
}
|
||||
],
|
||||
"outputPath": {
|
||||
"base": "../src/documents/static/frontend/",
|
||||
"browser": ""
|
||||
},
|
||||
"outputPath": "../src/documents/static/frontend/",
|
||||
"optimization": true,
|
||||
"outputHashing": "none",
|
||||
"sourceMap": false,
|
||||
"namedChunks": false,
|
||||
"extractLicenses": true,
|
||||
"budgets": [
|
||||
{
|
||||
@@ -148,7 +145,7 @@
|
||||
"defaultConfiguration": ""
|
||||
},
|
||||
"serve": {
|
||||
"builder": "@angular/build:dev-server",
|
||||
"builder": "@angular-builders/custom-webpack:dev-server",
|
||||
"options": {
|
||||
"buildTarget": "paperless-ui:build:en-US"
|
||||
},
|
||||
@@ -159,7 +156,7 @@
|
||||
}
|
||||
},
|
||||
"extract-i18n": {
|
||||
"builder": "@angular/build:extract-i18n",
|
||||
"builder": "@angular-builders/custom-webpack:extract-i18n",
|
||||
"options": {
|
||||
"buildTarget": "paperless-ui:build"
|
||||
}
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
import {
|
||||
CustomWebpackBrowserSchema,
|
||||
TargetOptions,
|
||||
} from '@angular-builders/custom-webpack'
|
||||
import * as webpack from 'webpack'
|
||||
const { codecovWebpackPlugin } = require('@codecov/webpack-plugin')
|
||||
|
||||
export default (
|
||||
config: webpack.Configuration,
|
||||
options: CustomWebpackBrowserSchema,
|
||||
targetOptions: TargetOptions
|
||||
) => {
|
||||
if (config.plugins) {
|
||||
config.plugins.push(
|
||||
codecovWebpackPlugin({
|
||||
enableBundleAnalysis: process.env.CODECOV_TOKEN !== undefined,
|
||||
bundleName: 'paperless-ngx',
|
||||
uploadToken: process.env.CODECOV_TOKEN,
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
return config
|
||||
}
|
||||
+714
-720
File diff suppressed because it is too large
Load Diff
+18
-15
@@ -12,13 +12,13 @@
|
||||
"private": true,
|
||||
"dependencies": {
|
||||
"@angular/cdk": "^22.0.6",
|
||||
"@angular/common": "~22.1.0",
|
||||
"@angular/compiler": "~22.1.0",
|
||||
"@angular/core": "~22.1.0",
|
||||
"@angular/forms": "~22.1.0",
|
||||
"@angular/localize": "~22.1.0",
|
||||
"@angular/platform-browser": "~22.1.0",
|
||||
"@angular/router": "~22.1.0",
|
||||
"@angular/common": "~22.0.8",
|
||||
"@angular/compiler": "~22.0.8",
|
||||
"@angular/core": "~22.0.8",
|
||||
"@angular/forms": "~22.0.8",
|
||||
"@angular/localize": "~22.0.8",
|
||||
"@angular/platform-browser": "~22.0.8",
|
||||
"@angular/router": "~22.0.8",
|
||||
"@ng-bootstrap/ng-bootstrap": "^21.0.0",
|
||||
"@ng-select/ng-select": "^23.5.0",
|
||||
"@ngneat/dirty-check-forms": "^3.0.3",
|
||||
@@ -32,24 +32,26 @@
|
||||
"ngx-device-detector": "^12.0.0",
|
||||
"ngx-ui-tour-ng-bootstrap": "^19.0.0",
|
||||
"normalize-diacritics": "^5.0.0",
|
||||
"pdfjs-dist": "^6.2.108",
|
||||
"pdfjs-dist": "^6.0.227",
|
||||
"rxjs": "^7.8.2",
|
||||
"tslib": "^2.8.1",
|
||||
"utif": "^3.1.0",
|
||||
"uuid": "^14.0.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@angular-builders/custom-webpack": "^22.0.1",
|
||||
"@angular-builders/jest": "^22.0.1",
|
||||
"@angular-devkit/core": "^22.1.2",
|
||||
"@angular-devkit/schematics": "^22.1.2",
|
||||
"@angular-devkit/core": "^22.0.8",
|
||||
"@angular-devkit/schematics": "^22.0.8",
|
||||
"@angular-eslint/builder": "22.1.0",
|
||||
"@angular-eslint/eslint-plugin": "22.1.0",
|
||||
"@angular-eslint/eslint-plugin-template": "22.1.0",
|
||||
"@angular-eslint/schematics": "22.1.0",
|
||||
"@angular-eslint/template-parser": "22.1.0",
|
||||
"@angular/build": "22.1.2",
|
||||
"@angular/cli": "22.1.2",
|
||||
"@angular/compiler-cli": "~22.1.0",
|
||||
"@angular/build": "^22.0.8",
|
||||
"@angular/cli": "~22.0.5",
|
||||
"@angular/compiler-cli": "~22.0.8",
|
||||
"@codecov/webpack-plugin": "^2.0.1",
|
||||
"@playwright/test": "^1.62.0",
|
||||
"@types/jest": "^30.0.0",
|
||||
"@types/node": "^26.1.1",
|
||||
@@ -64,7 +66,8 @@
|
||||
"jest-websocket-mock": "^2.5.0",
|
||||
"prettier-plugin-organize-imports": "^4.3.0",
|
||||
"ts-node": "~10.9.1",
|
||||
"typescript": "^6.0.3"
|
||||
"typescript": "^6.0.3",
|
||||
"webpack": "^5.107.2"
|
||||
},
|
||||
"packageManager": "pnpm@11.15.1"
|
||||
"packageManager": "pnpm@10.26.0"
|
||||
}
|
||||
|
||||
Generated
+1798
-1811
File diff suppressed because it is too large
Load Diff
@@ -5,7 +5,6 @@ trustPolicy: no-downgrade
|
||||
trustPolicyExclude:
|
||||
- "chokidar@4.0.3"
|
||||
- "semver@6.3.1 || 5.7.2"
|
||||
blockExoticSubdeps: true
|
||||
allowBuilds:
|
||||
"@parcel/watcher": true
|
||||
canvas: true
|
||||
|
||||
@@ -14,48 +14,43 @@
|
||||
<a ngbNavLink>{{category}}</a>
|
||||
<ng-template ngbNavContent>
|
||||
<div class="p-3">
|
||||
@for (section of getCategorySections(category); track section) {
|
||||
@if (section) {
|
||||
<h5 class="mt-4 mb-3">{{section}}</h5>
|
||||
}
|
||||
<div class="row row-cols-1 row-cols-md-2 row-cols-lg-3 g-2">
|
||||
@for (option of getCategoryOptions(category, section); track option.key) {
|
||||
<div class="col">
|
||||
<div class="card bg-light">
|
||||
<div class="card-body">
|
||||
<div class="card-title d-flex align-items-center">
|
||||
<h6 class="mb-0">
|
||||
{{option.title}}
|
||||
</h6>
|
||||
<a class="btn btn-sm btn-link" title="Read the documentation about this setting" i18n-title [href]="getDocsUrl(option.config_key)" target="_blank" referrerpolicy="no-referrer">
|
||||
<i-bs name="info-circle"></i-bs>
|
||||
</a>
|
||||
@if (isSet(option.key)) {
|
||||
<button type="button" class="btn btn-sm btn-link text-danger ms-auto pe-0" title="Reset" i18n-title (click)="resetOption(option.key)">
|
||||
<i-bs class="me-1" name="x"></i-bs><ng-container i18n>Reset</ng-container>
|
||||
</button>
|
||||
}
|
||||
</div>
|
||||
<div class="mb-n3">
|
||||
@switch (option.type) {
|
||||
@case (ConfigOptionType.Select) { <pngx-input-select [formControlName]="option.key" [error]="errors[option.key]" [items]="option.choices" [allowNull]="true"></pngx-input-select> }
|
||||
@case (ConfigOptionType.Number) { <pngx-input-number [formControlName]="option.key" [error]="errors[option.key]" [showAdd]="false"></pngx-input-number> }
|
||||
@case (ConfigOptionType.Boolean) { <pngx-input-switch [formControlName]="option.key" [error]="errors[option.key]" [showUnsetNote]="true" [horizontal]="true" title="Enable" i18n-title></pngx-input-switch> }
|
||||
@case (ConfigOptionType.String) { <pngx-input-text [formControlName]="option.key" [error]="errors[option.key]"></pngx-input-text> }
|
||||
@case (ConfigOptionType.JSON) { <pngx-input-text [formControlName]="option.key" [error]="errors[option.key]"></pngx-input-text> }
|
||||
@case (ConfigOptionType.File) { <pngx-input-file [formControlName]="option.key" (upload)="uploadFile($event, option.key)" [error]="errors[option.key]"></pngx-input-file> }
|
||||
@case (ConfigOptionType.Password) { <pngx-input-password [formControlName]="option.key" [error]="errors[option.key]"></pngx-input-password> }
|
||||
}
|
||||
</div>
|
||||
@if (option.note) {
|
||||
<div class="form-text fst-italic">{{option.note}}</div>
|
||||
<div class="row row-cols-1 row-cols-md-2 row-cols-lg-3 g-2">
|
||||
@for (option of getCategoryOptions(category); track option.key) {
|
||||
<div class="col">
|
||||
<div class="card bg-light">
|
||||
<div class="card-body">
|
||||
<div class="card-title d-flex align-items-center">
|
||||
<h6 class="mb-0">
|
||||
{{option.title}}
|
||||
</h6>
|
||||
<a class="btn btn-sm btn-link" title="Read the documentation about this setting" i18n-title [href]="getDocsUrl(option.config_key)" target="_blank" referrerpolicy="no-referrer">
|
||||
<i-bs name="info-circle"></i-bs>
|
||||
</a>
|
||||
@if (isSet(option.key)) {
|
||||
<button type="button" class="btn btn-sm btn-link text-danger ms-auto pe-0" title="Reset" i18n-title (click)="resetOption(option.key)">
|
||||
<i-bs class="me-1" name="x"></i-bs><ng-container i18n>Reset</ng-container>
|
||||
</button>
|
||||
}
|
||||
</div>
|
||||
<div class="mb-n3">
|
||||
@switch (option.type) {
|
||||
@case (ConfigOptionType.Select) { <pngx-input-select [formControlName]="option.key" [error]="errors[option.key]" [items]="option.choices" [allowNull]="true"></pngx-input-select> }
|
||||
@case (ConfigOptionType.Number) { <pngx-input-number [formControlName]="option.key" [error]="errors[option.key]" [showAdd]="false"></pngx-input-number> }
|
||||
@case (ConfigOptionType.Boolean) { <pngx-input-switch [formControlName]="option.key" [error]="errors[option.key]" [showUnsetNote]="true" [horizontal]="true" title="Enable" i18n-title></pngx-input-switch> }
|
||||
@case (ConfigOptionType.String) { <pngx-input-text [formControlName]="option.key" [error]="errors[option.key]"></pngx-input-text> }
|
||||
@case (ConfigOptionType.JSON) { <pngx-input-text [formControlName]="option.key" [error]="errors[option.key]"></pngx-input-text> }
|
||||
@case (ConfigOptionType.File) { <pngx-input-file [formControlName]="option.key" (upload)="uploadFile($event, option.key)" [error]="errors[option.key]"></pngx-input-file> }
|
||||
@case (ConfigOptionType.Password) { <pngx-input-password [formControlName]="option.key" [error]="errors[option.key]"></pngx-input-password> }
|
||||
}
|
||||
</div>
|
||||
@if (option.note) {
|
||||
<div class="form-text fst-italic">{{option.note}}</div>
|
||||
}
|
||||
</div>
|
||||
</div>
|
||||
}
|
||||
</div>
|
||||
}
|
||||
</div>
|
||||
}
|
||||
</div>
|
||||
</div>
|
||||
</ng-template>
|
||||
</li>
|
||||
|
||||
@@ -8,11 +8,7 @@ import { NgbModule } from '@ng-bootstrap/ng-bootstrap'
|
||||
import { NgSelectModule } from '@ng-select/ng-select'
|
||||
import { NgxBootstrapIconsModule, allIcons } from 'ngx-bootstrap-icons'
|
||||
import { of, throwError } from 'rxjs'
|
||||
import {
|
||||
ConfigCategory,
|
||||
ConfigSection,
|
||||
OutputTypeConfig,
|
||||
} from 'src/app/data/paperless-config'
|
||||
import { OutputTypeConfig } from 'src/app/data/paperless-config'
|
||||
import { ConfigService } from 'src/app/services/config.service'
|
||||
import { SettingsService } from 'src/app/services/settings.service'
|
||||
import { ToastService } from 'src/app/services/toast.service'
|
||||
@@ -162,24 +158,4 @@ describe('ConfigComponent', () => {
|
||||
component.resetOption('barcodes_enabled')
|
||||
expect(component.configForm.get('barcodes_enabled').value).toBeNull()
|
||||
})
|
||||
|
||||
it('should group options into sections within a category, or not', () => {
|
||||
const sections = component.getCategorySections(ConfigCategory.OCR)
|
||||
expect(sections).toEqual([null, ConfigSection.RemoteOCR])
|
||||
expect(
|
||||
component
|
||||
.getCategoryOptions(ConfigCategory.OCR)
|
||||
.map((option) => option.key)
|
||||
).toContain('output_type')
|
||||
expect(
|
||||
component
|
||||
.getCategoryOptions(ConfigCategory.OCR, ConfigSection.RemoteOCR)
|
||||
.map((option) => option.key)
|
||||
).toEqual([
|
||||
'remote_ocr_engine',
|
||||
'remote_ocr_api_key',
|
||||
'remote_ocr_endpoint',
|
||||
'remote_ocr_mode',
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
@@ -74,20 +74,8 @@ export class ConfigComponent
|
||||
return Object.values(ConfigCategory)
|
||||
}
|
||||
|
||||
getCategorySections(category: string): string[] {
|
||||
return [
|
||||
...new Set(
|
||||
PaperlessConfigOptions.filter((o) => o.category === category).map(
|
||||
(o) => o.section ?? null // null means no section
|
||||
)
|
||||
),
|
||||
]
|
||||
}
|
||||
|
||||
getCategoryOptions(category: string, section: string = null): ConfigOption[] {
|
||||
return PaperlessConfigOptions.filter(
|
||||
(o) => o.category === category && (o.section ?? null) === section
|
||||
)
|
||||
getCategoryOptions(category: string): ConfigOption[] {
|
||||
return PaperlessConfigOptions.filter((o) => o.category === category)
|
||||
}
|
||||
|
||||
initialConfig: PaperlessConfig
|
||||
|
||||
@@ -111,7 +111,7 @@
|
||||
routerLinkActive="active" (click)="closeMenu()" [ngbPopover]="view.name"
|
||||
[disablePopover]="!slimSidebarEnabled" placement="end" container="body" triggers="mouseenter:mouseleave"
|
||||
popoverClass="popover-slim">
|
||||
<i-bs class="me-2" name="funnel"></i-bs><span><div class="d-inline-flex view-name"><span class="overflow-hidden" [class.text-wrap]="!slimSidebarEnabled">{{view.name}}</span></div>
|
||||
<i-bs class="me-2" [name]="view.icon || 'funnel'"></i-bs><span><div class="d-inline-flex view-name"><span class="overflow-hidden" [class.text-wrap]="!slimSidebarEnabled">{{view.name}}</span></div>
|
||||
@if (showSidebarCounts && !slimSidebarEnabled) {
|
||||
<span class="badge bg-info text-dark ms-2 d-inline">{{ savedViewService.getDocumentCount(view) }}</span>
|
||||
}
|
||||
|
||||
-28
@@ -1,28 +0,0 @@
|
||||
<div class="modal-header">
|
||||
<h4 class="modal-title" id="modal-basic-title">{{title}}</h4>
|
||||
<button type="button" class="btn-close" aria-label="Close" (click)="cancel()">
|
||||
</button>
|
||||
</div>
|
||||
<div class="modal-body">
|
||||
@if (messageBold) {
|
||||
<p class="text-break"><b>{{messageBold}}</b></p>
|
||||
}
|
||||
@if (message) {
|
||||
<p class="mb-0 text-break" [innerHTML]="message"></p>
|
||||
}
|
||||
@if (showRemoteOcr) {
|
||||
<div class="form-check mt-3">
|
||||
<input class="form-check-input" type="checkbox" id="reprocessRemoteOcr" [(ngModel)]="remoteOcr" />
|
||||
<label class="form-check-label" for="reprocessRemoteOcr" i18n>Use remote OCR</label>
|
||||
<div class="form-text" i18n>Sends the document to the configured remote OCR service, which may incur costs.</div>
|
||||
</div>
|
||||
}
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<button type="button" class="btn" [class]="cancelBtnClass" (click)="cancel()" [disabled]="!buttonsEnabled">
|
||||
<span class="d-inline-block" style="padding-bottom: 1px;">{{cancelBtnCaption}}</span>
|
||||
</button>
|
||||
<button type="button" class="btn" [class]="btnClass" (click)="confirm()" [disabled]="!confirmButtonEnabled || !buttonsEnabled">
|
||||
{{btnCaption}}
|
||||
</button>
|
||||
</div>
|
||||
-72
@@ -1,72 +0,0 @@
|
||||
import { provideHttpClient, withInterceptorsFromDi } from '@angular/common/http'
|
||||
import { provideHttpClientTesting } from '@angular/common/http/testing'
|
||||
import { ComponentFixture, TestBed } from '@angular/core/testing'
|
||||
import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap'
|
||||
import { RemoteOCRModeConfig } from 'src/app/data/paperless-config'
|
||||
import { SETTINGS_KEYS } from 'src/app/data/ui-settings'
|
||||
import { SettingsService } from 'src/app/services/settings.service'
|
||||
import { ReprocessConfirmDialogComponent } from './reprocess-confirm-dialog.component'
|
||||
|
||||
describe('ReprocessConfirmDialogComponent', () => {
|
||||
let component: ReprocessConfirmDialogComponent
|
||||
let fixture: ComponentFixture<ReprocessConfirmDialogComponent>
|
||||
let settingsService: SettingsService
|
||||
|
||||
const createComponent = (configured: boolean, mode: string) => {
|
||||
settingsService.set(SETTINGS_KEYS.REMOTE_OCR_CONFIGURED, configured)
|
||||
settingsService.set(SETTINGS_KEYS.REMOTE_OCR_MODE, mode)
|
||||
|
||||
fixture = TestBed.createComponent(ReprocessConfirmDialogComponent)
|
||||
component = fixture.componentInstance
|
||||
fixture.detectChanges()
|
||||
}
|
||||
|
||||
beforeEach(async () => {
|
||||
TestBed.configureTestingModule({
|
||||
providers: [
|
||||
NgbActiveModal,
|
||||
provideHttpClient(withInterceptorsFromDi()),
|
||||
provideHttpClientTesting(),
|
||||
],
|
||||
imports: [ReprocessConfirmDialogComponent],
|
||||
}).compileComponents()
|
||||
|
||||
settingsService = TestBed.inject(SettingsService)
|
||||
})
|
||||
|
||||
it('should not request remote OCR by default', () => {
|
||||
createComponent(true, RemoteOCRModeConfig.WORKFLOW_ONLY)
|
||||
|
||||
expect(component.remoteOcr).toBeFalsy()
|
||||
})
|
||||
|
||||
it('should not offer remote OCR when no engine is configured', () => {
|
||||
createComponent(false, RemoteOCRModeConfig.WORKFLOW_ONLY)
|
||||
|
||||
expect(component.showRemoteOcr).toBeFalsy()
|
||||
expect(
|
||||
fixture.nativeElement.querySelector('#reprocessRemoteOcr')
|
||||
).toBeNull()
|
||||
})
|
||||
|
||||
it('should not offer remote OCR when it already handles every document', () => {
|
||||
createComponent(true, RemoteOCRModeConfig.ALWAYS)
|
||||
|
||||
expect(component.showRemoteOcr).toBeFalsy()
|
||||
expect(
|
||||
fixture.nativeElement.querySelector('#reprocessRemoteOcr')
|
||||
).toBeNull()
|
||||
})
|
||||
|
||||
it('should offer remote OCR when configured and selective', () => {
|
||||
createComponent(true, RemoteOCRModeConfig.WORKFLOW_ONLY)
|
||||
|
||||
expect(component.showRemoteOcr).toBeTruthy()
|
||||
const checkbox = fixture.nativeElement.querySelector('#reprocessRemoteOcr')
|
||||
expect(checkbox).not.toBeNull()
|
||||
|
||||
checkbox.click()
|
||||
fixture.detectChanges()
|
||||
expect(component.remoteOcr).toBeTruthy()
|
||||
})
|
||||
})
|
||||
-20
@@ -1,20 +0,0 @@
|
||||
import { Component, inject } from '@angular/core'
|
||||
import { FormsModule } from '@angular/forms'
|
||||
import { SettingsService } from 'src/app/services/settings.service'
|
||||
import { ConfirmDialogComponent } from '../confirm-dialog.component'
|
||||
|
||||
@Component({
|
||||
selector: 'pngx-reprocess-confirm-dialog',
|
||||
templateUrl: './reprocess-confirm-dialog.component.html',
|
||||
imports: [FormsModule],
|
||||
})
|
||||
export class ReprocessConfirmDialogComponent extends ConfirmDialogComponent {
|
||||
private settings = inject(SettingsService)
|
||||
|
||||
remoteOcr: boolean = false
|
||||
|
||||
public get showRemoteOcr(): boolean {
|
||||
// Hidden when it is not configured, or when it already handles every document anyway.
|
||||
return this.settings.remoteOCRIsSelectable
|
||||
}
|
||||
}
|
||||
-46
@@ -455,52 +455,6 @@
|
||||
</div>
|
||||
</div>
|
||||
}
|
||||
@case (WorkflowActionType.RemoteOcr) {
|
||||
<div class="row">
|
||||
<div class="col">
|
||||
<p class="text-muted small" i18n>The document will be sent to the configured remote OCR service. May incur costs.</p>
|
||||
</div>
|
||||
</div>
|
||||
}
|
||||
@case (WorkflowActionType.ApplyAiSuggestions) {
|
||||
<div class="row">
|
||||
<div class="col">
|
||||
<p class="text-muted small" i18n>The document will be sent to the configured AI service for suggestions. Consider costs and privacy.</p>
|
||||
<pngx-input-select
|
||||
i18n-title
|
||||
title="Apply suggestions for"
|
||||
[items]="aiSuggestionFieldOptions"
|
||||
[multiple]="true"
|
||||
formControlName="ai_suggestion_fields"
|
||||
[error]="error?.actions?.[i]?.ai_suggestion_fields"
|
||||
hint="Suggestions for fields that are not selected are discarded."
|
||||
i18n-hint
|
||||
></pngx-input-select>
|
||||
</div>
|
||||
</div>
|
||||
<div class="row">
|
||||
<div class="col-md-6">
|
||||
<pngx-input-switch
|
||||
[horizontal]="true"
|
||||
i18n-title
|
||||
title="Create missing items"
|
||||
formControlName="ai_create_missing"
|
||||
hint="Create suggested tags, correspondents and document types that do not exist yet."
|
||||
i18n-hint
|
||||
></pngx-input-switch>
|
||||
</div>
|
||||
<div class="col-md-6">
|
||||
<pngx-input-switch
|
||||
[horizontal]="true"
|
||||
i18n-title
|
||||
title="Overwrite existing values"
|
||||
formControlName="ai_overwrite_existing"
|
||||
hint="Apply suggestions even if the document already has a value. Tags are always added, never replaced."
|
||||
i18n-hint
|
||||
></pngx-input-switch>
|
||||
</div>
|
||||
</div>
|
||||
}
|
||||
}
|
||||
</div>
|
||||
</ng-template>
|
||||
|
||||
+3
-228
@@ -22,7 +22,6 @@ import {
|
||||
} from 'src/app/data/matching-model'
|
||||
import { Workflow } from 'src/app/data/workflow'
|
||||
import {
|
||||
AISuggestionField,
|
||||
WorkflowAction,
|
||||
WorkflowActionType,
|
||||
} from 'src/app/data/workflow-action'
|
||||
@@ -30,7 +29,6 @@ import {
|
||||
DocumentSource,
|
||||
WorkflowTriggerType,
|
||||
} from 'src/app/data/workflow-trigger'
|
||||
import { SETTINGS_KEYS } from 'src/app/data/ui-settings'
|
||||
import { IfOwnerDirective } from 'src/app/directives/if-owner.directive'
|
||||
import { IfPermissionsDirective } from 'src/app/directives/if-permissions.directive'
|
||||
import { CorrespondentService } from 'src/app/services/rest/correspondent.service'
|
||||
@@ -50,7 +48,6 @@ import { TagsComponent } from '../../input/tags/tags.component'
|
||||
import { TextComponent } from '../../input/text/text.component'
|
||||
import { EditDialogMode } from '../edit-dialog.component'
|
||||
import {
|
||||
AI_SUGGESTION_FIELD_OPTIONS,
|
||||
DOCUMENT_SOURCE_OPTIONS,
|
||||
SCHEDULE_DATE_FIELD_OPTIONS,
|
||||
TriggerFilterType,
|
||||
@@ -227,12 +224,7 @@ describe('WorkflowEditDialogComponent', () => {
|
||||
).toEqual('Document Added')
|
||||
expect(component.getTriggerTypeOptionName(null)).toEqual('')
|
||||
expect(component.sourceOptions).toEqual(DOCUMENT_SOURCE_OPTIONS)
|
||||
// Remote OCR is absent until the workflow has a consumption trigger
|
||||
expect(component.actionTypeOptions).toEqual(
|
||||
WORKFLOW_ACTION_OPTIONS.filter(
|
||||
(a) => a.id !== WorkflowActionType.RemoteOcr
|
||||
)
|
||||
)
|
||||
expect(component.actionTypeOptions).toEqual(WORKFLOW_ACTION_OPTIONS)
|
||||
expect(
|
||||
component.getActionTypeOptionName(WorkflowActionType.Assignment)
|
||||
).toEqual('Assignment')
|
||||
@@ -241,231 +233,14 @@ describe('WorkflowEditDialogComponent', () => {
|
||||
SCHEDULE_DATE_FIELD_OPTIONS
|
||||
)
|
||||
|
||||
// Email, remote OCR and AI all disabled
|
||||
// Email disabled
|
||||
jest.spyOn(settingsService, 'get').mockReturnValue(false)
|
||||
component.ngOnInit()
|
||||
expect(component.actionTypeOptions).toEqual(
|
||||
WORKFLOW_ACTION_OPTIONS.filter(
|
||||
(a) =>
|
||||
a.id !== WorkflowActionType.Email &&
|
||||
a.id !== WorkflowActionType.RemoteOcr &&
|
||||
a.id !== WorkflowActionType.ApplyAiSuggestions
|
||||
)
|
||||
WORKFLOW_ACTION_OPTIONS.filter((a) => a.id !== WorkflowActionType.Email)
|
||||
)
|
||||
})
|
||||
|
||||
it('should offer remote OCR only for consumption workflows', () => {
|
||||
jest.spyOn(settingsService, 'get').mockReturnValue(true)
|
||||
|
||||
// A consumption trigger makes the action reachable
|
||||
component.object = {
|
||||
name: 'Workflow 1',
|
||||
order: 0,
|
||||
enabled: true,
|
||||
triggers: [{ type: WorkflowTriggerType.Consumption }],
|
||||
actions: [],
|
||||
} as Workflow
|
||||
component.ngOnInit()
|
||||
expect(component.actionTypeOptions.map((a) => a.id)).toContain(
|
||||
WorkflowActionType.RemoteOcr
|
||||
)
|
||||
|
||||
// Any other trigger type runs after the document has been parsed
|
||||
component.object = {
|
||||
name: 'Workflow 2',
|
||||
order: 0,
|
||||
enabled: true,
|
||||
triggers: [{ type: WorkflowTriggerType.DocumentAdded }],
|
||||
actions: [],
|
||||
} as Workflow
|
||||
component.ngOnInit()
|
||||
expect(component.actionTypeOptions.map((a) => a.id)).not.toContain(
|
||||
WorkflowActionType.RemoteOcr
|
||||
)
|
||||
})
|
||||
|
||||
it('should offer remote OCR on a trigger added to a new workflow', () => {
|
||||
jest.spyOn(settingsService, 'get').mockReturnValue(true)
|
||||
component.ngOnInit()
|
||||
|
||||
// Nothing for the action to apply to yet
|
||||
expect(component.actionTypeOptions.map((a) => a.id)).not.toContain(
|
||||
WorkflowActionType.RemoteOcr
|
||||
)
|
||||
|
||||
// addTrigger creates the form field with emitEvent false, so the options
|
||||
// have to be computed on read rather than cached from valueChanges
|
||||
component.addTrigger()
|
||||
expect(component.actionTypeOptions.map((a) => a.id)).toContain(
|
||||
WorkflowActionType.RemoteOcr
|
||||
)
|
||||
|
||||
// Switching that trigger to a type that runs after parsing removes it
|
||||
component.triggerFields
|
||||
.at(0)
|
||||
.get('type')
|
||||
.setValue(WorkflowTriggerType.DocumentAdded)
|
||||
expect(component.actionTypeOptions.map((a) => a.id)).not.toContain(
|
||||
WorkflowActionType.RemoteOcr
|
||||
)
|
||||
})
|
||||
|
||||
it('should keep remote OCR listed when an action already uses it', () => {
|
||||
jest.spyOn(settingsService, 'get').mockReturnValue(true)
|
||||
|
||||
// Otherwise changing the trigger would silently blank the selection
|
||||
component.object = {
|
||||
name: 'Workflow 1',
|
||||
order: 0,
|
||||
enabled: true,
|
||||
triggers: [{ type: WorkflowTriggerType.DocumentAdded }],
|
||||
actions: [{ type: WorkflowActionType.RemoteOcr }],
|
||||
} as Workflow
|
||||
component.ngOnInit()
|
||||
|
||||
expect(component.actionTypeOptions.map((a) => a.id)).toContain(
|
||||
WorkflowActionType.RemoteOcr
|
||||
)
|
||||
})
|
||||
|
||||
it('should not offer remote OCR when no engine is configured', () => {
|
||||
jest
|
||||
.spyOn(settingsService, 'get')
|
||||
.mockImplementation((key) => key !== SETTINGS_KEYS.REMOTE_OCR_CONFIGURED)
|
||||
|
||||
component.object = {
|
||||
name: 'Workflow 1',
|
||||
order: 0,
|
||||
enabled: true,
|
||||
triggers: [{ type: WorkflowTriggerType.Consumption }],
|
||||
actions: [],
|
||||
} as Workflow
|
||||
component.ngOnInit()
|
||||
|
||||
expect(component.actionTypeOptions.map((a) => a.id)).not.toContain(
|
||||
WorkflowActionType.RemoteOcr
|
||||
)
|
||||
})
|
||||
|
||||
it('should offer apply AI suggestions unless every trigger is consumption', () => {
|
||||
jest.spyOn(settingsService, 'get').mockReturnValue(true)
|
||||
|
||||
// Consumption runs before the document has been parsed, so there would be
|
||||
// no content to make suggestions from
|
||||
component.object = {
|
||||
name: 'Workflow 1',
|
||||
order: 0,
|
||||
enabled: true,
|
||||
triggers: [{ type: WorkflowTriggerType.Consumption }],
|
||||
actions: [],
|
||||
} as Workflow
|
||||
component.ngOnInit()
|
||||
expect(component.actionTypeOptions.map((a) => a.id)).not.toContain(
|
||||
WorkflowActionType.ApplyAiSuggestions
|
||||
)
|
||||
|
||||
// A second, usable trigger is enough
|
||||
component.object = {
|
||||
name: 'Workflow 2',
|
||||
order: 0,
|
||||
enabled: true,
|
||||
triggers: [
|
||||
{ type: WorkflowTriggerType.Consumption },
|
||||
{ type: WorkflowTriggerType.DocumentAdded },
|
||||
],
|
||||
actions: [],
|
||||
} as Workflow
|
||||
component.ngOnInit()
|
||||
expect(component.actionTypeOptions.map((a) => a.id)).toContain(
|
||||
WorkflowActionType.ApplyAiSuggestions
|
||||
)
|
||||
})
|
||||
|
||||
it('should keep apply AI suggestions listed when an action already uses it', () => {
|
||||
jest.spyOn(settingsService, 'get').mockReturnValue(true)
|
||||
|
||||
// Otherwise changing the trigger would silently blank the selection
|
||||
component.object = {
|
||||
name: 'Workflow 1',
|
||||
order: 0,
|
||||
enabled: true,
|
||||
triggers: [{ type: WorkflowTriggerType.Consumption }],
|
||||
actions: [{ type: WorkflowActionType.ApplyAiSuggestions }],
|
||||
} as Workflow
|
||||
component.ngOnInit()
|
||||
|
||||
expect(component.actionTypeOptions.map((a) => a.id)).toContain(
|
||||
WorkflowActionType.ApplyAiSuggestions
|
||||
)
|
||||
})
|
||||
|
||||
it('should not offer apply AI suggestions when AI is disabled', () => {
|
||||
jest
|
||||
.spyOn(settingsService, 'get')
|
||||
.mockImplementation((key) => key !== SETTINGS_KEYS.AI_ENABLED)
|
||||
|
||||
component.object = {
|
||||
name: 'Workflow 1',
|
||||
order: 0,
|
||||
enabled: true,
|
||||
triggers: [{ type: WorkflowTriggerType.DocumentAdded }],
|
||||
actions: [],
|
||||
} as Workflow
|
||||
component.ngOnInit()
|
||||
|
||||
expect(component.actionTypeOptions.map((a) => a.id)).not.toContain(
|
||||
WorkflowActionType.ApplyAiSuggestions
|
||||
)
|
||||
})
|
||||
|
||||
it('should create form fields for apply AI suggestions options', () => {
|
||||
component.object = {
|
||||
name: 'Workflow 1',
|
||||
order: 0,
|
||||
enabled: true,
|
||||
triggers: [{ type: WorkflowTriggerType.DocumentAdded }],
|
||||
actions: [
|
||||
{
|
||||
type: WorkflowActionType.ApplyAiSuggestions,
|
||||
ai_suggestion_fields: [
|
||||
AISuggestionField.Title,
|
||||
AISuggestionField.Tags,
|
||||
],
|
||||
ai_create_missing: true,
|
||||
ai_overwrite_existing: true,
|
||||
},
|
||||
],
|
||||
} as Workflow
|
||||
component.ngOnInit()
|
||||
|
||||
const action = component.actionFields.at(0)
|
||||
expect(action.get('ai_suggestion_fields').value).toEqual([
|
||||
AISuggestionField.Title,
|
||||
AISuggestionField.Tags,
|
||||
])
|
||||
expect(action.get('ai_create_missing').value).toBeTruthy()
|
||||
expect(action.get('ai_overwrite_existing').value).toBeTruthy()
|
||||
expect(component.aiSuggestionFieldOptions).toEqual(
|
||||
AI_SUGGESTION_FIELD_OPTIONS
|
||||
)
|
||||
})
|
||||
|
||||
it('should default apply AI suggestions options on a new action', () => {
|
||||
component.object = {
|
||||
name: 'Workflow 1',
|
||||
order: 0,
|
||||
enabled: true,
|
||||
triggers: [{ type: WorkflowTriggerType.DocumentAdded }],
|
||||
actions: [],
|
||||
} as Workflow
|
||||
component.addAction()
|
||||
|
||||
const action = component.actionFields.at(component.actionFields.length - 1)
|
||||
expect(action.get('ai_suggestion_fields').value).toEqual([])
|
||||
expect(action.get('ai_create_missing').value).toBeFalsy()
|
||||
expect(action.get('ai_overwrite_existing').value).toBeFalsy()
|
||||
})
|
||||
|
||||
it('should support add and remove triggers and actions', () => {
|
||||
component.object = workflow
|
||||
component.addTrigger()
|
||||
|
||||
+10
-102
@@ -30,7 +30,6 @@ import { StoragePath } from 'src/app/data/storage-path'
|
||||
import { SETTINGS_KEYS } from 'src/app/data/ui-settings'
|
||||
import { Workflow } from 'src/app/data/workflow'
|
||||
import {
|
||||
AISuggestionField,
|
||||
WorkflowAction,
|
||||
WorkflowActionType,
|
||||
} from 'src/app/data/workflow-action'
|
||||
@@ -149,41 +148,6 @@ export const WORKFLOW_ACTION_OPTIONS = [
|
||||
id: WorkflowActionType.MoveToTrash,
|
||||
name: $localize`Move to trash`,
|
||||
},
|
||||
{
|
||||
id: WorkflowActionType.RemoteOcr,
|
||||
name: $localize`Remote OCR`,
|
||||
},
|
||||
{
|
||||
id: WorkflowActionType.ApplyAiSuggestions,
|
||||
name: $localize`Apply AI suggestions`,
|
||||
},
|
||||
]
|
||||
|
||||
export const AI_SUGGESTION_FIELD_OPTIONS = [
|
||||
{
|
||||
id: AISuggestionField.Title,
|
||||
name: $localize`Title`,
|
||||
},
|
||||
{
|
||||
id: AISuggestionField.Tags,
|
||||
name: $localize`Tags`,
|
||||
},
|
||||
{
|
||||
id: AISuggestionField.Correspondent,
|
||||
name: $localize`Correspondent`,
|
||||
},
|
||||
{
|
||||
id: AISuggestionField.DocumentType,
|
||||
name: $localize`Document type`,
|
||||
},
|
||||
{
|
||||
id: AISuggestionField.StoragePath,
|
||||
name: $localize`Storage path`,
|
||||
},
|
||||
{
|
||||
id: AISuggestionField.Created,
|
||||
name: $localize`Created date`,
|
||||
},
|
||||
]
|
||||
|
||||
export enum TriggerFilterType {
|
||||
@@ -540,6 +504,8 @@ export class WorkflowEditDialogComponent
|
||||
|
||||
expandedItem: number = null
|
||||
|
||||
readonly allowedActionTypes = signal([])
|
||||
|
||||
private readonly triggerFilterOptionsMap = new WeakMap<
|
||||
FormArray,
|
||||
TriggerFilterOption[]
|
||||
@@ -582,58 +548,13 @@ export class WorkflowEditDialogComponent
|
||||
this.checkRemovalActionFields.bind(this)
|
||||
)
|
||||
this.checkRemovalActionFields(this.objectForm.value)
|
||||
}
|
||||
|
||||
private allowedActionTypes: typeof WORKFLOW_ACTION_OPTIONS = null
|
||||
|
||||
private getAllowedActionTypes() {
|
||||
let allowed = WORKFLOW_ACTION_OPTIONS
|
||||
|
||||
if (!this.settingsService.get(SETTINGS_KEYS.EMAIL_ENABLED)) {
|
||||
allowed = allowed.filter((a) => a.id !== WorkflowActionType.Email)
|
||||
}
|
||||
|
||||
// Remote OCR is decided before the document is parsed, so it is only
|
||||
// offered for workflows that run at consumption.
|
||||
const formWorkflow: Workflow = this.objectForm?.value
|
||||
const remoteOcrUsable =
|
||||
this.settingsService.get(SETTINGS_KEYS.REMOTE_OCR_CONFIGURED) &&
|
||||
(formWorkflow?.triggers?.some(
|
||||
(trigger) => trigger.type === WorkflowTriggerType.Consumption
|
||||
) ||
|
||||
formWorkflow?.actions?.some(
|
||||
(action) => action.type === WorkflowActionType.RemoteOcr
|
||||
))
|
||||
if (!remoteOcrUsable) {
|
||||
allowed = allowed.filter((a) => a.id !== WorkflowActionType.RemoteOcr)
|
||||
}
|
||||
|
||||
// Only available after consumption. Unlike remote OCR this is hidden only
|
||||
// once every trigger is consumption, so it stays offered on a workflow
|
||||
// that has no triggers yet.
|
||||
const aiSuggestionsUsable =
|
||||
this.settingsService.get(SETTINGS_KEYS.AI_ENABLED) &&
|
||||
(!formWorkflow?.triggers?.length ||
|
||||
formWorkflow.triggers.some(
|
||||
(trigger) => trigger.type !== WorkflowTriggerType.Consumption
|
||||
) ||
|
||||
formWorkflow.actions?.some(
|
||||
(action) => action.type === WorkflowActionType.ApplyAiSuggestions
|
||||
))
|
||||
if (!aiSuggestionsUsable) {
|
||||
allowed = allowed.filter(
|
||||
(a) => a.id !== WorkflowActionType.ApplyAiSuggestions
|
||||
)
|
||||
}
|
||||
|
||||
if (
|
||||
this.allowedActionTypes?.length === allowed.length &&
|
||||
this.allowedActionTypes.every((a, i) => a.id === allowed[i].id)
|
||||
) {
|
||||
return this.allowedActionTypes
|
||||
}
|
||||
this.allowedActionTypes = allowed
|
||||
return allowed
|
||||
this.allowedActionTypes.set(
|
||||
this.settingsService.get(SETTINGS_KEYS.EMAIL_ENABLED)
|
||||
? WORKFLOW_ACTION_OPTIONS
|
||||
: WORKFLOW_ACTION_OPTIONS.filter(
|
||||
(a) => a.id !== WorkflowActionType.Email
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
private checkRemovalActionFields(formWorkflow: Workflow) {
|
||||
@@ -1277,11 +1198,6 @@ export class WorkflowEditDialogComponent
|
||||
passwords: new FormControl(
|
||||
this.formatPasswords(action.passwords ?? [])
|
||||
),
|
||||
ai_suggestion_fields: new FormControl(
|
||||
action.ai_suggestion_fields ?? []
|
||||
),
|
||||
ai_create_missing: new FormControl(!!action.ai_create_missing),
|
||||
ai_overwrite_existing: new FormControl(!!action.ai_overwrite_existing),
|
||||
}),
|
||||
{ emitEvent }
|
||||
)
|
||||
@@ -1363,18 +1279,13 @@ export class WorkflowEditDialogComponent
|
||||
|
||||
get actionTypeOptions() {
|
||||
this.settingsService.trackChanges()
|
||||
// Computed on read rather than cached
|
||||
return this.getAllowedActionTypes()
|
||||
return this.allowedActionTypes()
|
||||
}
|
||||
|
||||
getActionTypeOptionName(type: WorkflowActionType): string {
|
||||
return this.actionTypeOptions.find((t) => t.id === type)?.name ?? ''
|
||||
}
|
||||
|
||||
get aiSuggestionFieldOptions() {
|
||||
return AI_SUGGESTION_FIELD_OPTIONS
|
||||
}
|
||||
|
||||
addAction() {
|
||||
if (!this.object) {
|
||||
this.object = Object.assign({}, this.objectForm.value)
|
||||
@@ -1428,9 +1339,6 @@ export class WorkflowEditDialogComponent
|
||||
include_document: false,
|
||||
},
|
||||
passwords: [],
|
||||
ai_suggestion_fields: [],
|
||||
ai_create_missing: false,
|
||||
ai_overwrite_existing: false,
|
||||
}
|
||||
this.object.actions.push(action)
|
||||
this.createActionField(action)
|
||||
|
||||
@@ -36,7 +36,16 @@
|
||||
(focus)="clearLastSearchTerm()"
|
||||
(clear)="clearLastSearchTerm()"
|
||||
(blur)="onBlur()">
|
||||
<ng-template ng-label-tmp let-item="item">
|
||||
@if (iconField && item[iconField]) {
|
||||
<i-bs class="me-2" [name]="item[iconField]"></i-bs>
|
||||
}
|
||||
<span [title]="item[bindLabel]">{{item[bindLabel]}}</span>
|
||||
</ng-template>
|
||||
<ng-template ng-option-tmp let-item="item">
|
||||
@if (iconField && item[iconField]) {
|
||||
<i-bs class="me-2" [name]="item[iconField]"></i-bs>
|
||||
}
|
||||
<span [title]="item[bindLabel]">{{item[bindLabel]}}</span>
|
||||
</ng-template>
|
||||
</ng-select>
|
||||
|
||||
@@ -35,7 +35,7 @@ import { AbstractInputComponent } from '../abstract-input'
|
||||
NgxBootstrapIconsModule,
|
||||
],
|
||||
})
|
||||
export class SelectComponent extends AbstractInputComponent<number> {
|
||||
export class SelectComponent extends AbstractInputComponent<number | string> {
|
||||
constructor() {
|
||||
super()
|
||||
this.addItemRef = this.addItem.bind(this)
|
||||
@@ -100,6 +100,9 @@ export class SelectComponent extends AbstractInputComponent<number> {
|
||||
@Input()
|
||||
bindLabel: string = 'name'
|
||||
|
||||
@Input()
|
||||
iconField: string
|
||||
|
||||
public searchFn = (term: string, item: any): boolean =>
|
||||
matchesSearchText(item?.[this.bindLabel], term)
|
||||
|
||||
|
||||
@@ -151,13 +151,6 @@
|
||||
inset: 0;
|
||||
pointer-events: none;
|
||||
|
||||
& section {
|
||||
position: absolute;
|
||||
text-align: initial;
|
||||
box-sizing: border-box;
|
||||
transform-origin: 0 0;
|
||||
}
|
||||
|
||||
& .annotationTextContent {
|
||||
opacity: 0;
|
||||
}
|
||||
|
||||
@@ -13,7 +13,6 @@ import {
|
||||
ViewChild,
|
||||
} from '@angular/core'
|
||||
import {
|
||||
AnnotationMode,
|
||||
getDocument,
|
||||
GlobalWorkerOptions,
|
||||
PDFDocumentLoadingTask,
|
||||
@@ -222,7 +221,6 @@ export class PngxPdfViewerComponent
|
||||
linkService: this.linkService,
|
||||
findController: this.findController,
|
||||
textLayerMode,
|
||||
annotationMode: AnnotationMode.ENABLE,
|
||||
enableSelectionRendering: false,
|
||||
removePageBorders: true,
|
||||
}
|
||||
|
||||
+2
-8
@@ -2,16 +2,10 @@
|
||||
<button type="button" class="btn btn-sm btn-outline-primary" (click)="clickSuggest()" [disabled]="disabled() || loading() || (suggestions() && !aiEnabled())">
|
||||
@if (loading()) {
|
||||
<div class="spinner-border spinner-border-sm" role="status"></div>
|
||||
} @else if (noSuggestions) {
|
||||
<i-bs width="1.2em" height="1.2em" name="check-circle"></i-bs>
|
||||
} @else {
|
||||
<i-bs width="1.2em" height="1.2em" name="stars"></i-bs>
|
||||
}
|
||||
@if (noSuggestions) {
|
||||
<span class="d-none d-lg-inline ps-1" i18n>No suggestions</span>
|
||||
} @else {
|
||||
<span class="d-none d-lg-inline ps-1" i18n>Suggest</span>
|
||||
}
|
||||
<span class="d-none d-lg-inline ps-1" i18n>Suggest</span>
|
||||
@if (totalSuggestions > 0) {
|
||||
<span class="badge bg-primary ms-2">{{ totalSuggestions }}</span>
|
||||
}
|
||||
@@ -25,7 +19,7 @@
|
||||
|
||||
<div ngbDropdownMenu aria-labelledby="suggestionsDropdown" class="shadow suggestions-dropdown">
|
||||
<div class="list-group list-group-flush small pb-0">
|
||||
@if (totalSuggestions === 0) {
|
||||
@if (!suggestions()?.suggested_tags && !suggestions()?.suggested_document_types && !suggestions()?.suggested_correspondents) {
|
||||
<div class="list-group-item text-muted fst-italic">
|
||||
<small class="text-muted small fst-italic" i18n>No novel suggestions</small>
|
||||
</div>
|
||||
|
||||
-29
@@ -30,34 +30,6 @@ describe('SuggestionsDropdownComponent', () => {
|
||||
expect(component.totalSuggestions).toBe(4)
|
||||
})
|
||||
|
||||
it('should show when a completed request returned no suggestions', () => {
|
||||
fixture.componentRef.setInput('suggestions', {
|
||||
correspondents: [],
|
||||
tags: [],
|
||||
document_types: [],
|
||||
storage_paths: [],
|
||||
dates: [],
|
||||
})
|
||||
fixture.detectChanges()
|
||||
|
||||
expect(component.noSuggestions).toBeTruthy()
|
||||
expect(fixture.nativeElement.textContent).toContain('No suggestions')
|
||||
})
|
||||
|
||||
it('should not show the empty state before a request or with suggestions', () => {
|
||||
expect(component.noSuggestions).toBeFalsy()
|
||||
|
||||
fixture.componentRef.setInput('suggestions', {
|
||||
correspondents: [],
|
||||
tags: [42],
|
||||
document_types: [],
|
||||
storage_paths: [],
|
||||
dates: [],
|
||||
})
|
||||
|
||||
expect(component.noSuggestions).toBeFalsy()
|
||||
})
|
||||
|
||||
it('should emit getSuggestions when clickSuggest is called and suggestions are null', () => {
|
||||
jest.spyOn(component.getSuggestions, 'emit')
|
||||
fixture.componentRef.setInput('suggestions', null)
|
||||
@@ -87,6 +59,5 @@ describe('SuggestionsDropdownComponent', () => {
|
||||
})
|
||||
component.clickSuggest()
|
||||
expect(component.dropdown.open).toBeTruthy()
|
||||
expect(fixture.nativeElement.textContent).toContain('No novel suggestions')
|
||||
})
|
||||
})
|
||||
|
||||
-17
@@ -61,21 +61,4 @@ export class SuggestionsDropdownComponent {
|
||||
this.suggestions()?.suggested_document_types?.length || 0
|
||||
)
|
||||
}
|
||||
|
||||
get noSuggestions(): boolean {
|
||||
const suggestions = this.suggestions()
|
||||
return (
|
||||
suggestions != null &&
|
||||
!suggestions.title &&
|
||||
!suggestions.tags?.length &&
|
||||
!suggestions.suggested_tags?.length &&
|
||||
!suggestions.correspondents?.length &&
|
||||
!suggestions.suggested_correspondents?.length &&
|
||||
!suggestions.document_types?.length &&
|
||||
!suggestions.suggested_document_types?.length &&
|
||||
!suggestions.storage_paths?.length &&
|
||||
!suggestions.suggested_storage_paths?.length &&
|
||||
!suggestions.dates?.length
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
+1
@@ -1,6 +1,7 @@
|
||||
<pngx-widget-frame
|
||||
*pngxIfPermissions="{ action: PermissionAction.View, type: PermissionType.Document }"
|
||||
[title]="savedView.name"
|
||||
[titleIcon]="savedView.icon || 'funnel'"
|
||||
[loading]="false"
|
||||
[draggable]="savedView"
|
||||
>
|
||||
|
||||
+6
-1
@@ -8,7 +8,12 @@
|
||||
<i-bs name="grip-vertical"></i-bs>
|
||||
</div>
|
||||
}
|
||||
<h6 class="card-title mb-0">{{title()}}</h6>
|
||||
<h6 class="card-title mb-0">
|
||||
@if (titleIcon()) {
|
||||
<i-bs class="me-2" [name]="titleIcon()"></i-bs>
|
||||
}
|
||||
{{title()}}
|
||||
</h6>
|
||||
<ng-content select="[title-badge]"></ng-content>
|
||||
@if (badge() !== null && badge() !== undefined) {
|
||||
<span class="badge bg-info text-dark ms-2">{{badge()}}</span>
|
||||
|
||||
@@ -16,6 +16,8 @@ export class WidgetFrameComponent implements AfterViewInit {
|
||||
|
||||
title = input<string>()
|
||||
|
||||
titleIcon = input<string>()
|
||||
|
||||
draggable = input<any>()
|
||||
|
||||
cardless = input(false)
|
||||
|
||||
@@ -963,24 +963,12 @@ describe('DocumentDetailComponent', () => {
|
||||
component.reprocess()
|
||||
const modalCloseSpy = jest.spyOn(openModal, 'close')
|
||||
openModal.componentInstance.confirmClicked.next()
|
||||
expect(reprocessSpy).toHaveBeenCalledWith({ documents: [doc.id] }, false)
|
||||
expect(reprocessSpy).toHaveBeenCalledWith({ documents: [doc.id] })
|
||||
expect(modalSpy).toHaveBeenCalled()
|
||||
expect(toastSpy).toHaveBeenCalled()
|
||||
expect(modalCloseSpy).toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('should pass remote OCR choice when reprocessing', () => {
|
||||
initNormally()
|
||||
const reprocessSpy = jest.spyOn(documentService, 'reprocessDocuments')
|
||||
reprocessSpy.mockReturnValue(of(true))
|
||||
let openModal: NgbModalRef
|
||||
modalService.activeInstances.subscribe((modal) => (openModal = modal[0]))
|
||||
component.reprocess()
|
||||
openModal.componentInstance.remoteOcr = true
|
||||
openModal.componentInstance.confirmClicked.next()
|
||||
expect(reprocessSpy).toHaveBeenCalledWith({ documents: [doc.id] }, true)
|
||||
})
|
||||
|
||||
it('should show error if redo ocr call fails', () => {
|
||||
initNormally()
|
||||
const reprocessSpy = jest.spyOn(documentService, 'reprocessDocuments')
|
||||
@@ -2173,14 +2161,8 @@ describe('DocumentDetailComponent', () => {
|
||||
it('should support open share links and email modals', () => {
|
||||
const modalSpy = jest.spyOn(modalService, 'open')
|
||||
initNormally()
|
||||
component.selectedVersionId.set(10)
|
||||
component.openShareLinks()
|
||||
expect(modalSpy).toHaveBeenCalled()
|
||||
expect(
|
||||
(
|
||||
modalSpy.mock.results[0].value as NgbModalRef
|
||||
).componentInstance.documentId()
|
||||
).toBe(10)
|
||||
component.openEmailDocument()
|
||||
expect(modalSpy).toHaveBeenCalled()
|
||||
})
|
||||
|
||||
@@ -97,7 +97,6 @@ import { ISODateAdapter } from 'src/app/utils/ngb-iso-date-adapter'
|
||||
import * as UTIF from 'utif'
|
||||
import { DocumentDetailFieldID } from '../admin/settings/settings.component'
|
||||
import { ConfirmDialogComponent } from '../common/confirm-dialog/confirm-dialog.component'
|
||||
import { ReprocessConfirmDialogComponent } from '../common/confirm-dialog/reprocess-confirm-dialog/reprocess-confirm-dialog.component'
|
||||
import { PasswordRemovalConfirmDialogComponent } from '../common/confirm-dialog/password-removal-confirm-dialog/password-removal-confirm-dialog.component'
|
||||
import { CustomFieldsDropdownComponent } from '../common/custom-fields-dropdown/custom-fields-dropdown.component'
|
||||
import { CorrespondentEditDialogComponent } from '../common/edit-dialog/correspondent-edit-dialog/correspondent-edit-dialog.component'
|
||||
@@ -1403,7 +1402,7 @@ export class DocumentDetailComponent
|
||||
}
|
||||
|
||||
reprocess() {
|
||||
let modal = this.modalService.open(ReprocessConfirmDialogComponent, {
|
||||
let modal = this.modalService.open(ConfirmDialogComponent, {
|
||||
backdrop: 'static',
|
||||
})
|
||||
modal.componentInstance.title = $localize`Reprocess confirm`
|
||||
@@ -1414,10 +1413,7 @@ export class DocumentDetailComponent
|
||||
modal.componentInstance.confirmClicked.subscribe(() => {
|
||||
modal.componentInstance.buttonsEnabled = false
|
||||
this.documentsService
|
||||
.reprocessDocuments(
|
||||
{ documents: [this.document().id] },
|
||||
modal.componentInstance.remoteOcr
|
||||
)
|
||||
.reprocessDocuments({ documents: [this.document().id] })
|
||||
.subscribe({
|
||||
next: () => {
|
||||
this.toastService.showInfo(
|
||||
@@ -1963,9 +1959,7 @@ export class DocumentDetailComponent
|
||||
|
||||
public openShareLinks() {
|
||||
const modal = this.modalService.open(ShareLinksDialogComponent)
|
||||
modal.componentInstance.documentId.set(
|
||||
this.selectedVersionId() ?? this.document().id
|
||||
)
|
||||
modal.componentInstance.documentId.set(this.document().id)
|
||||
modal.componentInstance.hasArchiveVersion.set(
|
||||
this.metadata()?.has_archive_version ??
|
||||
!!this.document()?.archived_file_name
|
||||
|
||||
@@ -1122,7 +1122,6 @@ describe('BulkEditorComponent', () => {
|
||||
req.flush(true)
|
||||
expect(req.request.body).toEqual({
|
||||
documents: [3, 4],
|
||||
remote_ocr: false,
|
||||
})
|
||||
httpTestingController.match(
|
||||
`${environment.apiBaseUrl}documents/?page=1&page_size=50&ordering=-created&truncate_content=true&include_selection_data=true`
|
||||
|
||||
@@ -51,7 +51,6 @@ import { ToastService } from 'src/app/services/toast.service'
|
||||
import { flattenTags } from 'src/app/utils/flatten-tags'
|
||||
import { queryParamsFromFilterRules } from 'src/app/utils/query-params'
|
||||
import { MergeConfirmDialogComponent } from '../../common/confirm-dialog/merge-confirm-dialog/merge-confirm-dialog.component'
|
||||
import { ReprocessConfirmDialogComponent } from '../../common/confirm-dialog/reprocess-confirm-dialog/reprocess-confirm-dialog.component'
|
||||
import { RotateConfirmDialogComponent } from '../../common/confirm-dialog/rotate-confirm-dialog/rotate-confirm-dialog.component'
|
||||
import { CorrespondentEditDialogComponent } from '../../common/edit-dialog/correspondent-edit-dialog/correspondent-edit-dialog.component'
|
||||
import { CustomFieldEditDialogComponent } from '../../common/edit-dialog/custom-field-edit-dialog/custom-field-edit-dialog.component'
|
||||
@@ -910,7 +909,7 @@ export class BulkEditorComponent
|
||||
}
|
||||
|
||||
reprocessSelected() {
|
||||
let modal = this.modalService.open(ReprocessConfirmDialogComponent, {
|
||||
let modal = this.modalService.open(ConfirmDialogComponent, {
|
||||
backdrop: 'static',
|
||||
})
|
||||
modal.componentInstance.title = $localize`Reprocess confirm`
|
||||
@@ -924,10 +923,7 @@ export class BulkEditorComponent
|
||||
modal.componentInstance.buttonsEnabled = false
|
||||
this.executeDocumentAction(
|
||||
modal,
|
||||
this.documentService.reprocessDocuments(
|
||||
this.getSelectionQuery(),
|
||||
modal.componentInstance.remoteOcr
|
||||
)
|
||||
this.documentService.reprocessDocuments(this.getSelectionQuery())
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -97,7 +97,9 @@
|
||||
<div class="dropdown-menu shadow dropdown-menu-right" ngbDropdownMenu>
|
||||
@if (!list.activeSavedViewId) {
|
||||
@for (view of savedViewService.allViews; track view) {
|
||||
<button ngbDropdownItem (click)="loadViewConfig(view.id)">{{view.name}}</button>
|
||||
<button ngbDropdownItem (click)="loadViewConfig(view.id)">
|
||||
<i-bs class="me-2" [name]="view.icon || 'funnel'"></i-bs>{{view.name}}
|
||||
</button>
|
||||
}
|
||||
@if (savedViewService.allViews.length > 0) {
|
||||
<div class="dropdown-divider"></div>
|
||||
|
||||
@@ -457,6 +457,7 @@ export class DocumentListComponent
|
||||
modal.componentInstance.buttonsEnabled.set(false)
|
||||
let savedView: SavedView = {
|
||||
name: formValue.name,
|
||||
icon: formValue.icon,
|
||||
filter_rules: this.list.filterRules,
|
||||
sort_reverse: this.list.sortReverse,
|
||||
sort_field: this.list.sortField,
|
||||
|
||||
@@ -2213,20 +2213,6 @@ describe('FilterEditorComponent', () => {
|
||||
expect(blurSpy).toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('should only dismiss open autocomplete suggestions on Escape, keeping the query', () => {
|
||||
component.textFilter = 'foo bar'
|
||||
component.textFilterInput.nativeElement.value = 'foo bar'
|
||||
jest.spyOn(component.searchTypeahead, 'isPopupOpen').mockReturnValue(true)
|
||||
const dismissSpy = jest
|
||||
.spyOn(component.searchTypeahead, 'dismissPopup')
|
||||
.mockImplementation(() => {})
|
||||
component.textFilterInput.nativeElement.dispatchEvent(
|
||||
new KeyboardEvent('keydown', { key: 'Escape' })
|
||||
)
|
||||
expect(dismissSpy).toHaveBeenCalled()
|
||||
expect(component.textFilter).toEqual('foo bar')
|
||||
})
|
||||
|
||||
it('should adjust text filter targets if more like search', () => {
|
||||
const TEXT_FILTER_TARGET_FULLTEXT_MORELIKE = 'fulltext-morelike' // private const
|
||||
component.textFilterTarget = TEXT_FILTER_TARGET_FULLTEXT_MORELIKE
|
||||
|
||||
@@ -15,7 +15,6 @@ import {
|
||||
import { FormsModule, ReactiveFormsModule } from '@angular/forms'
|
||||
import {
|
||||
NgbDropdownModule,
|
||||
NgbTypeahead,
|
||||
NgbTypeaheadModule,
|
||||
} from '@ng-bootstrap/ng-bootstrap'
|
||||
import { NgxBootstrapIconsModule } from 'ngx-bootstrap-icons'
|
||||
@@ -352,9 +351,6 @@ export class FilterEditorComponent
|
||||
@ViewChild('textFilterInput')
|
||||
textFilterInput: ElementRef
|
||||
|
||||
@ViewChild(NgbTypeahead)
|
||||
searchTypeahead: NgbTypeahead
|
||||
|
||||
readonly customFields = signal<CustomField[]>([])
|
||||
|
||||
tagDocumentCounts: SelectionDataItem[]
|
||||
@@ -1154,7 +1150,6 @@ export class FilterEditorComponent
|
||||
}
|
||||
|
||||
set textFilter(value) {
|
||||
this._textFilter = value // set immediately to prevent loss of keystrokes
|
||||
this.textFilterDebounce.next(value)
|
||||
}
|
||||
|
||||
@@ -1247,9 +1242,9 @@ export class FilterEditorComponent
|
||||
distinctUntilChanged(),
|
||||
filter((query) => !query.length || query.length > 2)
|
||||
)
|
||||
.subscribe(() =>
|
||||
.subscribe((text) =>
|
||||
this.updateTextFilter(
|
||||
this._textFilter, // use the current value, not the debounced (possibly stale) one
|
||||
text,
|
||||
this.textFilterTarget !== TEXT_FILTER_TARGET_FULLTEXT_QUERY
|
||||
)
|
||||
)
|
||||
@@ -1325,11 +1320,6 @@ export class FilterEditorComponent
|
||||
this.updateTextFilter(filterString)
|
||||
}
|
||||
} else if (event.key === 'Escape') {
|
||||
if (this.searchTypeahead?.isPopupOpen()) {
|
||||
// only dismiss the suggestions, so longer query can use Enter
|
||||
this.searchTypeahead.dismissPopup()
|
||||
return
|
||||
}
|
||||
if (this._textFilter?.length) {
|
||||
this.resetTextField()
|
||||
} else {
|
||||
|
||||
+8
@@ -6,6 +6,14 @@
|
||||
</div>
|
||||
<div class="modal-body">
|
||||
<pngx-input-text i18n-title title="Name" formControlName="name" [error]="error()?.name" autocomplete="off"></pngx-input-text>
|
||||
<pngx-input-select
|
||||
i18n-title
|
||||
title="Icon"
|
||||
formControlName="icon"
|
||||
[items]="savedViewIcons"
|
||||
iconField="icon"
|
||||
[error]="error()?.icon">
|
||||
</pngx-input-select>
|
||||
<pngx-input-check i18n-title title="Show in sidebar" formControlName="showInSideBar"></pngx-input-check>
|
||||
<pngx-input-check i18n-title title="Show on dashboard" formControlName="showOnDashboard"></pngx-input-check>
|
||||
<pngx-permissions-form accordion="true" formControlName="permissions_form"></pngx-permissions-form>
|
||||
|
||||
+5
@@ -9,6 +9,7 @@ import { CheckComponent } from '../../common/input/check/check.component'
|
||||
import { PermissionsFormComponent } from '../../common/input/permissions/permissions-form/permissions-form.component'
|
||||
import { PermissionsGroupComponent } from '../../common/input/permissions/permissions-group/permissions-group.component'
|
||||
import { PermissionsUserComponent } from '../../common/input/permissions/permissions-user/permissions-user.component'
|
||||
import { SelectComponent } from '../../common/input/select/select.component'
|
||||
import { TextComponent } from '../../common/input/text/text.component'
|
||||
import { SaveViewConfigDialogComponent } from './save-view-config-dialog.component'
|
||||
|
||||
@@ -40,6 +41,7 @@ describe('SaveViewConfigDialogComponent', () => {
|
||||
ReactiveFormsModule,
|
||||
SaveViewConfigDialogComponent,
|
||||
TextComponent,
|
||||
SelectComponent,
|
||||
CheckComponent,
|
||||
PermissionsFormComponent,
|
||||
PermissionsUserComponent,
|
||||
@@ -63,6 +65,7 @@ describe('SaveViewConfigDialogComponent', () => {
|
||||
expect(component.defaultName()).toEqual(name)
|
||||
expect(result).toEqual({
|
||||
name,
|
||||
icon: 'funnel',
|
||||
showInSideBar: false,
|
||||
showOnDashboard: false,
|
||||
})
|
||||
@@ -94,6 +97,7 @@ describe('SaveViewConfigDialogComponent', () => {
|
||||
component.save()
|
||||
expect(result).toEqual({
|
||||
name,
|
||||
icon: 'funnel',
|
||||
showInSideBar: true,
|
||||
showOnDashboard: true,
|
||||
})
|
||||
@@ -113,6 +117,7 @@ describe('SaveViewConfigDialogComponent', () => {
|
||||
component.save()
|
||||
expect(result).toEqual({
|
||||
name: '',
|
||||
icon: 'funnel',
|
||||
showInSideBar: false,
|
||||
showOnDashboard: false,
|
||||
permissions_form: permissions,
|
||||
|
||||
+9
@@ -13,9 +13,14 @@ import {
|
||||
ReactiveFormsModule,
|
||||
} from '@angular/forms'
|
||||
import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap'
|
||||
import {
|
||||
DEFAULT_SAVED_VIEW_ICON,
|
||||
SAVED_VIEW_ICONS,
|
||||
} from 'src/app/data/saved-view-icons'
|
||||
import { User } from 'src/app/data/user'
|
||||
import { CheckComponent } from '../../common/input/check/check.component'
|
||||
import { PermissionsFormComponent } from '../../common/input/permissions/permissions-form/permissions-form.component'
|
||||
import { SelectComponent } from '../../common/input/select/select.component'
|
||||
import { TextComponent } from '../../common/input/text/text.component'
|
||||
|
||||
@Component({
|
||||
@@ -24,6 +29,7 @@ import { TextComponent } from '../../common/input/text/text.component'
|
||||
styleUrls: ['./save-view-config-dialog.component.scss'],
|
||||
imports: [
|
||||
CheckComponent,
|
||||
SelectComponent,
|
||||
TextComponent,
|
||||
PermissionsFormComponent,
|
||||
FormsModule,
|
||||
@@ -41,6 +47,7 @@ export class SaveViewConfigDialogComponent implements OnInit {
|
||||
public saveClicked = new EventEmitter()
|
||||
|
||||
users: User[]
|
||||
readonly savedViewIcons = SAVED_VIEW_ICONS
|
||||
|
||||
setDefaultName(value: string) {
|
||||
this.defaultName.set(value)
|
||||
@@ -49,6 +56,7 @@ export class SaveViewConfigDialogComponent implements OnInit {
|
||||
|
||||
saveViewConfigForm = new FormGroup({
|
||||
name: new FormControl(''),
|
||||
icon: new FormControl(DEFAULT_SAVED_VIEW_ICON),
|
||||
showInSideBar: new FormControl(false),
|
||||
showOnDashboard: new FormControl(false),
|
||||
permissions_form: new FormControl(null),
|
||||
@@ -65,6 +73,7 @@ export class SaveViewConfigDialogComponent implements OnInit {
|
||||
const formValue = this.saveViewConfigForm.value
|
||||
const saveViewConfig = {
|
||||
name: formValue.name,
|
||||
icon: formValue.icon,
|
||||
showInSideBar: formValue.showInSideBar,
|
||||
showOnDashboard: formValue.showOnDashboard,
|
||||
}
|
||||
|
||||
+1
-1
@@ -88,7 +88,7 @@
|
||||
@if (depth > 0) {
|
||||
<div class="indicator"></div>
|
||||
}
|
||||
<button class="btn btn-link ms-0 ps-0 text-start" style="user-select: text;" [disabled]="!userCanEdit(object)" (click)="userCanEdit(object) ? openEditDialog(object) : null; $event.stopPropagation()">{{ object.name }}</button>
|
||||
<button class="btn btn-link ms-0 ps-0 text-start" (click)="userCanEdit(object) ? openEditDialog(object) : null; $event.stopPropagation()">{{ object.name }}</button>
|
||||
</td>
|
||||
<td class="d-none d-sm-table-cell">{{ getMatching(object) }}</td>
|
||||
<td>{{ getDocumentCount(object) }}</td>
|
||||
|
||||
@@ -11,11 +11,20 @@
|
||||
<li class="list-group-item py-3">
|
||||
<div [formGroupName]="view.id">
|
||||
<div class="row">
|
||||
<div class="col">
|
||||
<div class="col-md">
|
||||
<pngx-input-text title="Name" formControlName="name"></pngx-input-text>
|
||||
</div>
|
||||
<div class="col-md">
|
||||
<pngx-input-select
|
||||
i18n-title
|
||||
title="Icon"
|
||||
formControlName="icon"
|
||||
[items]="savedViewIcons"
|
||||
iconField="icon">
|
||||
</pngx-input-select>
|
||||
</div>
|
||||
@if (canSaveSettings) {
|
||||
<div class="col">
|
||||
<div class="col-md">
|
||||
<div class="form-check form-switch mt-3">
|
||||
<input type="checkbox" class="form-check-input" id="show_on_dashboard_{{view.id}}" formControlName="show_on_dashboard">
|
||||
<label class="form-check-label" for="show_on_dashboard_{{view.id}}" i18n>Show on dashboard</label>
|
||||
|
||||
@@ -25,8 +25,20 @@ import { PageHeaderComponent } from '../../common/page-header/page-header.compon
|
||||
import { SavedViewsComponent } from './saved-views.component'
|
||||
|
||||
const savedViews = [
|
||||
{ id: 1, name: 'view1', show_in_sidebar: true, show_on_dashboard: true },
|
||||
{ id: 2, name: 'view2', show_in_sidebar: false, show_on_dashboard: false },
|
||||
{
|
||||
id: 1,
|
||||
name: 'view1',
|
||||
icon: 'archive',
|
||||
show_in_sidebar: true,
|
||||
show_on_dashboard: true,
|
||||
},
|
||||
{
|
||||
id: 2,
|
||||
name: 'view2',
|
||||
icon: 'funnel',
|
||||
show_in_sidebar: false,
|
||||
show_on_dashboard: false,
|
||||
},
|
||||
]
|
||||
|
||||
describe('SavedViewsComponent', () => {
|
||||
@@ -157,6 +169,24 @@ describe('SavedViewsComponent', () => {
|
||||
expect(patchBody.show_in_sidebar).toBeUndefined()
|
||||
})
|
||||
|
||||
it('should persist a changed icon', () => {
|
||||
const patchSpy = jest.spyOn(savedViewService, 'patchMany')
|
||||
const view = savedViews[0]
|
||||
const iconControl = component.savedViewsForm
|
||||
.get('savedViews')
|
||||
.get(view.id.toString())
|
||||
.get('icon')
|
||||
|
||||
iconControl.setValue('bell')
|
||||
iconControl.markAsDirty()
|
||||
component.save()
|
||||
|
||||
expect(patchSpy.mock.calls[0][0][0]).toMatchObject({
|
||||
id: view.id,
|
||||
icon: 'bell',
|
||||
})
|
||||
})
|
||||
|
||||
it('should persist visibility changes to user settings', () => {
|
||||
const patchSpy = jest.spyOn(savedViewService, 'patchMany')
|
||||
const updateVisibilitySpy = jest
|
||||
|
||||
@@ -13,6 +13,10 @@ import { BehaviorSubject, Observable, of, switchMap, takeUntil } from 'rxjs'
|
||||
import { PermissionsDialogComponent } from 'src/app/components/common/permissions-dialog/permissions-dialog.component'
|
||||
import { DisplayMode } from 'src/app/data/document'
|
||||
import { SavedView } from 'src/app/data/saved-view'
|
||||
import {
|
||||
DEFAULT_SAVED_VIEW_ICON,
|
||||
SAVED_VIEW_ICONS,
|
||||
} from 'src/app/data/saved-view-icons'
|
||||
import { IfPermissionsDirective } from 'src/app/directives/if-permissions.directive'
|
||||
import {
|
||||
PermissionAction,
|
||||
@@ -25,6 +29,7 @@ import { ToastService } from 'src/app/services/toast.service'
|
||||
import { ConfirmButtonComponent } from '../../common/confirm-button/confirm-button.component'
|
||||
import { DragDropSelectComponent } from '../../common/input/drag-drop-select/drag-drop-select.component'
|
||||
import { NumberComponent } from '../../common/input/number/number.component'
|
||||
import { SelectComponent } from '../../common/input/select/select.component'
|
||||
import { TextComponent } from '../../common/input/text/text.component'
|
||||
import { PageHeaderComponent } from '../../common/page-header/page-header.component'
|
||||
import { LoadingComponentWithPermissions } from '../../loading-component/loading.component'
|
||||
@@ -36,6 +41,7 @@ import { LoadingComponentWithPermissions } from '../../loading-component/loading
|
||||
PageHeaderComponent,
|
||||
ConfirmButtonComponent,
|
||||
NumberComponent,
|
||||
SelectComponent,
|
||||
TextComponent,
|
||||
IfPermissionsDirective,
|
||||
DragDropSelectComponent,
|
||||
@@ -56,6 +62,7 @@ export class SavedViewsComponent
|
||||
private readonly modalService = inject(NgbModal)
|
||||
|
||||
DisplayMode = DisplayMode
|
||||
readonly savedViewIcons = SAVED_VIEW_ICONS
|
||||
|
||||
readonly savedViews = signal<SavedView[]>(undefined)
|
||||
private savedViewsGroup = new FormGroup({})
|
||||
@@ -110,6 +117,7 @@ export class SavedViewsComponent
|
||||
storeData.savedViews[view.id.toString()] = {
|
||||
id: view.id,
|
||||
name: view.name,
|
||||
icon: view.icon ?? DEFAULT_SAVED_VIEW_ICON,
|
||||
show_on_dashboard: view.show_on_dashboard,
|
||||
show_in_sidebar: view.show_in_sidebar,
|
||||
page_size: view.page_size,
|
||||
@@ -122,6 +130,7 @@ export class SavedViewsComponent
|
||||
new FormGroup({
|
||||
id: new FormControl({ value: null, disabled: !canEdit }),
|
||||
name: new FormControl({ value: null, disabled: !canEdit }),
|
||||
icon: new FormControl({ value: null, disabled: !canEdit }),
|
||||
show_on_dashboard: new FormControl({
|
||||
value: null,
|
||||
disabled: false,
|
||||
@@ -200,6 +209,7 @@ export class SavedViewsComponent
|
||||
|
||||
const modelFieldsChanged =
|
||||
group.get('name')?.dirty ||
|
||||
group.get('icon')?.dirty ||
|
||||
group.get('page_size')?.dirty ||
|
||||
group.get('display_mode')?.dirty ||
|
||||
group.get('display_fields')?.dirty
|
||||
|
||||
@@ -54,10 +54,6 @@ export const ConfigCategory = {
|
||||
AI: $localize`AI Settings`,
|
||||
}
|
||||
|
||||
export const ConfigSection = {
|
||||
RemoteOCR: $localize`Remote OCR`,
|
||||
}
|
||||
|
||||
export const LLMEmbeddingBackendConfig = {
|
||||
OPENAI_LIKE: 'openai-like',
|
||||
HUGGINGFACE: 'huggingface',
|
||||
@@ -69,15 +65,6 @@ export const LLMBackendConfig = {
|
||||
OLLAMA: 'ollama',
|
||||
}
|
||||
|
||||
export const RemoteOCREngineConfig = {
|
||||
AZURE_AI: 'azureai',
|
||||
}
|
||||
|
||||
export const RemoteOCRModeConfig = {
|
||||
ALWAYS: 'always',
|
||||
WORKFLOW_ONLY: 'workflow_only',
|
||||
}
|
||||
|
||||
export interface ConfigOption {
|
||||
key: string
|
||||
title: string
|
||||
@@ -85,7 +72,6 @@ export interface ConfigOption {
|
||||
choices?: Array<{ id: string; name: string }>
|
||||
config_key?: string
|
||||
category: string
|
||||
section?: string
|
||||
note?: string
|
||||
}
|
||||
|
||||
@@ -195,43 +181,6 @@ export const PaperlessConfigOptions: ConfigOption[] = [
|
||||
config_key: 'PAPERLESS_OCR_USER_ARGS',
|
||||
category: ConfigCategory.OCR,
|
||||
},
|
||||
{
|
||||
key: 'remote_ocr_engine',
|
||||
title: $localize`Remote OCR Engine`,
|
||||
type: ConfigOptionType.Select,
|
||||
choices: mapToItems(RemoteOCREngineConfig),
|
||||
config_key: 'PAPERLESS_REMOTE_OCR_ENGINE',
|
||||
category: ConfigCategory.OCR,
|
||||
section: ConfigSection.RemoteOCR,
|
||||
note: $localize`Enabling remote OCR sends documents to a third-party service for processing. Consider the privacy implications as well as potential costs before enabling.`,
|
||||
},
|
||||
{
|
||||
key: 'remote_ocr_api_key',
|
||||
title: $localize`Remote OCR API Key`,
|
||||
type: ConfigOptionType.Password,
|
||||
config_key: 'PAPERLESS_REMOTE_OCR_API_KEY',
|
||||
category: ConfigCategory.OCR,
|
||||
section: ConfigSection.RemoteOCR,
|
||||
},
|
||||
{
|
||||
key: 'remote_ocr_endpoint',
|
||||
title: $localize`Remote OCR Endpoint`,
|
||||
type: ConfigOptionType.String,
|
||||
config_key: 'PAPERLESS_REMOTE_OCR_ENDPOINT',
|
||||
category: ConfigCategory.OCR,
|
||||
section: ConfigSection.RemoteOCR,
|
||||
note: $localize`Required when using the Azure AI engine.`,
|
||||
},
|
||||
{
|
||||
key: 'remote_ocr_mode',
|
||||
title: $localize`Remote OCR Mode`,
|
||||
type: ConfigOptionType.Select,
|
||||
choices: mapToItems(RemoteOCRModeConfig),
|
||||
config_key: 'PAPERLESS_REMOTE_OCR_MODE',
|
||||
category: ConfigCategory.OCR,
|
||||
section: ConfigSection.RemoteOCR,
|
||||
note: $localize`Which documents are sent to the remote engine. Use 'workflow_only' to keep remote OCR off unless a workflow enables it for a document.`,
|
||||
},
|
||||
{
|
||||
key: 'app_logo',
|
||||
title: $localize`Application Logo`,
|
||||
@@ -449,10 +398,6 @@ export interface PaperlessConfig extends ObjectWithId {
|
||||
barcode_enable_tag: boolean
|
||||
barcode_tag_mapping: object
|
||||
barcode_tag_split: boolean
|
||||
remote_ocr_engine: string
|
||||
remote_ocr_api_key: string
|
||||
remote_ocr_endpoint: string
|
||||
remote_ocr_mode: string
|
||||
ai_enabled: boolean
|
||||
llm_embedding_backend: string
|
||||
llm_embedding_model: string
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
export const DEFAULT_SAVED_VIEW_ICON = 'funnel'
|
||||
|
||||
export const SAVED_VIEW_ICONS = [
|
||||
{ id: 'archive', name: $localize`Archive`, icon: 'archive' },
|
||||
{ id: 'bank', name: $localize`Bank`, icon: 'bank' },
|
||||
{ id: 'basket', name: $localize`Basket`, icon: 'basket' },
|
||||
{ id: 'bell', name: $localize`Bell`, icon: 'bell' },
|
||||
{ id: 'bookmark', name: $localize`Bookmark`, icon: 'bookmark' },
|
||||
{ id: 'boxes', name: $localize`Boxes`, icon: 'boxes' },
|
||||
{ id: 'briefcase', name: $localize`Briefcase`, icon: 'briefcase' },
|
||||
{ id: 'building', name: $localize`Building`, icon: 'building' },
|
||||
{ id: 'calculator', name: $localize`Calculator`, icon: 'calculator' },
|
||||
{ id: 'calendar', name: $localize`Calendar`, icon: 'calendar' },
|
||||
{ id: 'camera', name: $localize`Camera`, icon: 'camera' },
|
||||
{
|
||||
id: 'card-checklist',
|
||||
name: $localize`Checklist`,
|
||||
icon: 'card-checklist',
|
||||
},
|
||||
{ id: 'cash', name: $localize`Cash`, icon: 'cash' },
|
||||
{ id: 'chat-left-text', name: $localize`Chat`, icon: 'chat-left-text' },
|
||||
{ id: 'check-circle', name: $localize`Check`, icon: 'check-circle' },
|
||||
{ id: 'clipboard', name: $localize`Clipboard`, icon: 'clipboard' },
|
||||
{ id: 'clock-history', name: $localize`Clock`, icon: 'clock-history' },
|
||||
{ id: 'credit-card', name: $localize`Credit card`, icon: 'credit-card' },
|
||||
{ id: 'download', name: $localize`Download`, icon: 'download' },
|
||||
{ id: 'envelope', name: $localize`Envelope`, icon: 'envelope' },
|
||||
{
|
||||
id: 'exclamation-triangle',
|
||||
name: $localize`Warning`,
|
||||
icon: 'exclamation-triangle',
|
||||
},
|
||||
{ id: 'file-earmark', name: $localize`File`, icon: 'file-earmark' },
|
||||
{
|
||||
id: 'file-earmark-check',
|
||||
name: $localize`Checked file`,
|
||||
icon: 'file-earmark-check',
|
||||
},
|
||||
{
|
||||
id: 'file-earmark-lock',
|
||||
name: $localize`Locked file`,
|
||||
icon: 'file-earmark-lock',
|
||||
},
|
||||
{
|
||||
id: 'file-earmark-medical',
|
||||
name: $localize`Medical file`,
|
||||
icon: 'file-earmark-medical',
|
||||
},
|
||||
{
|
||||
id: 'file-earmark-person',
|
||||
name: $localize`Person file`,
|
||||
icon: 'file-earmark-person',
|
||||
},
|
||||
{
|
||||
id: 'file-earmark-spreadsheet',
|
||||
name: $localize`Spreadsheet`,
|
||||
icon: 'file-earmark-spreadsheet',
|
||||
},
|
||||
{ id: 'file-text', name: $localize`Text file`, icon: 'file-text' },
|
||||
{ id: 'files', name: $localize`Files`, icon: 'files' },
|
||||
{ id: 'folder', name: $localize`Folder`, icon: 'folder' },
|
||||
{ id: 'funnel', name: $localize`Filter`, icon: 'funnel' },
|
||||
{ id: 'gear', name: $localize`Gear`, icon: 'gear' },
|
||||
{ id: 'globe2', name: $localize`Globe`, icon: 'globe2' },
|
||||
{ id: 'hash', name: $localize`Hash`, icon: 'hash' },
|
||||
{ id: 'heart', name: $localize`Heart`, icon: 'heart' },
|
||||
{ id: 'house', name: $localize`House`, icon: 'house' },
|
||||
{ id: 'inbox', name: $localize`Inbox`, icon: 'inbox' },
|
||||
{ id: 'journals', name: $localize`Journals`, icon: 'journals' },
|
||||
{ id: 'list-task', name: $localize`Task list`, icon: 'list-task' },
|
||||
{ id: 'newspaper', name: $localize`Newspaper`, icon: 'newspaper' },
|
||||
{ id: 'paperclip', name: $localize`Attachment`, icon: 'paperclip' },
|
||||
{ id: 'people', name: $localize`People`, icon: 'people' },
|
||||
{ id: 'person', name: $localize`Person`, icon: 'person' },
|
||||
{ id: 'printer', name: $localize`Printer`, icon: 'printer' },
|
||||
{ id: 'receipt', name: $localize`Receipt`, icon: 'receipt' },
|
||||
{ id: 'safe', name: $localize`Safe`, icon: 'safe' },
|
||||
{ id: 'search', name: $localize`Search`, icon: 'search' },
|
||||
{ id: 'send', name: $localize`Send`, icon: 'send' },
|
||||
{ id: 'shop', name: $localize`Shop`, icon: 'shop' },
|
||||
{ id: 'stack', name: $localize`Stack`, icon: 'stack' },
|
||||
{ id: 'stars', name: $localize`Stars`, icon: 'stars' },
|
||||
{ id: 'tag', name: $localize`Tag`, icon: 'tag' },
|
||||
{ id: 'tags', name: $localize`Tags`, icon: 'tags' },
|
||||
{ id: 'telephone', name: $localize`Telephone`, icon: 'telephone' },
|
||||
{ id: 'truck', name: $localize`Truck`, icon: 'truck' },
|
||||
{ id: 'upc-scan', name: $localize`Barcode`, icon: 'upc-scan' },
|
||||
{ id: 'wallet2', name: $localize`Wallet`, icon: 'wallet2' },
|
||||
]
|
||||
@@ -5,6 +5,8 @@ import { ObjectWithPermissions } from './object-with-permissions'
|
||||
export interface SavedView extends ObjectWithPermissions {
|
||||
name?: string
|
||||
|
||||
icon?: string
|
||||
|
||||
show_on_dashboard?: boolean
|
||||
|
||||
show_in_sidebar?: boolean
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import { PdfEditorEditMode } from '../components/common/pdf-editor/pdf-editor-edit-mode'
|
||||
import { PdfZoomScale } from '../components/common/pdf-viewer/pdf-viewer.types'
|
||||
import { RemoteOCRModeConfig } from './paperless-config'
|
||||
import { User } from './user'
|
||||
|
||||
export interface UiSettings {
|
||||
@@ -95,8 +94,6 @@ export const SETTINGS_KEYS = {
|
||||
OUTLOOK_OAUTH_URL: 'outlook_oauth_url',
|
||||
EMAIL_ENABLED: 'email_enabled',
|
||||
AI_ENABLED: 'ai_enabled',
|
||||
REMOTE_OCR_CONFIGURED: 'remote_ocr:configured',
|
||||
REMOTE_OCR_MODE: 'remote_ocr:mode',
|
||||
}
|
||||
|
||||
export const SETTINGS: UiSetting[] = [
|
||||
@@ -350,14 +347,4 @@ export const SETTINGS: UiSetting[] = [
|
||||
type: 'string',
|
||||
default: PdfEditorEditMode.Create,
|
||||
},
|
||||
{
|
||||
key: SETTINGS_KEYS.REMOTE_OCR_CONFIGURED,
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
},
|
||||
{
|
||||
key: SETTINGS_KEYS.REMOTE_OCR_MODE,
|
||||
type: 'string',
|
||||
default: RemoteOCRModeConfig.ALWAYS,
|
||||
},
|
||||
]
|
||||
|
||||
@@ -7,18 +7,6 @@ export enum WorkflowActionType {
|
||||
Webhook = 4,
|
||||
PasswordRemoval = 5,
|
||||
MoveToTrash = 6,
|
||||
RemoteOcr = 7,
|
||||
ApplyAiSuggestions = 8,
|
||||
}
|
||||
|
||||
// see src/documents/models.py AISuggestionField
|
||||
export enum AISuggestionField {
|
||||
Title = 'title',
|
||||
Tags = 'tags',
|
||||
Correspondent = 'correspondent',
|
||||
DocumentType = 'document_type',
|
||||
StoragePath = 'storage_path',
|
||||
Created = 'created',
|
||||
}
|
||||
|
||||
export interface WorkflowActionEmail extends ObjectWithId {
|
||||
@@ -113,10 +101,4 @@ export interface WorkflowAction extends ObjectWithId {
|
||||
webhook?: WorkflowActionWebhook
|
||||
|
||||
passwords?: string[]
|
||||
|
||||
ai_suggestion_fields?: AISuggestionField[]
|
||||
|
||||
ai_create_missing?: boolean
|
||||
|
||||
ai_overwrite_existing?: boolean
|
||||
}
|
||||
|
||||
@@ -284,21 +284,6 @@ describe(`DocumentService`, () => {
|
||||
expect(req.request.method).toEqual('POST')
|
||||
expect(req.request.body).toEqual({
|
||||
documents: ids,
|
||||
remote_ocr: false,
|
||||
})
|
||||
})
|
||||
|
||||
it('should request remote OCR when reprocessing with it enabled', () => {
|
||||
const ids = [1, 2, 3]
|
||||
subscription = service
|
||||
.reprocessDocuments({ documents: ids }, true)
|
||||
.subscribe()
|
||||
const req = httpTestingController.expectOne(
|
||||
`${environment.apiBaseUrl}${endpoint}/reprocess/`
|
||||
)
|
||||
expect(req.request.body).toEqual({
|
||||
documents: ids,
|
||||
remote_ocr: true,
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
@@ -349,13 +349,9 @@ export class DocumentService extends AbstractPaperlessService<Document> {
|
||||
})
|
||||
}
|
||||
|
||||
reprocessDocuments(
|
||||
selection: DocumentSelectionQuery,
|
||||
remoteOcr: boolean = false
|
||||
) {
|
||||
reprocessDocuments(selection: DocumentSelectionQuery) {
|
||||
return this.http.post(this.getResourceUrl(null, 'reprocess'), {
|
||||
...selection,
|
||||
remote_ocr: remoteOcr,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -13,7 +13,6 @@ import { environment } from 'src/environments/environment'
|
||||
import { CustomFieldDataType } from '../data/custom-field'
|
||||
import { DEFAULT_DISPLAY_FIELDS, DisplayField } from '../data/document'
|
||||
import { SavedView } from '../data/saved-view'
|
||||
import { RemoteOCRModeConfig } from '../data/paperless-config'
|
||||
import { SETTINGS_KEYS, UiSettings } from '../data/ui-settings'
|
||||
import { PermissionsService } from './permissions.service'
|
||||
import { CustomFieldsService } from './rest/custom-fields.service'
|
||||
@@ -435,26 +434,4 @@ describe('SettingsService', () => {
|
||||
).name
|
||||
).toEqual(customFields[0].name)
|
||||
})
|
||||
it('should offer remote OCR only when configured and selective', () => {
|
||||
settingsService.set(SETTINGS_KEYS.REMOTE_OCR_CONFIGURED, false)
|
||||
settingsService.set(
|
||||
SETTINGS_KEYS.REMOTE_OCR_MODE,
|
||||
RemoteOCRModeConfig.WORKFLOW_ONLY
|
||||
)
|
||||
expect(settingsService.remoteOCRIsSelectable).toBeFalsy()
|
||||
|
||||
// configured, but already handling every document
|
||||
settingsService.set(SETTINGS_KEYS.REMOTE_OCR_CONFIGURED, true)
|
||||
settingsService.set(
|
||||
SETTINGS_KEYS.REMOTE_OCR_MODE,
|
||||
RemoteOCRModeConfig.ALWAYS
|
||||
)
|
||||
expect(settingsService.remoteOCRIsSelectable).toBeFalsy()
|
||||
|
||||
settingsService.set(
|
||||
SETTINGS_KEYS.REMOTE_OCR_MODE,
|
||||
RemoteOCRModeConfig.WORKFLOW_ONLY
|
||||
)
|
||||
expect(settingsService.remoteOCRIsSelectable).toBeTruthy()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -19,7 +19,6 @@ import {
|
||||
} from 'src/app/utils/color'
|
||||
import { DEFAULT_APP_TITLE, environment } from 'src/environments/environment'
|
||||
import { DEFAULT_DISPLAY_FIELDS, DisplayField } from '../data/document'
|
||||
import { RemoteOCRModeConfig } from '../data/paperless-config'
|
||||
import { SavedView } from '../data/saved-view'
|
||||
import {
|
||||
PAPERLESS_GREEN_HEX,
|
||||
@@ -688,17 +687,6 @@ export class SettingsService {
|
||||
return this.settingIsSet(SETTINGS_KEYS.UPDATE_CHECKING_ENABLED)
|
||||
}
|
||||
|
||||
/**
|
||||
* Offering remote OCR as a choice only makes sense when an engine
|
||||
* is configured but is not already handling every document.
|
||||
*/
|
||||
get remoteOCRIsSelectable(): boolean {
|
||||
return (
|
||||
this.get(SETTINGS_KEYS.REMOTE_OCR_CONFIGURED) &&
|
||||
this.get(SETTINGS_KEYS.REMOTE_OCR_MODE) !== RemoteOCRModeConfig.ALWAYS
|
||||
)
|
||||
}
|
||||
|
||||
offerTour(): boolean {
|
||||
return this.dashboardIsEmpty() && !this.get(SETTINGS_KEYS.TOUR_COMPLETE)
|
||||
}
|
||||
|
||||
@@ -35,19 +35,27 @@ import {
|
||||
arrowRightShort,
|
||||
arrowUpRight,
|
||||
asterisk,
|
||||
bank,
|
||||
basket,
|
||||
bell,
|
||||
bodyText,
|
||||
bookmark,
|
||||
boxArrowUp,
|
||||
boxArrowUpRight,
|
||||
boxes,
|
||||
braces,
|
||||
briefcase,
|
||||
building,
|
||||
calculator,
|
||||
calendar,
|
||||
calendarEvent,
|
||||
calendarEventFill,
|
||||
camera,
|
||||
cardChecklist,
|
||||
cardHeading,
|
||||
caretDown,
|
||||
caretUp,
|
||||
cash,
|
||||
chatLeftText,
|
||||
chatSquareDots,
|
||||
check,
|
||||
@@ -65,6 +73,7 @@ import {
|
||||
clipboardCheckFill,
|
||||
clipboardFill,
|
||||
clockHistory,
|
||||
creditCard,
|
||||
dash,
|
||||
dashCircle,
|
||||
diagram3,
|
||||
@@ -83,9 +92,12 @@ import {
|
||||
fileEarmarkDiff,
|
||||
fileEarmarkFill,
|
||||
fileEarmarkLock,
|
||||
fileEarmarkMedical,
|
||||
fileEarmarkMinus,
|
||||
fileEarmarkPerson,
|
||||
fileEarmarkPlus,
|
||||
fileEarmarkRichtext,
|
||||
fileEarmarkSpreadsheet,
|
||||
fileText,
|
||||
files,
|
||||
filter,
|
||||
@@ -93,12 +105,15 @@ import {
|
||||
folderFill,
|
||||
funnel,
|
||||
gear,
|
||||
globe2,
|
||||
google,
|
||||
grid,
|
||||
gripVertical,
|
||||
hash,
|
||||
hddStack,
|
||||
heart,
|
||||
house,
|
||||
inbox,
|
||||
infoCircle,
|
||||
journals,
|
||||
link,
|
||||
@@ -106,7 +121,9 @@ import {
|
||||
listTask,
|
||||
listUl,
|
||||
microsoft,
|
||||
newspaper,
|
||||
nodePlus,
|
||||
paperclip,
|
||||
pencil,
|
||||
people,
|
||||
peopleFill,
|
||||
@@ -121,9 +138,12 @@ import {
|
||||
plusCircle,
|
||||
printer,
|
||||
questionCircle,
|
||||
receipt,
|
||||
safe,
|
||||
scissors,
|
||||
search,
|
||||
send,
|
||||
shop,
|
||||
slashCircle,
|
||||
sliders2Vertical,
|
||||
sortAlphaDown,
|
||||
@@ -133,14 +153,17 @@ import {
|
||||
tag,
|
||||
tagFill,
|
||||
tags,
|
||||
telephone,
|
||||
textIndentLeft,
|
||||
textLeft,
|
||||
threeDots,
|
||||
threeDotsVertical,
|
||||
trash,
|
||||
truck,
|
||||
uiRadios,
|
||||
unlock,
|
||||
upcScan,
|
||||
wallet2,
|
||||
windowStack,
|
||||
x,
|
||||
xCircle,
|
||||
@@ -258,15 +281,22 @@ const icons = {
|
||||
arrowRightShort,
|
||||
arrowUpRight,
|
||||
asterisk,
|
||||
bank,
|
||||
basket,
|
||||
bell,
|
||||
braces,
|
||||
bodyText,
|
||||
bookmark,
|
||||
boxArrowUp,
|
||||
boxArrowUpRight,
|
||||
boxes,
|
||||
briefcase,
|
||||
building,
|
||||
calculator,
|
||||
calendar,
|
||||
calendarEvent,
|
||||
calendarEventFill,
|
||||
camera,
|
||||
cardChecklist,
|
||||
cardHeading,
|
||||
caretDown,
|
||||
@@ -288,6 +318,8 @@ const icons = {
|
||||
clipboardCheckFill,
|
||||
clipboardFill,
|
||||
clockHistory,
|
||||
cash,
|
||||
creditCard,
|
||||
dash,
|
||||
dashCircle,
|
||||
diagram3,
|
||||
@@ -306,9 +338,12 @@ const icons = {
|
||||
fileEarmarkDiff,
|
||||
fileEarmarkFill,
|
||||
fileEarmarkLock,
|
||||
fileEarmarkMedical,
|
||||
fileEarmarkMinus,
|
||||
fileEarmarkPerson,
|
||||
fileEarmarkPlus,
|
||||
fileEarmarkRichtext,
|
||||
fileEarmarkSpreadsheet,
|
||||
files,
|
||||
fileText,
|
||||
filter,
|
||||
@@ -316,12 +351,15 @@ const icons = {
|
||||
folderFill,
|
||||
funnel,
|
||||
gear,
|
||||
globe2,
|
||||
google,
|
||||
grid,
|
||||
gripVertical,
|
||||
hash,
|
||||
hddStack,
|
||||
heart,
|
||||
house,
|
||||
inbox,
|
||||
infoCircle,
|
||||
journals,
|
||||
link,
|
||||
@@ -329,8 +367,10 @@ const icons = {
|
||||
listTask,
|
||||
listUl,
|
||||
microsoft,
|
||||
newspaper,
|
||||
nodePlus,
|
||||
pencil,
|
||||
paperclip,
|
||||
people,
|
||||
peopleFill,
|
||||
person,
|
||||
@@ -344,10 +384,13 @@ const icons = {
|
||||
plusCircle,
|
||||
printer,
|
||||
questionCircle,
|
||||
receipt,
|
||||
safe,
|
||||
scissors,
|
||||
search,
|
||||
send,
|
||||
slashCircle,
|
||||
shop,
|
||||
sliders2Vertical,
|
||||
sortAlphaDown,
|
||||
sortAlphaUpAlt,
|
||||
@@ -358,12 +401,15 @@ const icons = {
|
||||
tags,
|
||||
textIndentLeft,
|
||||
textLeft,
|
||||
telephone,
|
||||
threeDots,
|
||||
threeDotsVertical,
|
||||
trash,
|
||||
truck,
|
||||
uiRadios,
|
||||
unlock,
|
||||
upcScan,
|
||||
wallet2,
|
||||
windowStack,
|
||||
x,
|
||||
xCircle,
|
||||
|
||||
@@ -19,13 +19,6 @@ export const GlobalWorkerOptions = {
|
||||
workerSrc: '',
|
||||
}
|
||||
|
||||
export const AnnotationMode = {
|
||||
DISABLE: 0,
|
||||
ENABLE: 1,
|
||||
ENABLE_FORMS: 2,
|
||||
ENABLE_STORAGE: 3,
|
||||
}
|
||||
|
||||
export const getDocument = (_src: unknown): PDFDocumentLoadingTask => {
|
||||
return new PDFDocumentLoadingTask(Promise.resolve(new PDFDocumentProxy()))
|
||||
}
|
||||
|
||||
@@ -394,16 +394,10 @@ def delete(doc_ids: list[int]) -> Literal["OK"]:
|
||||
return "OK"
|
||||
|
||||
|
||||
def reprocess(doc_ids: list[int], *, remote_ocr: bool = False) -> Literal["OK"]:
|
||||
"""
|
||||
Re-run parsing for the given documents.
|
||||
|
||||
Consumption workflows do not run here, so ``remote_ocr`` is how the user
|
||||
asks for the remote engine when it is not configured to handle everything.
|
||||
"""
|
||||
def reprocess(doc_ids: list[int]) -> Literal["OK"]:
|
||||
for document_id in doc_ids:
|
||||
update_document_content_maybe_archive_file.apply_async(
|
||||
kwargs={"document_id": document_id, "remote_ocr": remote_ocr},
|
||||
kwargs={"document_id": document_id},
|
||||
headers={"trigger_source": PaperlessTask.TriggerSource.MANUAL},
|
||||
)
|
||||
|
||||
|
||||
@@ -53,7 +53,6 @@ from documents.utils import copy_basic_file_stats
|
||||
from documents.utils import copy_file_with_basic_stats
|
||||
from documents.utils import run_subprocess
|
||||
from paperless.config import OcrConfig
|
||||
from paperless.config import RemoteOCRConfig
|
||||
from paperless.models import ArchiveFileGenerationChoices
|
||||
from paperless.parsers import ParserContext
|
||||
from paperless.parsers import ParserProtocol
|
||||
@@ -452,19 +451,12 @@ class ConsumerPlugin(
|
||||
except Exception as e:
|
||||
self.log.error(f"Error attempting to clean PDF: {e}")
|
||||
|
||||
# Workflows have already run at this point, so the metadata knows
|
||||
# whether this document was singled out for remote OCR
|
||||
allow_remote = (
|
||||
self.metadata.remote_ocr or RemoteOCRConfig().remote_ocr_by_default
|
||||
)
|
||||
|
||||
# Based on the mime type, get the parser for that type
|
||||
parser_class: type[ParserProtocol] | None = (
|
||||
get_parser_registry().get_parser_for_file(
|
||||
mime_type,
|
||||
self.filename,
|
||||
self.working_copy,
|
||||
allow_remote=allow_remote,
|
||||
)
|
||||
)
|
||||
if not parser_class:
|
||||
@@ -473,16 +465,6 @@ class ConsumerPlugin(
|
||||
f"Unsupported mime type {mime_type}",
|
||||
)
|
||||
|
||||
if self.metadata.remote_ocr and not getattr(
|
||||
parser_class,
|
||||
"uses_remote_service",
|
||||
False,
|
||||
):
|
||||
self.log.warning(
|
||||
"Remote OCR was requested for this document but no remote "
|
||||
"parser is available for it, processing locally instead.",
|
||||
)
|
||||
|
||||
# Notify all listeners that we're going to do some work.
|
||||
|
||||
document_consumption_started.send(
|
||||
|
||||
@@ -34,7 +34,6 @@ class DocumentMetadataOverrides:
|
||||
skip_asn_if_exists: bool = False
|
||||
version_label: str | None = None
|
||||
actor_id: int | None = None
|
||||
remote_ocr: bool = False
|
||||
|
||||
def update(self, other: "DocumentMetadataOverrides") -> "DocumentMetadataOverrides":
|
||||
"""
|
||||
@@ -58,8 +57,6 @@ class DocumentMetadataOverrides:
|
||||
self.actor_id = other.actor_id
|
||||
if other.skip_asn_if_exists:
|
||||
self.skip_asn_if_exists = True
|
||||
if other.remote_ocr:
|
||||
self.remote_ocr = True
|
||||
if other.version_label is not None:
|
||||
self.version_label = other.version_label
|
||||
|
||||
|
||||
@@ -1,346 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import abc
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
import zipfile
|
||||
from contextlib import AbstractContextManager
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from pathlib import PurePosixPath
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.serializers.json import DjangoJSONEncoder
|
||||
|
||||
from documents.file_handling import delete_empty_directories
|
||||
from documents.utils import compute_checksum
|
||||
from documents.utils import copy_file_with_basic_stats
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterator
|
||||
from typing import TextIO
|
||||
|
||||
|
||||
def _dumps(content: list | dict) -> str:
|
||||
"""Serialize export JSON consistently across all sinks."""
|
||||
return json.dumps(content, cls=DjangoJSONEncoder, indent=2, ensure_ascii=False)
|
||||
|
||||
|
||||
class StreamingManifestWriter:
|
||||
"""Incrementally writes a JSON array to a text handle, one record at a time.
|
||||
|
||||
Knows nothing about folders or zips: it writes the array framing and records
|
||||
to whatever handle the sink's ``stream()`` yields. The sink owns the handle's
|
||||
lifecycle (atomic rename, compare, spooling).
|
||||
"""
|
||||
|
||||
def __init__(self, handle: TextIO) -> None:
|
||||
self._file = handle
|
||||
self._first = True
|
||||
self._file.write("[")
|
||||
|
||||
def write_record(self, record: dict) -> None:
|
||||
if not self._first:
|
||||
self._file.write(",\n")
|
||||
else:
|
||||
self._first = False
|
||||
self._file.write(_dumps(record))
|
||||
|
||||
def write_batch(self, records: list[dict]) -> None:
|
||||
for record in records:
|
||||
self.write_record(record)
|
||||
|
||||
def close(self) -> None:
|
||||
"""Write the closing bracket. Does NOT close the handle (the sink owns it)."""
|
||||
self._file.write("\n]")
|
||||
|
||||
|
||||
class ExportSink(AbstractContextManager, abc.ABC):
|
||||
"""Destination for a document export.
|
||||
|
||||
The command declares export contents via three verbs; the sink decides how to
|
||||
persist each. ``arcname`` is always a relative POSIX path
|
||||
(e.g. ``"manifest.json"``, ``"originals/foo.pdf"``).
|
||||
|
||||
Contract:
|
||||
* At most one ``stream()`` open at a time (it is the manifest);
|
||||
``add_file``/``add_json`` may be called while it is open.
|
||||
* Context-manager: normal exit finalizes, an exception aborts. No partial or
|
||||
failed run leaves a complete-looking artifact.
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def add_file(
|
||||
self,
|
||||
source: Path,
|
||||
arcname: str,
|
||||
*,
|
||||
checksum: str | None = None,
|
||||
) -> None: ...
|
||||
|
||||
@abc.abstractmethod
|
||||
def add_json(self, content: list | dict, arcname: str) -> None: ...
|
||||
|
||||
@abc.abstractmethod
|
||||
def stream(self, arcname: str) -> AbstractContextManager[TextIO]: ...
|
||||
|
||||
def _open(self) -> None:
|
||||
"""Hook called on context entry. Override as needed."""
|
||||
|
||||
@abc.abstractmethod
|
||||
def _finalize(self) -> None:
|
||||
"""Commit on clean exit."""
|
||||
|
||||
@abc.abstractmethod
|
||||
def _abort(self) -> None:
|
||||
"""Roll back on exception."""
|
||||
|
||||
def __enter__(self) -> ExportSink:
|
||||
self._open()
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb) -> None:
|
||||
if exc_type is not None:
|
||||
self._abort()
|
||||
else:
|
||||
self._finalize()
|
||||
|
||||
|
||||
class DirectoryExportSink(ExportSink):
|
||||
"""Writes loose files into a target directory, with incremental sync.
|
||||
|
||||
Owns the snapshot/skip/compare/prune machinery that used to live in the
|
||||
command (``files_in_export_dir``, ``check_and_copy``, ``check_and_write_json``,
|
||||
and the ``--delete`` pass).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
target: Path,
|
||||
*,
|
||||
compare_checksums: bool,
|
||||
compare_json: bool,
|
||||
delete: bool,
|
||||
) -> None:
|
||||
self._target = target.resolve()
|
||||
self._compare_checksums = compare_checksums
|
||||
self._compare_json = compare_json
|
||||
self._delete = delete
|
||||
self._snapshot: set[Path] = set()
|
||||
self._stream_open = False
|
||||
|
||||
def _open(self) -> None:
|
||||
for x in self._target.glob("**/*"):
|
||||
if x.is_file():
|
||||
self._snapshot.add(x.resolve())
|
||||
|
||||
def add_file(
|
||||
self,
|
||||
source: Path,
|
||||
arcname: str,
|
||||
*,
|
||||
checksum: str | None = None,
|
||||
) -> None:
|
||||
target = (self._target / arcname).resolve()
|
||||
self._snapshot.discard(target)
|
||||
perform_copy = False
|
||||
if target.exists():
|
||||
source_stat = source.stat()
|
||||
target_stat = target.stat()
|
||||
if self._compare_checksums and checksum:
|
||||
perform_copy = compute_checksum(target) != checksum
|
||||
elif (
|
||||
source_stat.st_mtime != target_stat.st_mtime
|
||||
or source_stat.st_size != target_stat.st_size
|
||||
):
|
||||
perform_copy = True
|
||||
else:
|
||||
perform_copy = True
|
||||
if perform_copy:
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
copy_file_with_basic_stats(source, target)
|
||||
|
||||
@staticmethod
|
||||
def _content_unchanged(target: Path, new_bytes: bytes) -> bool:
|
||||
"""True if ``target`` already holds byte-identical content (BLAKE2b)."""
|
||||
return (
|
||||
hashlib.blake2b(target.read_bytes()).hexdigest()
|
||||
== hashlib.blake2b(new_bytes).hexdigest()
|
||||
)
|
||||
|
||||
def add_json(self, content: list | dict, arcname: str) -> None:
|
||||
target = (self._target / arcname).resolve()
|
||||
json_str = _dumps(content)
|
||||
perform_write = True
|
||||
if target in self._snapshot:
|
||||
self._snapshot.discard(target)
|
||||
if self._compare_json and self._content_unchanged(
|
||||
target,
|
||||
json_str.encode("utf-8"),
|
||||
):
|
||||
perform_write = False
|
||||
if perform_write:
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_text(json_str, encoding="utf-8")
|
||||
|
||||
@contextmanager
|
||||
def stream(self, arcname: str) -> Iterator[TextIO]:
|
||||
if self._stream_open:
|
||||
raise RuntimeError("A stream is already open on this sink")
|
||||
target = (self._target / arcname).resolve()
|
||||
tmp = target.with_suffix(target.suffix + ".tmp")
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
handle = tmp.open("w", encoding="utf-8")
|
||||
self._stream_open = True
|
||||
try:
|
||||
yield handle
|
||||
except BaseException:
|
||||
handle.close()
|
||||
tmp.unlink(missing_ok=True)
|
||||
raise
|
||||
else:
|
||||
handle.close()
|
||||
self._commit_streamed_file(target, tmp)
|
||||
finally:
|
||||
self._stream_open = False
|
||||
|
||||
def _commit_streamed_file(self, target: Path, tmp: Path) -> None:
|
||||
if target in self._snapshot:
|
||||
self._snapshot.discard(target)
|
||||
if self._compare_json and self._content_unchanged(
|
||||
target,
|
||||
tmp.read_bytes(),
|
||||
):
|
||||
tmp.unlink()
|
||||
return
|
||||
tmp.rename(target)
|
||||
|
||||
def _finalize(self) -> None:
|
||||
if self._delete:
|
||||
for f in self._snapshot:
|
||||
if not f.is_relative_to(self._target): # pragma: no cover
|
||||
# Defense in depth: a symlink inside the export dir can
|
||||
# resolve outside of it; never delete outside the target.
|
||||
continue
|
||||
f.unlink()
|
||||
delete_empty_directories(f.parent, self._target)
|
||||
|
||||
def _abort(self) -> None:
|
||||
# Folder mode is in-place/incremental: streamed .tmp files are already
|
||||
# cleaned in stream(); leave everything else intact and skip the prune.
|
||||
return None
|
||||
|
||||
|
||||
class ZipExportSink(ExportSink):
|
||||
"""Writes a single zip archive, produced atomically only on success.
|
||||
|
||||
Builds into ``<target>/<zip_name>.zip.tmp`` and renames to ``.zip`` on clean
|
||||
finalize. The manifest stream is spooled to a temp file in SCRATCH_DIR and
|
||||
added as an entry at finalize (a zip entry cannot be interleaved with others).
|
||||
"""
|
||||
|
||||
def __init__(self, target: Path, zip_name: str, *, delete: bool = False) -> None:
|
||||
self._target = target.resolve()
|
||||
self._zip_path = (self._target / zip_name).with_suffix(".zip")
|
||||
self._tmp_path = self._zip_path.with_name(self._zip_path.name + ".tmp")
|
||||
self._delete = delete
|
||||
self._zip: zipfile.ZipFile | None = None
|
||||
self._dirs: set[str] = set()
|
||||
self._pending_manifest: tuple[Path, str] | None = None
|
||||
self._stream_open = False
|
||||
|
||||
def _open(self) -> None:
|
||||
settings.SCRATCH_DIR.mkdir(parents=True, exist_ok=True)
|
||||
self._zip = zipfile.ZipFile(
|
||||
self._tmp_path,
|
||||
"w",
|
||||
compression=zipfile.ZIP_DEFLATED,
|
||||
allowZip64=True,
|
||||
)
|
||||
|
||||
def _ensure_dirs(self, arcname: str) -> None:
|
||||
assert self._zip is not None
|
||||
dir_arc = ""
|
||||
for part in PurePosixPath(arcname).parts[:-1]:
|
||||
dir_arc += f"{part}/"
|
||||
if dir_arc not in self._dirs:
|
||||
self._dirs.add(dir_arc)
|
||||
self._zip.mkdir(dir_arc)
|
||||
|
||||
def add_file(
|
||||
self,
|
||||
source: Path,
|
||||
arcname: str,
|
||||
*,
|
||||
checksum: str | None = None,
|
||||
) -> None:
|
||||
assert self._zip is not None
|
||||
self._ensure_dirs(arcname)
|
||||
self._zip.write(source, arcname=arcname)
|
||||
|
||||
def add_json(self, content: list | dict, arcname: str) -> None:
|
||||
assert self._zip is not None
|
||||
self._ensure_dirs(arcname)
|
||||
self._zip.writestr(arcname, _dumps(content))
|
||||
|
||||
@contextmanager
|
||||
def stream(self, arcname: str) -> Iterator[TextIO]:
|
||||
if self._stream_open:
|
||||
raise RuntimeError("A stream is already open on this sink")
|
||||
settings.SCRATCH_DIR.mkdir(parents=True, exist_ok=True)
|
||||
fd, tmp_name = tempfile.mkstemp(
|
||||
dir=settings.SCRATCH_DIR,
|
||||
prefix="export-manifest-",
|
||||
suffix=".json",
|
||||
)
|
||||
tmp = Path(tmp_name)
|
||||
handle = os.fdopen(fd, "w", encoding="utf-8")
|
||||
self._stream_open = True
|
||||
try:
|
||||
yield handle
|
||||
except BaseException:
|
||||
handle.close()
|
||||
tmp.unlink(missing_ok=True)
|
||||
raise
|
||||
else:
|
||||
handle.close()
|
||||
self._pending_manifest = (tmp, arcname)
|
||||
finally:
|
||||
self._stream_open = False
|
||||
|
||||
def _finalize(self) -> None:
|
||||
assert self._zip is not None
|
||||
if self._pending_manifest is not None:
|
||||
tmp, arcname = self._pending_manifest
|
||||
self._ensure_dirs(arcname)
|
||||
self._zip.write(tmp, arcname=arcname)
|
||||
tmp.unlink(missing_ok=True)
|
||||
self._pending_manifest = None
|
||||
self._zip.close()
|
||||
self._zip = None
|
||||
if self._delete:
|
||||
self._wipe_destination()
|
||||
self._tmp_path.replace(self._zip_path)
|
||||
|
||||
def _wipe_destination(self) -> None:
|
||||
skip = {self._zip_path.resolve(), self._tmp_path.resolve()}
|
||||
for item in self._target.glob("*"):
|
||||
if item.resolve() in skip:
|
||||
continue
|
||||
if item.is_dir():
|
||||
shutil.rmtree(item)
|
||||
else:
|
||||
item.unlink()
|
||||
|
||||
def _abort(self) -> None:
|
||||
if self._zip is not None:
|
||||
self._zip.close()
|
||||
self._zip = None
|
||||
self._tmp_path.unlink(missing_ok=True)
|
||||
if self._pending_manifest is not None:
|
||||
self._pending_manifest[0].unlink(missing_ok=True)
|
||||
self._pending_manifest = None
|
||||
+49
-30
@@ -39,6 +39,7 @@ from guardian.utils import get_user_obj_perms_model
|
||||
from rest_framework import serializers
|
||||
from rest_framework.filters import BaseFilterBackend
|
||||
from rest_framework.filters import OrderingFilter
|
||||
from rest_framework_guardian.filters import ObjectPermissionsFilter
|
||||
|
||||
from documents.models import Correspondent
|
||||
from documents.models import CustomField
|
||||
@@ -50,7 +51,7 @@ from documents.models import ShareLink
|
||||
from documents.models import ShareLinkBundle
|
||||
from documents.models import StoragePath
|
||||
from documents.models import Tag
|
||||
from documents.permissions import permitted_object_ids
|
||||
from documents.permissions import permitted_document_ids
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
@@ -1027,41 +1028,59 @@ class PaperlessTaskFilterSet(FilterSet):
|
||||
return queryset.exclude(status__in=PaperlessTask.COMPLETE_STATUSES)
|
||||
|
||||
|
||||
class PermittedObjectsFilter(BaseFilterBackend):
|
||||
class ObjectOwnedOrGrantedPermissionsFilter(ObjectPermissionsFilter):
|
||||
"""
|
||||
Filters a queryset down to objects the requesting user owns, are
|
||||
unowned, or (when ``include_granted`` is True) has an explicit
|
||||
user/group guardian permission on. Backed by ``permitted_object_ids``
|
||||
-- a single ``id__in`` subquery, not a join -- so it can't produce
|
||||
duplicate rows even when the base queryset already carries independent
|
||||
joins (e.g. multi-value ``tags__id__all`` filtering), and stays
|
||||
index-friendly at scale instead of falling back to guardian's
|
||||
varchar-cast join.
|
||||
|
||||
Set ``include_granted = False`` on a subclass for endpoints that
|
||||
intentionally only show owned/unowned objects regardless of explicit
|
||||
shares (e.g. ``TrashView``).
|
||||
A filter backend that limits results to those where the requesting user
|
||||
has read object level permissions, owns the objects, or objects without
|
||||
an owner (for backwards compat)
|
||||
"""
|
||||
|
||||
include_granted: bool = True
|
||||
perm_codename: str | None = None
|
||||
|
||||
def filter_queryset(self, request, queryset, view):
|
||||
# Before the superuser and owner-only paths, neither of which consults
|
||||
# permitted_object_ids. Scoped to authenticated users so anonymous
|
||||
# access (AnonymousUser.is_active is False) keeps its existing
|
||||
# unowned-only behaviour.
|
||||
if request.user.is_authenticated and not request.user.is_active:
|
||||
return queryset.none()
|
||||
if request.user.is_superuser:
|
||||
return queryset
|
||||
if not self.include_granted:
|
||||
return queryset.filter(Q(owner=request.user) | Q(owner__isnull=True))
|
||||
model = queryset.model
|
||||
perm = self.perm_codename or f"view_{model._meta.model_name}"
|
||||
return queryset.filter(
|
||||
id__in=permitted_object_ids(request.user, model, perm),
|
||||
)
|
||||
objects_with_perms = super().filter_queryset(request, queryset, view)
|
||||
objects_owned = queryset.filter(owner=request.user)
|
||||
objects_unowned = queryset.filter(owner__isnull=True)
|
||||
return objects_with_perms | objects_owned | objects_unowned
|
||||
|
||||
|
||||
class DocumentPermissionsFilter(BaseFilterBackend):
|
||||
"""
|
||||
A filter backend limiting Document results to those the requesting user
|
||||
owns, are unowned, or has explicit (user- or group-level) view
|
||||
permission on.
|
||||
|
||||
Unlike ``ObjectOwnedOrGrantedPermissionsFilter``, this does not build an
|
||||
``objects_with_perms | objects_owned | objects_unowned`` union of
|
||||
querysets derived from the same base queryset. When that base queryset
|
||||
already carries independent joins on a multi-valued relation (e.g. two
|
||||
separate joins from ``tags__id__all`` filtering on two tags), each
|
||||
OR-ed branch can end up pairing those joins' aliases differently,
|
||||
letting more than one row out of the join's cross product satisfy the
|
||||
combined WHERE -- returning the same document more than once. Filtering
|
||||
via a single ``id__in`` against ``permitted_document_ids`` (a plain
|
||||
subquery, not a join) sidesteps that entirely and is also cheaper than
|
||||
guardian's join-based permission check.
|
||||
"""
|
||||
|
||||
def filter_queryset(self, request, queryset, view):
|
||||
if request.user.is_superuser:
|
||||
return queryset
|
||||
return queryset.filter(id__in=permitted_document_ids(request.user))
|
||||
|
||||
|
||||
class ObjectOwnedPermissionsFilter(ObjectPermissionsFilter):
|
||||
"""
|
||||
A filter backend that limits results to those where the requesting user
|
||||
owns the objects or objects without an owner (for backwards compat)
|
||||
"""
|
||||
|
||||
def filter_queryset(self, request, queryset, view):
|
||||
if request.user.is_superuser:
|
||||
return queryset
|
||||
objects_owned = queryset.filter(owner=request.user)
|
||||
objects_unowned = queryset.filter(owner__isnull=True)
|
||||
return objects_owned | objects_unowned
|
||||
|
||||
|
||||
class DocumentsOrderingFilter(OrderingFilter):
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
from itertools import islice
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
@@ -15,6 +19,7 @@ from django.contrib.auth.models import User
|
||||
from django.contrib.contenttypes.models import ContentType
|
||||
from django.core import serializers
|
||||
from django.core.management.base import CommandError
|
||||
from django.core.serializers.json import DjangoJSONEncoder
|
||||
from django.db import transaction
|
||||
from django.utils import timezone
|
||||
from filelock import FileLock
|
||||
@@ -29,10 +34,7 @@ if TYPE_CHECKING:
|
||||
if settings.AUDIT_LOG_ENABLED:
|
||||
from auditlog.models import LogEntry
|
||||
|
||||
from documents.export.sinks import DirectoryExportSink
|
||||
from documents.export.sinks import ExportSink
|
||||
from documents.export.sinks import StreamingManifestWriter
|
||||
from documents.export.sinks import ZipExportSink
|
||||
from documents.file_handling import delete_empty_directories
|
||||
from documents.file_handling import generate_filename
|
||||
from documents.management.commands.base import PaperlessCommand
|
||||
from documents.management.commands.mixins import CryptMixin
|
||||
@@ -58,7 +60,8 @@ from documents.settings import EXPORTER_ARCHIVE_NAME
|
||||
from documents.settings import EXPORTER_FILE_NAME
|
||||
from documents.settings import EXPORTER_SHARE_LINK_BUNDLE_NAME
|
||||
from documents.settings import EXPORTER_THUMBNAIL_NAME
|
||||
from documents.utils import QuerySetStream
|
||||
from documents.utils import compute_checksum
|
||||
from documents.utils import copy_file_with_basic_stats
|
||||
from paperless import version
|
||||
from paperless.models import ApplicationConfiguration
|
||||
from paperless_mail.models import MailAccount
|
||||
@@ -81,6 +84,87 @@ def serialize_queryset_batched(
|
||||
yield serializers.serialize("python", chunk)
|
||||
|
||||
|
||||
class StreamingManifestWriter:
|
||||
"""Incrementally writes a JSON array to a file, one record at a time.
|
||||
|
||||
Writes to <target>.tmp first; on close(), optionally BLAKE2b-compares
|
||||
with the existing file (--compare-json) and renames or discards accordingly.
|
||||
On exception, discard() deletes the tmp file and leaves the original intact.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
path: Path,
|
||||
*,
|
||||
compare_json: bool = False,
|
||||
files_in_export_dir: "set[Path] | None" = None,
|
||||
) -> None:
|
||||
self._path = path.resolve()
|
||||
self._tmp_path = self._path.with_suffix(self._path.suffix + ".tmp")
|
||||
self._compare_json = compare_json
|
||||
self._files_in_export_dir: set[Path] = (
|
||||
files_in_export_dir if files_in_export_dir is not None else set()
|
||||
)
|
||||
self._file = None
|
||||
self._first = True
|
||||
|
||||
def open(self) -> None:
|
||||
self._path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self._file = self._tmp_path.open("w", encoding="utf-8")
|
||||
self._file.write("[")
|
||||
self._first = True
|
||||
|
||||
def write_record(self, record: dict) -> None:
|
||||
if not self._first:
|
||||
self._file.write(",\n")
|
||||
else:
|
||||
self._first = False
|
||||
self._file.write(
|
||||
json.dumps(record, cls=DjangoJSONEncoder, indent=2, ensure_ascii=False),
|
||||
)
|
||||
|
||||
def write_batch(self, records: list[dict]) -> None:
|
||||
for record in records:
|
||||
self.write_record(record)
|
||||
|
||||
def close(self) -> None:
|
||||
if self._file is None:
|
||||
return
|
||||
self._file.write("\n]")
|
||||
self._file.close()
|
||||
self._file = None
|
||||
self._finalize()
|
||||
|
||||
def discard(self) -> None:
|
||||
if self._file is not None:
|
||||
self._file.close()
|
||||
self._file = None
|
||||
if self._tmp_path.exists():
|
||||
self._tmp_path.unlink()
|
||||
|
||||
def _finalize(self) -> None:
|
||||
"""Compare with existing file (if --compare-json) then rename or discard tmp."""
|
||||
if self._path in self._files_in_export_dir:
|
||||
self._files_in_export_dir.remove(self._path)
|
||||
if self._compare_json:
|
||||
existing_hash = hashlib.blake2b(self._path.read_bytes()).hexdigest()
|
||||
new_hash = hashlib.blake2b(self._tmp_path.read_bytes()).hexdigest()
|
||||
if existing_hash == new_hash:
|
||||
self._tmp_path.unlink()
|
||||
return
|
||||
self._tmp_path.rename(self._path)
|
||||
|
||||
def __enter__(self) -> "StreamingManifestWriter":
|
||||
self.open()
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb) -> None:
|
||||
if exc_type is not None:
|
||||
self.discard()
|
||||
else:
|
||||
self.close()
|
||||
|
||||
|
||||
class Command(CryptMixin, PaperlessCommand):
|
||||
help = (
|
||||
"Decrypt and rename all files in our collection into a given target "
|
||||
@@ -230,13 +314,20 @@ class Command(CryptMixin, PaperlessCommand):
|
||||
self.passphrase: str | None = options.get("passphrase")
|
||||
self.batch_size: int = options["batch_size"]
|
||||
|
||||
self.files_in_export_dir: set[Path] = set()
|
||||
self.exported_files: set[str] = set()
|
||||
|
||||
if self.zip_export and (self.compare_checksums or self.compare_json):
|
||||
raise CommandError(
|
||||
"--compare-checksums and --compare-json have no effect when "
|
||||
"used with --zip",
|
||||
# If zipping, save the original target for later and
|
||||
# get a temporary directory for the target instead
|
||||
temp_dir = None
|
||||
self.original_target = self.target
|
||||
if self.zip_export:
|
||||
settings.SCRATCH_DIR.mkdir(parents=True, exist_ok=True)
|
||||
temp_dir = tempfile.TemporaryDirectory(
|
||||
dir=settings.SCRATCH_DIR,
|
||||
prefix="paperless-export",
|
||||
)
|
||||
self.target = Path(temp_dir.name).resolve()
|
||||
|
||||
if not self.target.exists():
|
||||
raise CommandError("That path doesn't exist")
|
||||
@@ -247,28 +338,33 @@ class Command(CryptMixin, PaperlessCommand):
|
||||
if not os.access(self.target, os.W_OK):
|
||||
raise CommandError("That path doesn't appear to be writable")
|
||||
|
||||
sink: ExportSink
|
||||
if self.zip_export:
|
||||
sink = ZipExportSink(
|
||||
self.target,
|
||||
options["zip_name"],
|
||||
delete=self.delete,
|
||||
)
|
||||
else:
|
||||
sink = DirectoryExportSink(
|
||||
self.target,
|
||||
compare_checksums=self.compare_checksums,
|
||||
compare_json=self.compare_json,
|
||||
delete=self.delete,
|
||||
)
|
||||
try:
|
||||
# Prevent any ongoing changes in the documents
|
||||
with FileLock(settings.MEDIA_LOCK):
|
||||
self.dump()
|
||||
|
||||
# Prevent any ongoing changes in the documents while exporting
|
||||
with FileLock(settings.MEDIA_LOCK), sink:
|
||||
self.dump(sink)
|
||||
# We've written everything to the temporary directory in this case,
|
||||
# now make an archive in the original target, with all files stored
|
||||
if self.zip_export and temp_dir is not None:
|
||||
shutil.make_archive(
|
||||
self.original_target / options["zip_name"],
|
||||
format="zip",
|
||||
root_dir=temp_dir.name,
|
||||
)
|
||||
|
||||
def dump(self, sink: ExportSink) -> None:
|
||||
# 1. Create manifest, containing all correspondents, types, tags, storage
|
||||
# paths, note, documents and ui_settings
|
||||
finally:
|
||||
# Always cleanup the temporary directory, if one was created
|
||||
if self.zip_export and temp_dir is not None:
|
||||
temp_dir.cleanup()
|
||||
|
||||
def dump(self) -> None:
|
||||
# 1. Take a snapshot of what files exist in the current export folder
|
||||
for x in self.target.glob("**/*"):
|
||||
if x.is_file():
|
||||
self.files_in_export_dir.add(x.resolve())
|
||||
|
||||
# 2. Create manifest, containing all correspondents, types, tags, storage paths
|
||||
# note, documents and ui_settings
|
||||
_excluded_usernames = ["consumer", "AnonymousUser"]
|
||||
manifest_key_to_object_query: dict[str, QuerySet[Any]] = {
|
||||
"correspondents": Correspondent.objects.all(),
|
||||
@@ -331,9 +427,13 @@ class Command(CryptMixin, PaperlessCommand):
|
||||
|
||||
document_manifest: list[dict] = []
|
||||
share_link_bundle_manifest: list[dict] = []
|
||||
manifest_path = (self.target / "manifest.json").resolve()
|
||||
|
||||
with sink.stream("manifest.json") as handle:
|
||||
writer = StreamingManifestWriter(handle)
|
||||
with StreamingManifestWriter(
|
||||
manifest_path,
|
||||
compare_json=self.compare_json,
|
||||
files_in_export_dir=self.files_in_export_dir,
|
||||
) as writer:
|
||||
with transaction.atomic():
|
||||
for key, qs in manifest_key_to_object_query.items():
|
||||
if key == "documents":
|
||||
@@ -369,6 +469,9 @@ class Command(CryptMixin, PaperlessCommand):
|
||||
self._encrypt_record_inline(record)
|
||||
writer.write_batch(batch)
|
||||
|
||||
document_map: dict[int, Document] = {
|
||||
d.pk: d for d in Document.global_objects.order_by("id")
|
||||
}
|
||||
share_link_bundle_map: dict[int, ShareLinkBundle] = {
|
||||
b.pk: b
|
||||
for b in ShareLinkBundle.objects.order_by("id").prefetch_related(
|
||||
@@ -376,72 +479,84 @@ class Command(CryptMixin, PaperlessCommand):
|
||||
)
|
||||
}
|
||||
|
||||
# 2. Export files from each document
|
||||
# document_manifest and this stream are both ordered by id from the
|
||||
# same underlying rows, so zip them in lockstep instead of building
|
||||
# a dict of every Document instance up front (QuerySetStream keeps
|
||||
# only one batch of documents resident at a time).
|
||||
documents_stream = QuerySetStream(
|
||||
Document.global_objects.order_by("id"),
|
||||
chunk_size=self.batch_size,
|
||||
)
|
||||
for document_dict, document in self.track(
|
||||
zip(document_manifest, documents_stream, strict=True),
|
||||
description="Exporting documents...",
|
||||
total=len(document_manifest),
|
||||
# 3. Export files from each document
|
||||
for index, document_dict in enumerate(
|
||||
self.track(
|
||||
document_manifest,
|
||||
description="Exporting documents...",
|
||||
total=len(document_manifest),
|
||||
),
|
||||
):
|
||||
# Both document_manifest and documents_stream come from the same
|
||||
# Document.global_objects.order_by("id") query, taken while
|
||||
# MEDIA_LOCK is held, so this should be unreachable -- it guards
|
||||
# against silent data corruption if that invariant ever breaks.
|
||||
if document.pk != document_dict["pk"]: # pragma: no cover
|
||||
raise CommandError(
|
||||
"Document export ordering mismatch: expected "
|
||||
f"pk={document_dict['pk']}, got pk={document.pk}. "
|
||||
"Documents may have changed during export.",
|
||||
)
|
||||
document = document_map[document_dict["pk"]]
|
||||
|
||||
# generate a unique filename, then the arcnames for its files
|
||||
# 3.1. generate a unique filename
|
||||
base_name = self.generate_base_name(document)
|
||||
original_arc, thumbnail_arc, archive_arc = (
|
||||
|
||||
# 3.2. write filenames into manifest
|
||||
original_target, thumbnail_target, archive_target = (
|
||||
self.generate_document_targets(document, base_name, document_dict)
|
||||
)
|
||||
|
||||
# 3.3. write files to target folder
|
||||
if not self.data_only:
|
||||
self.copy_document_files(
|
||||
document,
|
||||
sink,
|
||||
original_arc,
|
||||
thumbnail_arc,
|
||||
archive_arc,
|
||||
original_target,
|
||||
thumbnail_target,
|
||||
archive_target,
|
||||
)
|
||||
|
||||
if self.split_manifest:
|
||||
self._write_split_manifest(sink, document_dict, document, base_name)
|
||||
self._write_split_manifest(document_dict, document, base_name)
|
||||
else:
|
||||
writer.write_record(document_dict)
|
||||
|
||||
for bundle_dict in share_link_bundle_manifest:
|
||||
bundle = share_link_bundle_map[bundle_dict["pk"]]
|
||||
bundle_arc = self.generate_share_link_bundle_target(
|
||||
|
||||
bundle_target = self.generate_share_link_bundle_target(
|
||||
bundle,
|
||||
bundle_dict,
|
||||
)
|
||||
if not self.data_only and bundle_arc is not None:
|
||||
self.copy_share_link_bundle_file(bundle, sink, bundle_arc)
|
||||
|
||||
if not self.data_only and bundle_target is not None:
|
||||
self.copy_share_link_bundle_file(bundle, bundle_target)
|
||||
|
||||
writer.write_record(bundle_dict)
|
||||
|
||||
writer.close()
|
||||
|
||||
# 3. Write version (and crypto params) to metadata.json
|
||||
# Django stores most crypto values in the field itself; we store
|
||||
# them once here for the whole export
|
||||
# 4.2 write version information to target folder
|
||||
extra_metadata_path = (self.target / "metadata.json").resolve()
|
||||
metadata: dict[str, str | int | dict[str, str | int]] = {
|
||||
"version": version.__full_version_str__,
|
||||
}
|
||||
|
||||
# 4.2.1 If needed, write the crypto values into the metadata
|
||||
# Django stores most of these in the field itself, we store them once here
|
||||
if self.passphrase:
|
||||
metadata.update(self.get_crypt_params())
|
||||
sink.add_json(metadata, "metadata.json")
|
||||
|
||||
self.check_and_write_json(
|
||||
metadata,
|
||||
extra_metadata_path,
|
||||
)
|
||||
|
||||
if self.delete:
|
||||
# 5. Remove files which we did not explicitly export in this run
|
||||
if not self.zip_export:
|
||||
for f in self.files_in_export_dir:
|
||||
f.unlink()
|
||||
|
||||
delete_empty_directories(
|
||||
f.parent,
|
||||
self.target,
|
||||
)
|
||||
else:
|
||||
# 5. Remove anything in the original location (before moving the zip)
|
||||
for item in self.original_target.glob("*"):
|
||||
if item.is_dir():
|
||||
shutil.rmtree(item)
|
||||
else:
|
||||
item.unlink()
|
||||
|
||||
def generate_base_name(self, document: Document) -> Path:
|
||||
"""
|
||||
@@ -469,69 +584,73 @@ class Command(CryptMixin, PaperlessCommand):
|
||||
document: Document,
|
||||
base_name: Path,
|
||||
document_dict: dict,
|
||||
) -> tuple[str, str | None, str | None]:
|
||||
) -> tuple[Path, Path | None, Path | None]:
|
||||
"""
|
||||
Generates the relative POSIX arcnames for a document's original, thumbnail
|
||||
and archive files (depending on settings), and records them in the manifest.
|
||||
Generates the targets for a given document, including the original file, archive file and thumbnail (depending on settings).
|
||||
"""
|
||||
original_name = base_name
|
||||
if self.use_folder_prefix:
|
||||
original_name = Path("originals") / original_name
|
||||
original_arc = original_name.as_posix()
|
||||
document_dict[EXPORTER_FILE_NAME] = original_arc
|
||||
original_target = (self.target / original_name).resolve()
|
||||
document_dict[EXPORTER_FILE_NAME] = str(original_name)
|
||||
|
||||
if not self.no_thumbnail:
|
||||
thumbnail_name = base_name.parent / (base_name.stem + "-thumbnail.webp")
|
||||
if self.use_folder_prefix:
|
||||
thumbnail_name = Path("thumbnails") / thumbnail_name
|
||||
thumbnail_arc = thumbnail_name.as_posix()
|
||||
document_dict[EXPORTER_THUMBNAIL_NAME] = thumbnail_arc
|
||||
thumbnail_target = (self.target / thumbnail_name).resolve()
|
||||
document_dict[EXPORTER_THUMBNAIL_NAME] = str(thumbnail_name)
|
||||
else:
|
||||
thumbnail_arc = None
|
||||
thumbnail_target = None
|
||||
|
||||
if not self.no_archive and document.has_archive_version:
|
||||
archive_name = base_name.parent / (base_name.stem + "-archive.pdf")
|
||||
if self.use_folder_prefix:
|
||||
archive_name = Path("archive") / archive_name
|
||||
archive_arc = archive_name.as_posix()
|
||||
document_dict[EXPORTER_ARCHIVE_NAME] = archive_arc
|
||||
archive_target = (self.target / archive_name).resolve()
|
||||
document_dict[EXPORTER_ARCHIVE_NAME] = str(archive_name)
|
||||
else:
|
||||
archive_arc = None
|
||||
archive_target = None
|
||||
|
||||
return original_arc, thumbnail_arc, archive_arc
|
||||
return original_target, thumbnail_target, archive_target
|
||||
|
||||
def copy_document_files(
|
||||
self,
|
||||
document: Document,
|
||||
sink: ExportSink,
|
||||
original_arc: str,
|
||||
thumbnail_arc: str | None,
|
||||
archive_arc: str | None,
|
||||
original_target: Path,
|
||||
thumbnail_target: Path | None,
|
||||
archive_target: Path | None,
|
||||
) -> None:
|
||||
"""
|
||||
Hands the document's files to the sink (original, thumbnail, archive).
|
||||
Copies files from the document storage location to the specified target location.
|
||||
|
||||
If the document is encrypted, the files are decrypted before copying them to the target location.
|
||||
"""
|
||||
sink.add_file(document.source_path, original_arc, checksum=document.checksum)
|
||||
self.check_and_copy(
|
||||
document.source_path,
|
||||
document.checksum,
|
||||
original_target,
|
||||
)
|
||||
|
||||
if thumbnail_arc:
|
||||
sink.add_file(document.thumbnail_path, thumbnail_arc)
|
||||
if thumbnail_target:
|
||||
self.check_and_copy(document.thumbnail_path, None, thumbnail_target)
|
||||
|
||||
if archive_arc:
|
||||
if archive_target:
|
||||
if TYPE_CHECKING:
|
||||
assert isinstance(document.archive_path, Path)
|
||||
sink.add_file(
|
||||
self.check_and_copy(
|
||||
document.archive_path,
|
||||
archive_arc,
|
||||
checksum=document.archive_checksum,
|
||||
document.archive_checksum,
|
||||
archive_target,
|
||||
)
|
||||
|
||||
def generate_share_link_bundle_target(
|
||||
self,
|
||||
bundle: ShareLinkBundle,
|
||||
bundle_dict: dict,
|
||||
) -> str | None:
|
||||
) -> Path | None:
|
||||
"""
|
||||
Generates the relative POSIX arcname for a share link bundle file, if any.
|
||||
Generates the export target for a share link bundle file, when present.
|
||||
"""
|
||||
if not bundle.file_path:
|
||||
return None
|
||||
@@ -547,22 +666,25 @@ class Command(CryptMixin, PaperlessCommand):
|
||||
bundle_dict["fields"]["file_path"] = portable_bundle_path.as_posix()
|
||||
bundle_dict[EXPORTER_SHARE_LINK_BUNDLE_NAME] = export_bundle_path.as_posix()
|
||||
|
||||
return export_bundle_path.as_posix()
|
||||
return (self.target / export_bundle_path).resolve()
|
||||
|
||||
def copy_share_link_bundle_file(
|
||||
self,
|
||||
bundle: ShareLinkBundle,
|
||||
sink: ExportSink,
|
||||
bundle_arc: str,
|
||||
bundle_target: Path,
|
||||
) -> None:
|
||||
"""
|
||||
Hands a share link bundle ZIP to the sink.
|
||||
Copies a share link bundle ZIP into the export directory.
|
||||
"""
|
||||
bundle_source_path = bundle.absolute_file_path
|
||||
if bundle_source_path is None:
|
||||
raise FileNotFoundError(f"Share link bundle {bundle.pk} has no file path")
|
||||
|
||||
sink.add_file(bundle_source_path, bundle_arc)
|
||||
self.check_and_copy(
|
||||
bundle_source_path,
|
||||
None,
|
||||
bundle_target,
|
||||
)
|
||||
|
||||
def _encrypt_record_inline(self, record: dict) -> None:
|
||||
"""Encrypt sensitive fields in a single record, if passphrase is set."""
|
||||
@@ -578,7 +700,6 @@ class Command(CryptMixin, PaperlessCommand):
|
||||
|
||||
def _write_split_manifest(
|
||||
self,
|
||||
sink: ExportSink,
|
||||
document_dict: dict,
|
||||
document: Document,
|
||||
base_name: Path,
|
||||
@@ -600,4 +721,81 @@ class Command(CryptMixin, PaperlessCommand):
|
||||
manifest_name = base_name.with_name(f"{base_name.stem}-manifest.json")
|
||||
if self.use_folder_prefix:
|
||||
manifest_name = Path("json") / manifest_name
|
||||
sink.add_json(content, manifest_name.as_posix())
|
||||
manifest_name = (self.target / manifest_name).resolve()
|
||||
manifest_name.parent.mkdir(parents=True, exist_ok=True)
|
||||
self.check_and_write_json(content, manifest_name)
|
||||
|
||||
def check_and_write_json(
|
||||
self,
|
||||
content: list[dict] | dict,
|
||||
target: Path,
|
||||
) -> None:
|
||||
"""
|
||||
Writes the source content to the target json file.
|
||||
If --compare-json arg was used, don't write to target file if
|
||||
the file exists and checksum is identical to content checksum.
|
||||
This preserves the file timestamps when no changes are made.
|
||||
"""
|
||||
|
||||
target = target.resolve()
|
||||
perform_write = True
|
||||
if target in self.files_in_export_dir:
|
||||
self.files_in_export_dir.remove(target)
|
||||
if self.compare_json:
|
||||
target_checksum = hashlib.blake2b(target.read_bytes()).hexdigest()
|
||||
src_str = json.dumps(
|
||||
content,
|
||||
cls=DjangoJSONEncoder,
|
||||
indent=2,
|
||||
ensure_ascii=False,
|
||||
)
|
||||
src_checksum = hashlib.blake2b(src_str.encode("utf-8")).hexdigest()
|
||||
if src_checksum == target_checksum:
|
||||
perform_write = False
|
||||
|
||||
if perform_write:
|
||||
target.write_text(
|
||||
json.dumps(
|
||||
content,
|
||||
cls=DjangoJSONEncoder,
|
||||
indent=2,
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
def check_and_copy(
|
||||
self,
|
||||
source: Path,
|
||||
source_checksum: str | None,
|
||||
target: Path,
|
||||
) -> None:
|
||||
"""
|
||||
Copies the source to the target, if target doesn't exist or the target doesn't seem to match
|
||||
the source attributes
|
||||
"""
|
||||
|
||||
target = target.resolve()
|
||||
if target in self.files_in_export_dir:
|
||||
self.files_in_export_dir.remove(target)
|
||||
|
||||
perform_copy = False
|
||||
|
||||
if target.exists():
|
||||
source_stat = source.stat()
|
||||
target_stat = target.stat()
|
||||
if self.compare_checksums and source_checksum:
|
||||
target_checksum = compute_checksum(target)
|
||||
perform_copy = target_checksum != source_checksum
|
||||
elif (
|
||||
source_stat.st_mtime != target_stat.st_mtime
|
||||
or source_stat.st_size != target_stat.st_size
|
||||
):
|
||||
perform_copy = True
|
||||
else:
|
||||
# Copy if it does not exist
|
||||
perform_copy = True
|
||||
|
||||
if perform_copy:
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
copy_file_with_basic_stats(source, target)
|
||||
|
||||
+14
-10
@@ -19,7 +19,7 @@ from documents.models import StoragePath
|
||||
from documents.models import Tag
|
||||
from documents.models import Workflow
|
||||
from documents.models import WorkflowTrigger
|
||||
from documents.permissions import permitted_object_ids
|
||||
from documents.permissions import get_objects_for_user_owner_aware
|
||||
from documents.regex import safe_regex_search
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -55,8 +55,10 @@ def match_correspondents(document: Document, classifier: DocumentClassifier, use
|
||||
user = document.owner
|
||||
|
||||
if user is not None:
|
||||
correspondents = Correspondent.objects.filter(
|
||||
id__in=permitted_object_ids(user, Correspondent, "view_correspondent"),
|
||||
correspondents = get_objects_for_user_owner_aware(
|
||||
user,
|
||||
"documents.view_correspondent",
|
||||
Correspondent,
|
||||
)
|
||||
else:
|
||||
correspondents = Correspondent.objects.all()
|
||||
@@ -84,8 +86,10 @@ def match_document_types(document: Document, classifier: DocumentClassifier, use
|
||||
user = document.owner
|
||||
|
||||
if user is not None:
|
||||
document_types = DocumentType.objects.filter(
|
||||
id__in=permitted_object_ids(user, DocumentType, "view_documenttype"),
|
||||
document_types = get_objects_for_user_owner_aware(
|
||||
user,
|
||||
"documents.view_documenttype",
|
||||
DocumentType,
|
||||
)
|
||||
else:
|
||||
document_types = DocumentType.objects.all()
|
||||
@@ -112,9 +116,7 @@ def match_tags(document: Document, classifier: DocumentClassifier, user=None):
|
||||
user = document.owner
|
||||
|
||||
if user is not None:
|
||||
tags = Tag.objects.filter(
|
||||
id__in=permitted_object_ids(user, Tag, "view_tag"),
|
||||
)
|
||||
tags = get_objects_for_user_owner_aware(user, "documents.view_tag", Tag)
|
||||
else:
|
||||
tags = Tag.objects.all()
|
||||
|
||||
@@ -143,8 +145,10 @@ def match_storage_paths(document: Document, classifier: DocumentClassifier, user
|
||||
user = document.owner
|
||||
|
||||
if user is not None:
|
||||
storage_paths = StoragePath.objects.filter(
|
||||
id__in=permitted_object_ids(user, StoragePath, "view_storagepath"),
|
||||
storage_paths = get_objects_for_user_owner_aware(
|
||||
user,
|
||||
"documents.view_storagepath",
|
||||
StoragePath,
|
||||
)
|
||||
else:
|
||||
storage_paths = StoragePath.objects.all()
|
||||
|
||||
@@ -1,30 +0,0 @@
|
||||
# Generated by Django 5.2.16 on 2026-08-10 17:27
|
||||
|
||||
from django.db import migrations
|
||||
from django.db import models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("documents", "0022_add_perf_indexes"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name="workflowaction",
|
||||
name="type",
|
||||
field=models.PositiveSmallIntegerField(
|
||||
choices=[
|
||||
(1, "Assignment"),
|
||||
(2, "Removal"),
|
||||
(3, "Email"),
|
||||
(4, "Webhook"),
|
||||
(5, "Password removal"),
|
||||
(6, "Move to trash"),
|
||||
(7, "Remote OCR"),
|
||||
],
|
||||
default=1,
|
||||
verbose_name="Workflow Action Type",
|
||||
),
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,79 @@
|
||||
from django.db import migrations
|
||||
from django.db import models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("documents", "0022_add_perf_indexes"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name="savedview",
|
||||
name="icon",
|
||||
field=models.CharField(
|
||||
choices=[
|
||||
("archive", "Archive"),
|
||||
("bank", "Bank"),
|
||||
("basket", "Basket"),
|
||||
("bell", "Bell"),
|
||||
("bookmark", "Bookmark"),
|
||||
("boxes", "Boxes"),
|
||||
("briefcase", "Briefcase"),
|
||||
("building", "Building"),
|
||||
("calculator", "Calculator"),
|
||||
("calendar", "Calendar"),
|
||||
("camera", "Camera"),
|
||||
("card-checklist", "Checklist"),
|
||||
("cash", "Cash"),
|
||||
("chat-left-text", "Chat"),
|
||||
("check-circle", "Check"),
|
||||
("clipboard", "Clipboard"),
|
||||
("clock-history", "Clock"),
|
||||
("credit-card", "Credit card"),
|
||||
("download", "Download"),
|
||||
("envelope", "Envelope"),
|
||||
("exclamation-triangle", "Warning"),
|
||||
("file-earmark", "File"),
|
||||
("file-earmark-check", "Checked file"),
|
||||
("file-earmark-lock", "Locked file"),
|
||||
("file-earmark-medical", "Medical file"),
|
||||
("file-earmark-person", "Person file"),
|
||||
("file-earmark-spreadsheet", "Spreadsheet"),
|
||||
("file-text", "Text file"),
|
||||
("files", "Files"),
|
||||
("folder", "Folder"),
|
||||
("funnel", "Filter"),
|
||||
("gear", "Gear"),
|
||||
("globe2", "Globe"),
|
||||
("hash", "Hash"),
|
||||
("heart", "Heart"),
|
||||
("house", "House"),
|
||||
("inbox", "Inbox"),
|
||||
("journals", "Journals"),
|
||||
("list-task", "Task list"),
|
||||
("newspaper", "Newspaper"),
|
||||
("paperclip", "Attachment"),
|
||||
("people", "People"),
|
||||
("person", "Person"),
|
||||
("printer", "Printer"),
|
||||
("receipt", "Receipt"),
|
||||
("safe", "Safe"),
|
||||
("search", "Search"),
|
||||
("send", "Send"),
|
||||
("shop", "Shop"),
|
||||
("stack", "Stack"),
|
||||
("stars", "Stars"),
|
||||
("tag", "Tag"),
|
||||
("tags", "Tags"),
|
||||
("telephone", "Telephone"),
|
||||
("truck", "Truck"),
|
||||
("upc-scan", "Barcode"),
|
||||
("wallet2", "Wallet"),
|
||||
],
|
||||
default="funnel",
|
||||
max_length=64,
|
||||
verbose_name="icon",
|
||||
),
|
||||
),
|
||||
]
|
||||
@@ -1,84 +0,0 @@
|
||||
# Generated by Django 5.2.16 on 2026-08-10 18:26
|
||||
|
||||
from django.db import migrations
|
||||
from django.db import models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("documents", "0023_alter_workflowaction_type"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name="workflowaction",
|
||||
name="ai_create_missing",
|
||||
field=models.BooleanField(
|
||||
default=False,
|
||||
help_text="Create suggested tags, correspondents, document types and storage paths that do not already exist instead of skipping them.",
|
||||
verbose_name="create missing objects",
|
||||
),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="workflowaction",
|
||||
name="ai_overwrite_existing",
|
||||
field=models.BooleanField(
|
||||
default=False,
|
||||
help_text="Apply suggestions even if the document already has a value for that field. Tags are always added to, never replaced.",
|
||||
verbose_name="overwrite existing values",
|
||||
),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="workflowaction",
|
||||
name="ai_suggestion_fields",
|
||||
field=models.JSONField(
|
||||
blank=True,
|
||||
help_text="Which of the AI-suggested fields to apply to the document.",
|
||||
null=True,
|
||||
verbose_name="AI suggestion fields",
|
||||
),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name="workflowaction",
|
||||
name="type",
|
||||
field=models.PositiveSmallIntegerField(
|
||||
choices=[
|
||||
(1, "Assignment"),
|
||||
(2, "Removal"),
|
||||
(3, "Email"),
|
||||
(4, "Webhook"),
|
||||
(5, "Password removal"),
|
||||
(6, "Move to trash"),
|
||||
(7, "Remote OCR"),
|
||||
(8, "Apply AI suggestions"),
|
||||
],
|
||||
default=1,
|
||||
verbose_name="Workflow Action Type",
|
||||
),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name="paperlesstask",
|
||||
name="task_type",
|
||||
field=models.CharField(
|
||||
choices=[
|
||||
("consume_file", "Consume File"),
|
||||
("train_classifier", "Train Classifier"),
|
||||
("sanity_check", "Sanity Check"),
|
||||
("index_optimize", "Index Optimize"),
|
||||
("mail_fetch", "Mail Fetch"),
|
||||
("llm_index", "LLM Index"),
|
||||
("empty_trash", "Empty Trash"),
|
||||
("check_workflows", "Check Workflows"),
|
||||
("bulk_update", "Bulk Update"),
|
||||
("reprocess_document", "Reprocess Document"),
|
||||
("build_share_link", "Build Share Link"),
|
||||
("bulk_delete", "Bulk Delete"),
|
||||
("apply_ai_suggestions", "Apply AI Suggestions"),
|
||||
],
|
||||
db_index=True,
|
||||
help_text="The kind of work being performed",
|
||||
max_length=50,
|
||||
verbose_name="Task Type",
|
||||
),
|
||||
),
|
||||
]
|
||||
+69
-44
@@ -519,6 +519,68 @@ class Document(SoftDeleteModel, ModelWithOwner): # type: ignore[django-manager-
|
||||
|
||||
|
||||
class SavedView(ModelWithOwner):
|
||||
class Icon(models.TextChoices):
|
||||
ARCHIVE = ("archive", _("Archive"))
|
||||
BANK = ("bank", _("Bank"))
|
||||
BASKET = ("basket", _("Basket"))
|
||||
BELL = ("bell", _("Bell"))
|
||||
BOOKMARK = ("bookmark", _("Bookmark"))
|
||||
BOXES = ("boxes", _("Boxes"))
|
||||
BRIEFCASE = ("briefcase", _("Briefcase"))
|
||||
BUILDING = ("building", _("Building"))
|
||||
CALCULATOR = ("calculator", _("Calculator"))
|
||||
CALENDAR = ("calendar", _("Calendar"))
|
||||
CAMERA = ("camera", _("Camera"))
|
||||
CARD_CHECKLIST = ("card-checklist", _("Checklist"))
|
||||
CASH = ("cash", _("Cash"))
|
||||
CHAT_LEFT_TEXT = ("chat-left-text", _("Chat"))
|
||||
CHECK_CIRCLE = ("check-circle", _("Check"))
|
||||
CLIPBOARD = ("clipboard", _("Clipboard"))
|
||||
CLOCK_HISTORY = ("clock-history", _("Clock"))
|
||||
CREDIT_CARD = ("credit-card", _("Credit card"))
|
||||
DOWNLOAD = ("download", _("Download"))
|
||||
ENVELOPE = ("envelope", _("Envelope"))
|
||||
EXCLAMATION_TRIANGLE = ("exclamation-triangle", _("Warning"))
|
||||
FILE_EARMARK = ("file-earmark", _("File"))
|
||||
FILE_EARMARK_CHECK = ("file-earmark-check", _("Checked file"))
|
||||
FILE_EARMARK_LOCK = ("file-earmark-lock", _("Locked file"))
|
||||
FILE_EARMARK_MEDICAL = ("file-earmark-medical", _("Medical file"))
|
||||
FILE_EARMARK_PERSON = ("file-earmark-person", _("Person file"))
|
||||
FILE_EARMARK_SPREADSHEET = (
|
||||
"file-earmark-spreadsheet",
|
||||
_("Spreadsheet"),
|
||||
)
|
||||
FILE_TEXT = ("file-text", _("Text file"))
|
||||
FILES = ("files", _("Files"))
|
||||
FOLDER = ("folder", _("Folder"))
|
||||
FUNNEL = ("funnel", _("Filter"))
|
||||
GEAR = ("gear", _("Gear"))
|
||||
GLOBE = ("globe2", _("Globe"))
|
||||
HASH = ("hash", _("Hash"))
|
||||
HEART = ("heart", _("Heart"))
|
||||
HOUSE = ("house", _("House"))
|
||||
INBOX = ("inbox", _("Inbox"))
|
||||
JOURNALS = ("journals", _("Journals"))
|
||||
LIST_TASK = ("list-task", _("Task list"))
|
||||
NEWSPAPER = ("newspaper", _("Newspaper"))
|
||||
PAPERCLIP = ("paperclip", _("Attachment"))
|
||||
PEOPLE = ("people", _("People"))
|
||||
PERSON = ("person", _("Person"))
|
||||
PRINTER = ("printer", _("Printer"))
|
||||
RECEIPT = ("receipt", _("Receipt"))
|
||||
SAFE = ("safe", _("Safe"))
|
||||
SEARCH = ("search", _("Search"))
|
||||
SEND = ("send", _("Send"))
|
||||
SHOP = ("shop", _("Shop"))
|
||||
STACK = ("stack", _("Stack"))
|
||||
STARS = ("stars", _("Stars"))
|
||||
TAG = ("tag", _("Tag"))
|
||||
TAGS = ("tags", _("Tags"))
|
||||
TELEPHONE = ("telephone", _("Telephone"))
|
||||
TRUCK = ("truck", _("Truck"))
|
||||
UPC_SCAN = ("upc-scan", _("Barcode"))
|
||||
WALLET = ("wallet2", _("Wallet"))
|
||||
|
||||
class DisplayMode(models.TextChoices):
|
||||
TABLE = ("table", _("Table"))
|
||||
SMALL_CARDS = ("smallCards", _("Small Cards"))
|
||||
@@ -541,6 +603,13 @@ class SavedView(ModelWithOwner):
|
||||
|
||||
name = models.CharField(_("name"), max_length=128)
|
||||
|
||||
icon = models.CharField(
|
||||
_("icon"),
|
||||
max_length=64,
|
||||
choices=Icon.choices,
|
||||
default=Icon.FUNNEL,
|
||||
)
|
||||
|
||||
sort_field = models.CharField(
|
||||
_("sort field"),
|
||||
max_length=128,
|
||||
@@ -695,7 +764,6 @@ class PaperlessTask(ModelWithOwner):
|
||||
REPROCESS_DOCUMENT = "reprocess_document", _("Reprocess Document")
|
||||
BUILD_SHARE_LINK = "build_share_link", _("Build Share Link")
|
||||
BULK_DELETE = "bulk_delete", _("Bulk Delete")
|
||||
APPLY_AI_SUGGESTIONS = "apply_ai_suggestions", _("Apply AI Suggestions")
|
||||
|
||||
COMPLETE_STATUSES = (
|
||||
Status.SUCCESS,
|
||||
@@ -1600,22 +1668,6 @@ class WorkflowAction(models.Model):
|
||||
6,
|
||||
_("Move to trash"),
|
||||
)
|
||||
REMOTE_OCR = (
|
||||
7,
|
||||
_("Remote OCR"),
|
||||
)
|
||||
APPLY_AI_SUGGESTIONS = (
|
||||
8,
|
||||
_("Apply AI suggestions"),
|
||||
)
|
||||
|
||||
class AISuggestionField(models.TextChoices):
|
||||
TITLE = ("title", _("Title"))
|
||||
TAGS = ("tags", _("Tags"))
|
||||
CORRESPONDENT = ("correspondent", _("Correspondent"))
|
||||
DOCUMENT_TYPE = ("document_type", _("Document type"))
|
||||
STORAGE_PATH = ("storage_path", _("Storage path"))
|
||||
CREATED = ("created", _("Created date"))
|
||||
|
||||
type = models.PositiveSmallIntegerField(
|
||||
_("Workflow Action Type"),
|
||||
@@ -1854,33 +1906,6 @@ class WorkflowAction(models.Model):
|
||||
),
|
||||
)
|
||||
|
||||
ai_suggestion_fields = models.JSONField(
|
||||
_("AI suggestion fields"),
|
||||
null=True,
|
||||
blank=True,
|
||||
help_text=_(
|
||||
"Which of the AI-suggested fields to apply to the document.",
|
||||
),
|
||||
)
|
||||
|
||||
ai_create_missing = models.BooleanField(
|
||||
_("create missing objects"),
|
||||
default=False,
|
||||
help_text=_(
|
||||
"Create suggested tags, correspondents, document types and storage "
|
||||
"paths that do not already exist instead of skipping them.",
|
||||
),
|
||||
)
|
||||
|
||||
ai_overwrite_existing = models.BooleanField(
|
||||
_("overwrite existing values"),
|
||||
default=False,
|
||||
help_text=_(
|
||||
"Apply suggestions even if the document already has a value for that "
|
||||
"field. Tags are always added to, never replaced.",
|
||||
),
|
||||
)
|
||||
|
||||
class Meta:
|
||||
verbose_name = _("workflow action")
|
||||
verbose_name_plural = _("workflow actions")
|
||||
|
||||
@@ -7,7 +7,6 @@ from django.contrib.contenttypes.models import ContentType
|
||||
from django.db.models import Case
|
||||
from django.db.models import Count
|
||||
from django.db.models import IntegerField
|
||||
from django.db.models import Model
|
||||
from django.db.models import Q
|
||||
from django.db.models import QuerySet
|
||||
from django.db.models import Value
|
||||
@@ -54,15 +53,11 @@ class PaperlessObjectPermissions(DjangoObjectPermissions):
|
||||
|
||||
class PaperlessAdminPermissions(BasePermission):
|
||||
def has_permission(self, request, view):
|
||||
return request.user.is_active and request.user.is_staff
|
||||
return request.user.is_staff
|
||||
|
||||
|
||||
def has_global_statistics_permission(user: User | None) -> bool:
|
||||
if (
|
||||
user is None
|
||||
or not getattr(user, "is_active", False)
|
||||
or not getattr(user, "is_authenticated", False)
|
||||
):
|
||||
if user is None or not getattr(user, "is_authenticated", False):
|
||||
return False
|
||||
|
||||
return getattr(user, "is_superuser", False) or user.has_perm(
|
||||
@@ -71,11 +66,7 @@ def has_global_statistics_permission(user: User | None) -> bool:
|
||||
|
||||
|
||||
def has_system_status_permission(user: User | None) -> bool:
|
||||
if (
|
||||
user is None
|
||||
or not getattr(user, "is_active", False)
|
||||
or not getattr(user, "is_authenticated", False)
|
||||
):
|
||||
if user is None or not getattr(user, "is_authenticated", False):
|
||||
return False
|
||||
|
||||
return (
|
||||
@@ -172,39 +163,30 @@ def set_permissions_for_object(
|
||||
)
|
||||
|
||||
|
||||
def permitted_object_ids(
|
||||
user: User | None,
|
||||
model: type[Model],
|
||||
perm: str,
|
||||
def permitted_document_ids(
|
||||
user,
|
||||
*,
|
||||
perm: str = "view_document",
|
||||
include_deleted: bool = False,
|
||||
) -> QuerySet[int]:
|
||||
):
|
||||
"""
|
||||
Generic version of ``permitted_document_ids`` for any model with an
|
||||
``owner`` field and guardian object-level permissions. ``include_deleted``
|
||||
only has an effect for models exposing a ``global_objects``/``deleted_at``
|
||||
soft-delete pattern (currently only ``Document``); for every other model
|
||||
it is accepted but has no effect, since those models have no soft-delete
|
||||
concept.
|
||||
Return a queryset of document IDs the user has ``perm`` on (default
|
||||
``"view_document"``). By default limited to non-deleted documents; pass
|
||||
``include_deleted=True`` for callers that need to check permission on
|
||||
soft-deleted documents (e.g. trash restore). This intentionally avoids
|
||||
``get_objects_for_user`` to keep the subquery small and index-friendly.
|
||||
"""
|
||||
has_soft_delete = hasattr(model, "global_objects")
|
||||
manager = (
|
||||
model.global_objects if include_deleted and has_soft_delete else model.objects
|
||||
)
|
||||
base_qs = manager.all().only("id", "owner")
|
||||
|
||||
manager = Document.global_objects if include_deleted else Document.objects
|
||||
base_docs = manager.all()
|
||||
base_docs = base_docs.only("id", "owner")
|
||||
|
||||
if user is None or not getattr(user, "is_authenticated", False):
|
||||
return base_qs.filter(owner__isnull=True).values_list("id", flat=True)
|
||||
|
||||
# Deactivated users get nothing, deactivated superusers included, so this
|
||||
# has to come before the superuser shortcut. guardian's
|
||||
# ObjectPermissionChecker denies inactive users, but get_objects_for_user
|
||||
# (the pattern this replaces) does not, so it would not be inherited.
|
||||
if not getattr(user, "is_active", False):
|
||||
return base_qs.none().values_list("id", flat=True)
|
||||
# Just Anonymous user e.g. for drf-spectacular
|
||||
return base_docs.filter(owner__isnull=True).values_list("id", flat=True)
|
||||
|
||||
if getattr(user, "is_superuser", False):
|
||||
return base_qs.values_list("id", flat=True)
|
||||
return base_docs.values_list("id", flat=True)
|
||||
|
||||
# Guardian's UserObjectPermission/GroupObjectPermission always store a bare
|
||||
# codename, but has_perm()-style callers commonly pass the qualified
|
||||
@@ -212,46 +194,31 @@ def permitted_object_ids(
|
||||
# codename, so just drop any prefix rather than silently under-permitting.
|
||||
perm = perm.rsplit(".", 1)[-1]
|
||||
|
||||
content_type = ContentType.objects.get_for_model(model)
|
||||
document_ct = ContentType.objects.get_for_model(Document)
|
||||
perm_filter = {
|
||||
"permission__codename": perm,
|
||||
"permission__content_type": content_type,
|
||||
"permission__content_type": document_ct,
|
||||
}
|
||||
|
||||
user_perm_ids = (
|
||||
user_perm_docs = (
|
||||
UserObjectPermission.objects.filter(user=user, **perm_filter)
|
||||
.annotate(object_pk_int=Cast("object_pk", IntegerField()))
|
||||
.values_list("object_pk_int", flat=True)
|
||||
)
|
||||
group_perm_ids = (
|
||||
|
||||
group_perm_docs = (
|
||||
GroupObjectPermission.objects.filter(group__user=user, **perm_filter)
|
||||
.annotate(object_pk_int=Cast("object_pk", IntegerField()))
|
||||
.values_list("object_pk_int", flat=True)
|
||||
)
|
||||
permitted_ids = user_perm_ids.union(group_perm_ids)
|
||||
|
||||
return base_qs.filter(
|
||||
Q(owner=user) | Q(owner__isnull=True) | Q(id__in=permitted_ids),
|
||||
permitted_documents = user_perm_docs.union(group_perm_docs)
|
||||
|
||||
return base_docs.filter(
|
||||
Q(owner=user) | Q(owner__isnull=True) | Q(id__in=permitted_documents),
|
||||
).values_list("id", flat=True)
|
||||
|
||||
|
||||
def permitted_document_ids(
|
||||
user: User | None,
|
||||
*,
|
||||
perm: str = "view_document",
|
||||
include_deleted: bool = False,
|
||||
) -> QuerySet[int]:
|
||||
"""
|
||||
Document-specific convenience wrapper around ``permitted_object_ids``.
|
||||
Return a queryset of document IDs the user has ``perm`` on (default
|
||||
``"view_document"``). By default limited to non-deleted documents; pass
|
||||
``include_deleted=True`` for callers that need to check permission on
|
||||
soft-deleted documents (e.g. trash restore). This intentionally avoids
|
||||
``get_objects_for_user`` to keep the subquery small and index-friendly.
|
||||
"""
|
||||
return permitted_object_ids(user, Document, perm, include_deleted=include_deleted)
|
||||
|
||||
|
||||
def get_document_count_filter_for_user(user, related_name: str = "documents"):
|
||||
"""
|
||||
Return the Q object used to filter document counts for the given user.
|
||||
@@ -374,13 +341,6 @@ def get_objects_for_user_owner_aware(
|
||||
"""
|
||||
Returns objects the user owns, are unowned, or has explicit perms.
|
||||
When include_deleted is True, soft-deleted items are also included.
|
||||
|
||||
Legacy slow path (guardian-backed, O(n) style permission resolution).
|
||||
Most queryset-filtering call sites have migrated onto
|
||||
``PermittedObjectsFilter``/``permitted_object_ids()``, but this function
|
||||
is kept because production callers still remain. Several callers remain
|
||||
across ``documents/``, ``paperless_mail/``, and ``paperless_ai/`` --
|
||||
grep for this function name before removing it.
|
||||
"""
|
||||
manager = (
|
||||
Model.global_objects
|
||||
@@ -400,15 +360,6 @@ def get_objects_for_user_owner_aware(
|
||||
|
||||
|
||||
def has_perms_owner_aware(user, perms, obj):
|
||||
"""
|
||||
Legacy slow path (guardian-backed) single-object permission check.
|
||||
|
||||
The queryset-filtering side of this migrated onto
|
||||
``PermittedObjectsFilter``/``permitted_object_ids()``, but this
|
||||
single-object check still has many production callers. Several callers
|
||||
remain across ``documents/``, ``paperless_mail/``, and ``paperless_ai/``
|
||||
-- grep for this function name before removing it.
|
||||
"""
|
||||
checker = ObjectPermissionChecker(user)
|
||||
return obj.owner is None or obj.owner == user or checker.has_perm(perms, obj)
|
||||
|
||||
|
||||
@@ -1383,6 +1383,7 @@ class SavedViewSerializer(OwnedObjectSerializer):
|
||||
fields = [
|
||||
"id",
|
||||
"name",
|
||||
"icon",
|
||||
"sort_field",
|
||||
"sort_reverse",
|
||||
"filter_rules",
|
||||
@@ -1744,7 +1745,7 @@ class DeleteDocumentsSerializer(DocumentSelectionSerializer):
|
||||
|
||||
|
||||
class ReprocessDocumentsSerializer(DocumentSelectionSerializer):
|
||||
remote_ocr = serializers.BooleanField(required=False, default=False)
|
||||
pass
|
||||
|
||||
|
||||
class BulkEditSerializer(
|
||||
@@ -1969,8 +1970,6 @@ class BulkEditSerializer(
|
||||
return ownerUser
|
||||
|
||||
def _validate_parameters_set_permissions(self, parameters) -> None:
|
||||
if "set_permissions" not in parameters:
|
||||
raise serializers.ValidationError("set_permissions not specified")
|
||||
parameters["set_permissions"] = self.validate_set_permissions(
|
||||
parameters["set_permissions"],
|
||||
)
|
||||
@@ -2086,13 +2085,6 @@ class BulkEditSerializer(
|
||||
f"Page {op['page']} is out of bounds for document with {doc.page_count} pages.",
|
||||
)
|
||||
|
||||
def _validate_parameters_reprocess(self, parameters) -> None:
|
||||
if "remote_ocr" in parameters:
|
||||
if not isinstance(parameters["remote_ocr"], bool):
|
||||
raise serializers.ValidationError("remote_ocr must be a boolean")
|
||||
else:
|
||||
parameters["remote_ocr"] = False
|
||||
|
||||
def validate_parameters_remove_password(self, parameters):
|
||||
if "password" not in parameters:
|
||||
raise serializers.ValidationError("password not specified")
|
||||
@@ -2157,8 +2149,6 @@ class BulkEditSerializer(
|
||||
self._validate_parameters_edit_pdf(parameters, attrs["documents"][0])
|
||||
elif method == bulk_edit.remove_password:
|
||||
self.validate_parameters_remove_password(parameters)
|
||||
elif method == bulk_edit.reprocess:
|
||||
self._validate_parameters_reprocess(parameters)
|
||||
|
||||
return attrs
|
||||
|
||||
@@ -3184,9 +3174,6 @@ class WorkflowActionSerializer(serializers.ModelSerializer[WorkflowAction]):
|
||||
"email",
|
||||
"webhook",
|
||||
"passwords",
|
||||
"ai_suggestion_fields",
|
||||
"ai_create_missing",
|
||||
"ai_overwrite_existing",
|
||||
]
|
||||
|
||||
def validate(self, attrs):
|
||||
@@ -3260,23 +3247,6 @@ class WorkflowActionSerializer(serializers.ModelSerializer[WorkflowAction]):
|
||||
"Passwords are required for password removal actions",
|
||||
)
|
||||
|
||||
if (
|
||||
"type" in attrs
|
||||
and attrs["type"] == WorkflowAction.WorkflowActionType.APPLY_AI_SUGGESTIONS
|
||||
):
|
||||
fields = attrs.get("ai_suggestion_fields")
|
||||
valid_fields = set(WorkflowAction.AISuggestionField.values)
|
||||
if (
|
||||
fields is None
|
||||
or not isinstance(fields, list)
|
||||
or len(fields) == 0
|
||||
or any(field not in valid_fields for field in fields)
|
||||
):
|
||||
raise serializers.ValidationError(
|
||||
"At least one valid field is required for apply AI "
|
||||
f"suggestions actions, options are: {sorted(valid_fields)}",
|
||||
)
|
||||
|
||||
return attrs
|
||||
|
||||
|
||||
@@ -3297,40 +3267,6 @@ class WorkflowSerializer(serializers.ModelSerializer[Workflow]):
|
||||
"actions",
|
||||
]
|
||||
|
||||
def validate(self, attrs):
|
||||
attrs = super().validate(attrs)
|
||||
|
||||
triggers = attrs.get("triggers") or []
|
||||
actions = attrs.get("actions") or []
|
||||
|
||||
# Remote OCR can only work with consumption triggers
|
||||
if any(
|
||||
action.get("type") == WorkflowAction.WorkflowActionType.REMOTE_OCR
|
||||
for action in actions
|
||||
) and not any(
|
||||
trigger.get("type") == WorkflowTrigger.WorkflowTriggerType.CONSUMPTION
|
||||
for trigger in triggers
|
||||
):
|
||||
raise serializers.ValidationError(
|
||||
"Remote OCR actions require a consumption started trigger",
|
||||
)
|
||||
|
||||
# Suggestions are made from the document content, which does not exist
|
||||
# until after consumption has finished
|
||||
if any(
|
||||
action.get("type") == WorkflowAction.WorkflowActionType.APPLY_AI_SUGGESTIONS
|
||||
for action in actions
|
||||
) and not any(
|
||||
trigger.get("type") != WorkflowTrigger.WorkflowTriggerType.CONSUMPTION
|
||||
for trigger in triggers
|
||||
):
|
||||
raise serializers.ValidationError(
|
||||
"Apply AI suggestions actions require a trigger other than "
|
||||
"consumption started",
|
||||
)
|
||||
|
||||
return attrs
|
||||
|
||||
def update_triggers_and_actions(
|
||||
self,
|
||||
instance: Workflow,
|
||||
|
||||
@@ -971,39 +971,6 @@ def run_workflows(
|
||||
)
|
||||
elif action.type == WorkflowAction.WorkflowActionType.MOVE_TO_TRASH:
|
||||
has_move_to_trash_action = True
|
||||
elif action.type == WorkflowAction.WorkflowActionType.REMOTE_OCR:
|
||||
if use_overrides and overrides:
|
||||
overrides.remote_ocr = True
|
||||
else:
|
||||
# If a workflow has a consumption trigger *and* another type,
|
||||
# the document has already been parsed by the time the other one fires
|
||||
logger.debug(
|
||||
"Remote OCR action only applies to consumption "
|
||||
"triggers, ignoring",
|
||||
extra={"group": logging_group},
|
||||
)
|
||||
elif (
|
||||
action.type
|
||||
== WorkflowAction.WorkflowActionType.APPLY_AI_SUGGESTIONS
|
||||
):
|
||||
if use_overrides:
|
||||
# The document has not been parsed yet, so there is no
|
||||
# content for the LLM to make suggestions from
|
||||
logger.debug(
|
||||
"Apply AI suggestions action does not apply to "
|
||||
"consumption triggers, ignoring",
|
||||
extra={"group": logging_group},
|
||||
)
|
||||
else:
|
||||
# Queued rather than run sync
|
||||
from documents.tasks import apply_ai_suggestions
|
||||
|
||||
# kwargs so the PaperlessTask record can note the
|
||||
# document, see _extract_input_data
|
||||
apply_ai_suggestions.delay(
|
||||
action_id=action.pk,
|
||||
document_id=document.pk,
|
||||
)
|
||||
|
||||
if not use_overrides:
|
||||
# limit title to 128 characters
|
||||
@@ -1059,7 +1026,6 @@ TRACKED_TASKS: dict[str, PaperlessTask.TaskType] = {
|
||||
"documents.tasks.update_document_content_maybe_archive_file": PaperlessTask.TaskType.REPROCESS_DOCUMENT,
|
||||
"documents.tasks.build_share_link_bundle": PaperlessTask.TaskType.BUILD_SHARE_LINK,
|
||||
"documents.bulk_edit.delete": PaperlessTask.TaskType.BULK_DELETE,
|
||||
"documents.tasks.apply_ai_suggestions": PaperlessTask.TaskType.APPLY_AI_SUGGESTIONS,
|
||||
}
|
||||
|
||||
_CELERY_STATE_TO_STATUS: dict[str, PaperlessTask.Status] = {
|
||||
@@ -1113,12 +1079,6 @@ def _extract_input_data(
|
||||
return {"account_ids": account_ids}
|
||||
return {}
|
||||
|
||||
if task_type == PaperlessTask.TaskType.APPLY_AI_SUGGESTIONS:
|
||||
document_id = task_kwargs.get("document_id")
|
||||
if document_id is not None:
|
||||
return {"document_id": document_id}
|
||||
return {}
|
||||
|
||||
return {}
|
||||
|
||||
|
||||
|
||||
+1
-49
@@ -66,7 +66,6 @@ from documents.utils import compute_checksum
|
||||
from documents.utils import identity
|
||||
from documents.workflows.utils import get_workflows_for_trigger
|
||||
from paperless.config import AIConfig
|
||||
from paperless.config import RemoteOCRConfig
|
||||
from paperless.logging import consume_task_id
|
||||
from paperless.parsers import ParserContext
|
||||
from paperless.parsers.registry import get_parser_registry
|
||||
@@ -338,17 +337,10 @@ def bulk_update_documents(document_ids) -> None:
|
||||
|
||||
|
||||
@shared_task
|
||||
def update_document_content_maybe_archive_file(
|
||||
document_id,
|
||||
*,
|
||||
remote_ocr: bool = False,
|
||||
) -> None:
|
||||
def update_document_content_maybe_archive_file(document_id) -> None:
|
||||
"""
|
||||
Re-creates OCR content and thumbnail for a document, and archive file if
|
||||
it exists.
|
||||
|
||||
Remote OCR is used only when the engine is configured to handle everything
|
||||
or if explicitly asked for via ``remote_ocr``.
|
||||
"""
|
||||
document = Document.objects.get(id=document_id)
|
||||
|
||||
@@ -358,7 +350,6 @@ def update_document_content_maybe_archive_file(
|
||||
mime_type,
|
||||
document.original_filename or "",
|
||||
document.source_path,
|
||||
allow_remote=remote_ocr or RemoteOCRConfig().remote_ocr_by_default,
|
||||
)
|
||||
|
||||
if not parser_class:
|
||||
@@ -713,45 +704,6 @@ def llmindex_index(
|
||||
)
|
||||
|
||||
|
||||
@shared_task(
|
||||
bind=True,
|
||||
autoretry_for=(Exception,),
|
||||
max_retries=3,
|
||||
retry_backoff=60,
|
||||
retry_backoff_max=600,
|
||||
retry_jitter=True,
|
||||
)
|
||||
def apply_ai_suggestions(self, action_id: int, document_id: int) -> None:
|
||||
"""
|
||||
Deferred "apply AI suggestions" workflow action.
|
||||
"""
|
||||
from documents.models import WorkflowAction
|
||||
from documents.workflows.ai import apply_ai_suggestions_to_document
|
||||
|
||||
try:
|
||||
action = WorkflowAction.objects.get(pk=action_id)
|
||||
document = Document.objects.select_related("owner").get(pk=document_id)
|
||||
except (WorkflowAction.DoesNotExist, Document.DoesNotExist):
|
||||
logger.warning(
|
||||
"Workflow action %s or document %s no longer exists, "
|
||||
"not applying AI suggestions",
|
||||
action_id,
|
||||
document_id,
|
||||
)
|
||||
return
|
||||
|
||||
if not apply_ai_suggestions_to_document(action, document):
|
||||
return
|
||||
|
||||
# No document_updated signal to avoid loop
|
||||
clear_document_caches(document.pk)
|
||||
index_document.delay(document.pk)
|
||||
|
||||
ai_config = AIConfig()
|
||||
if ai_config.llm_index_enabled:
|
||||
update_document_in_llm_index.apply_async(kwargs={"document": document})
|
||||
|
||||
|
||||
@shared_task
|
||||
def update_document_in_llm_index(document) -> None:
|
||||
llm_index_add_or_update_document(document)
|
||||
|
||||
@@ -70,7 +70,8 @@
|
||||
]
|
||||
</script>
|
||||
</pngx-root>
|
||||
<script src="{% static polyfills_js %}" type="module"></script>
|
||||
<script src="{% static main_js %}" type="module"></script>
|
||||
<script src="{% static runtime_js %}" defer></script>
|
||||
<script src="{% static polyfills_js %}" defer></script>
|
||||
<script src="{% static main_js %}" defer></script>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@@ -1,327 +0,0 @@
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from pytest_django.fixtures import SettingsWrapper
|
||||
|
||||
from documents.export.sinks import DirectoryExportSink
|
||||
from documents.export.sinks import ExportSink
|
||||
from documents.export.sinks import StreamingManifestWriter
|
||||
from documents.export.sinks import ZipExportSink
|
||||
from documents.export.sinks import _dumps
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def source_file(tmp_path: Path) -> Path:
|
||||
src: Path = tmp_path / "src" / "doc.pdf"
|
||||
src.parent.mkdir(parents=True)
|
||||
src.write_bytes(b"PDF-CONTENT")
|
||||
return src
|
||||
|
||||
|
||||
class TestDumps:
|
||||
def test_dumps_is_indented_unicode_json(self) -> None:
|
||||
result: str = _dumps({"a": "é", "b": 1})
|
||||
assert '"é"' in result # ensure_ascii=False keeps unicode literal
|
||||
assert "\n" in result # indent=2 produces newlines
|
||||
assert json.loads(result) == {"a": "é", "b": 1}
|
||||
|
||||
|
||||
class TestStreamingManifestWriter:
|
||||
def test_writes_json_array_of_records(self) -> None:
|
||||
handle: io.StringIO = io.StringIO()
|
||||
writer: StreamingManifestWriter = StreamingManifestWriter(handle)
|
||||
writer.write_batch([{"pk": 1}, {"pk": 2}])
|
||||
writer.write_record({"pk": 3})
|
||||
writer.close()
|
||||
assert json.loads(handle.getvalue()) == [{"pk": 1}, {"pk": 2}, {"pk": 3}]
|
||||
|
||||
def test_empty_manifest_is_valid_empty_array(self) -> None:
|
||||
handle: io.StringIO = io.StringIO()
|
||||
writer: StreamingManifestWriter = StreamingManifestWriter(handle)
|
||||
writer.close()
|
||||
assert json.loads(handle.getvalue()) == []
|
||||
|
||||
|
||||
class TestDirectoryExportSink:
|
||||
def test_add_file_copies_to_relative_arcname(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
source_file: Path,
|
||||
) -> None:
|
||||
target: Path = tmp_path / "out"
|
||||
target.mkdir()
|
||||
with DirectoryExportSink(
|
||||
target,
|
||||
compare_checksums=False,
|
||||
compare_json=False,
|
||||
delete=False,
|
||||
) as sink:
|
||||
sink.add_file(source_file, "originals/doc.pdf")
|
||||
assert (target / "originals" / "doc.pdf").read_bytes() == b"PDF-CONTENT"
|
||||
|
||||
def test_add_json_writes_file(self, tmp_path: Path) -> None:
|
||||
target: Path = tmp_path / "out"
|
||||
target.mkdir()
|
||||
with DirectoryExportSink(
|
||||
target,
|
||||
compare_checksums=False,
|
||||
compare_json=False,
|
||||
delete=False,
|
||||
) as sink:
|
||||
sink.add_json({"version": "x"}, "metadata.json")
|
||||
assert json.loads((target / "metadata.json").read_text()) == {"version": "x"}
|
||||
|
||||
def test_stream_writes_manifest(self, tmp_path: Path) -> None:
|
||||
target: Path = tmp_path / "out"
|
||||
target.mkdir()
|
||||
with DirectoryExportSink(
|
||||
target,
|
||||
compare_checksums=False,
|
||||
compare_json=False,
|
||||
delete=False,
|
||||
) as sink:
|
||||
with sink.stream("manifest.json") as handle:
|
||||
writer: StreamingManifestWriter = StreamingManifestWriter(handle)
|
||||
writer.write_record({"pk": 1})
|
||||
writer.close()
|
||||
assert json.loads((target / "manifest.json").read_text()) == [{"pk": 1}]
|
||||
|
||||
def test_add_file_skips_when_size_and_mtime_match(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
source_file: Path,
|
||||
) -> None:
|
||||
# Pre-existing target with identical size+mtime but DIFFERENT content:
|
||||
# if add_file skips (no compare-checksums), the old content survives.
|
||||
target: Path = tmp_path / "out"
|
||||
target.mkdir()
|
||||
existing: Path = target / "originals" / "doc.pdf"
|
||||
existing.parent.mkdir(parents=True)
|
||||
# Same byte length as the source but different content + matching mtime,
|
||||
# so a size/mtime comparison treats it as unchanged and skips the copy.
|
||||
existing.write_bytes(b"X" * len(b"PDF-CONTENT"))
|
||||
stat = source_file.stat()
|
||||
os.utime(existing, (stat.st_atime, stat.st_mtime))
|
||||
with DirectoryExportSink(
|
||||
target,
|
||||
compare_checksums=False,
|
||||
compare_json=False,
|
||||
delete=False,
|
||||
) as sink:
|
||||
sink.add_file(source_file, "originals/doc.pdf", checksum="abc")
|
||||
assert existing.read_bytes() == b"X" * len(b"PDF-CONTENT") # skipped
|
||||
|
||||
def test_add_file_recopies_when_compare_checksums_differ(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
source_file: Path,
|
||||
) -> None:
|
||||
target: Path = tmp_path / "out"
|
||||
target.mkdir()
|
||||
existing: Path = target / "originals" / "doc.pdf"
|
||||
existing.parent.mkdir(parents=True)
|
||||
existing.write_bytes(b"X" * len(b"PDF-CONTENT"))
|
||||
stat = source_file.stat()
|
||||
os.utime(existing, (stat.st_atime, stat.st_mtime))
|
||||
with DirectoryExportSink(
|
||||
target,
|
||||
compare_checksums=True,
|
||||
compare_json=False,
|
||||
delete=False,
|
||||
) as sink:
|
||||
# wrong checksum forces recopy despite matching size/mtime
|
||||
sink.add_file(source_file, "originals/doc.pdf", checksum="not-the-real-sum")
|
||||
assert existing.read_bytes() == b"PDF-CONTENT" # recopied
|
||||
|
||||
def test_delete_prunes_unwritten_snapshot_files(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
source_file: Path,
|
||||
) -> None:
|
||||
target: Path = tmp_path / "out"
|
||||
target.mkdir()
|
||||
stale: Path = target / "stale.pdf"
|
||||
stale.write_bytes(b"STALE")
|
||||
with DirectoryExportSink(
|
||||
target,
|
||||
compare_checksums=False,
|
||||
compare_json=False,
|
||||
delete=True,
|
||||
) as sink:
|
||||
sink.add_file(source_file, "originals/doc.pdf")
|
||||
assert not stale.exists()
|
||||
assert (target / "originals" / "doc.pdf").exists()
|
||||
|
||||
def test_no_delete_keeps_unwritten_files(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
source_file: Path,
|
||||
) -> None:
|
||||
target: Path = tmp_path / "out"
|
||||
target.mkdir()
|
||||
stale: Path = target / "stale.pdf"
|
||||
stale.write_bytes(b"STALE")
|
||||
with DirectoryExportSink(
|
||||
target,
|
||||
compare_checksums=False,
|
||||
compare_json=False,
|
||||
delete=False,
|
||||
) as sink:
|
||||
sink.add_file(source_file, "originals/doc.pdf")
|
||||
assert stale.exists()
|
||||
|
||||
|
||||
class TestZipExportSink:
|
||||
def test_round_trip_files_json_and_stream(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
source_file: Path,
|
||||
) -> None:
|
||||
target: Path = tmp_path / "out"
|
||||
target.mkdir()
|
||||
with ZipExportSink(target, "export", delete=False) as sink:
|
||||
sink.add_file(source_file, "originals/doc.pdf")
|
||||
sink.add_json({"version": "x"}, "metadata.json")
|
||||
with sink.stream("manifest.json") as handle:
|
||||
writer = StreamingManifestWriter(handle)
|
||||
writer.write_record({"pk": 1})
|
||||
writer.close()
|
||||
zip_path: Path = target / "export.zip"
|
||||
assert zip_path.exists()
|
||||
assert not (target / "export.zip.tmp").exists()
|
||||
with zipfile.ZipFile(zip_path) as zf:
|
||||
names = set(zf.namelist())
|
||||
assert {"originals/doc.pdf", "metadata.json", "manifest.json"} <= names
|
||||
assert zf.read("originals/doc.pdf") == b"PDF-CONTENT"
|
||||
assert json.loads(zf.read("manifest.json")) == [{"pk": 1}]
|
||||
|
||||
def test_nested_arcname_emits_directory_marker(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
source_file: Path,
|
||||
) -> None:
|
||||
target: Path = tmp_path / "out"
|
||||
target.mkdir()
|
||||
with ZipExportSink(target, "export", delete=False) as sink:
|
||||
sink.add_file(source_file, "originals/doc.pdf")
|
||||
with zipfile.ZipFile(target / "export.zip") as zf:
|
||||
assert "originals/" in zf.namelist()
|
||||
|
||||
def test_flat_arcname_has_no_directory_markers(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
source_file: Path,
|
||||
) -> None:
|
||||
target: Path = tmp_path / "out"
|
||||
target.mkdir()
|
||||
with ZipExportSink(target, "export", delete=False) as sink:
|
||||
sink.add_file(source_file, "doc.pdf")
|
||||
with zipfile.ZipFile(target / "export.zip") as zf:
|
||||
assert all(not n.endswith("/") for n in zf.namelist())
|
||||
|
||||
def test_exception_leaves_no_zip_and_no_tmp(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
source_file: Path,
|
||||
) -> None:
|
||||
target: Path = tmp_path / "out"
|
||||
target.mkdir()
|
||||
with pytest.raises(RuntimeError):
|
||||
with ZipExportSink(target, "export", delete=False) as sink:
|
||||
sink.add_file(source_file, "doc.pdf")
|
||||
raise RuntimeError("boom")
|
||||
assert not (target / "export.zip").exists()
|
||||
assert not (target / "export.zip.tmp").exists()
|
||||
|
||||
def test_exception_inside_stream_cleans_up_manifest_tmp(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
source_file: Path,
|
||||
settings: SettingsWrapper,
|
||||
) -> None:
|
||||
scratch_dir = tmp_path / "scratch"
|
||||
settings.SCRATCH_DIR = scratch_dir
|
||||
target: Path = tmp_path / "out"
|
||||
target.mkdir()
|
||||
with pytest.raises(RuntimeError):
|
||||
with ZipExportSink(target, "export", delete=False) as sink:
|
||||
sink.add_file(source_file, "doc.pdf")
|
||||
with sink.stream("manifest.json") as handle:
|
||||
handle.write("[")
|
||||
raise RuntimeError("boom")
|
||||
assert list(scratch_dir.glob("export-manifest-*")) == []
|
||||
assert not (target / "export.zip").exists()
|
||||
assert not (target / "export.zip.tmp").exists()
|
||||
|
||||
def test_abort_after_manifest_written_cleans_up_pending_tmp(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
settings: SettingsWrapper,
|
||||
) -> None:
|
||||
scratch_dir = tmp_path / "scratch"
|
||||
settings.SCRATCH_DIR = scratch_dir
|
||||
target: Path = tmp_path / "out"
|
||||
target.mkdir()
|
||||
with pytest.raises(RuntimeError):
|
||||
with ZipExportSink(target, "export", delete=False) as sink:
|
||||
with sink.stream("manifest.json") as handle:
|
||||
handle.write("[]")
|
||||
raise RuntimeError("boom")
|
||||
assert list(scratch_dir.glob("export-manifest-*")) == []
|
||||
assert not (target / "export.zip").exists()
|
||||
|
||||
def test_delete_wipes_destination_on_success(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
source_file: Path,
|
||||
) -> None:
|
||||
target: Path = tmp_path / "out"
|
||||
target.mkdir()
|
||||
(target / "preexisting.txt").write_text("old")
|
||||
(target / "olddir").mkdir()
|
||||
with ZipExportSink(target, "export", delete=True) as sink:
|
||||
sink.add_file(source_file, "doc.pdf")
|
||||
assert (target / "export.zip").exists()
|
||||
assert not (target / "preexisting.txt").exists()
|
||||
assert not (target / "olddir").exists()
|
||||
|
||||
def test_abort_with_delete_does_not_wipe_destination(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
source_file: Path,
|
||||
) -> None:
|
||||
target: Path = tmp_path / "out"
|
||||
target.mkdir()
|
||||
(target / "preexisting.txt").write_text("old")
|
||||
with pytest.raises(RuntimeError):
|
||||
with ZipExportSink(target, "export", delete=True) as sink:
|
||||
sink.add_file(source_file, "doc.pdf")
|
||||
raise RuntimeError("boom")
|
||||
assert (target / "preexisting.txt").exists()
|
||||
assert not (target / "export.zip").exists()
|
||||
|
||||
|
||||
class TestStreamContract:
|
||||
@pytest.fixture(params=["dir", "zip"])
|
||||
def sink(self, request: pytest.FixtureRequest, tmp_path: Path) -> ExportSink:
|
||||
target: Path = tmp_path / "out"
|
||||
target.mkdir()
|
||||
if request.param == "dir":
|
||||
return DirectoryExportSink(
|
||||
target,
|
||||
compare_checksums=False,
|
||||
compare_json=False,
|
||||
delete=False,
|
||||
)
|
||||
return ZipExportSink(target, "export", delete=False)
|
||||
|
||||
def test_second_concurrent_stream_is_rejected(self, sink: ExportSink) -> None:
|
||||
with sink:
|
||||
with sink.stream("manifest.json"):
|
||||
with pytest.raises(RuntimeError, match="already open"):
|
||||
with sink.stream("other.json"):
|
||||
pass
|
||||
@@ -12,7 +12,6 @@ from rest_framework import status
|
||||
from rest_framework.test import APITestCase
|
||||
|
||||
from documents.tests.utils import DirectoriesMixin
|
||||
from documents.tests.utils import read_streaming_response
|
||||
from paperless.models import ApplicationConfiguration
|
||||
from paperless.models import ColorConvertChoices
|
||||
|
||||
@@ -72,10 +71,6 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
|
||||
"barcode_enable_tag": None,
|
||||
"barcode_tag_mapping": None,
|
||||
"barcode_tag_split": None,
|
||||
"remote_ocr_engine": None,
|
||||
"remote_ocr_api_key": None,
|
||||
"remote_ocr_endpoint": None,
|
||||
"remote_ocr_mode": None,
|
||||
"ai_enabled": False,
|
||||
"llm_embedding_backend": None,
|
||||
"llm_embedding_model": None,
|
||||
@@ -198,7 +193,6 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
|
||||
response = self.client.get("/logo/simple.jpg")
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
self.assertIn("image/jpeg", response["Content-Type"])
|
||||
response.close()
|
||||
|
||||
config = ApplicationConfiguration.objects.first()
|
||||
assert config is not None
|
||||
@@ -218,46 +212,6 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
|
||||
)
|
||||
self.assertFalse(Path(old_logo.path).exists())
|
||||
|
||||
@override_settings(APP_LOGO="/logo/simple.jpg")
|
||||
def test_serve_app_logo_from_environment_setting(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- No uploaded app logo
|
||||
- PAPERLESS_APP_LOGO points to a file in the media logo directory
|
||||
WHEN:
|
||||
- The configured logo URL is requested
|
||||
THEN:
|
||||
- The environment-configured logo is served
|
||||
"""
|
||||
logo = self.dirs.media_dir / "logo" / "simple.jpg"
|
||||
logo.parent.mkdir()
|
||||
expected_content = (
|
||||
Path(__file__).parent / "samples" / "simple.jpg"
|
||||
).read_bytes()
|
||||
logo.write_bytes(expected_content)
|
||||
|
||||
response = self.client.get("/logo/simple.jpg")
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
self.assertIn("image/jpeg", response["Content-Type"])
|
||||
self.assertEqual(read_streaming_response(response), expected_content)
|
||||
|
||||
@override_settings(APP_LOGO="/logo/../outside-logo.jpg")
|
||||
def test_environment_app_logo_must_be_inside_logo_directory(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- PAPERLESS_APP_LOGO resolves outside the media logo directory
|
||||
WHEN:
|
||||
- The configured logo URL is requested
|
||||
THEN:
|
||||
- The file is not served
|
||||
"""
|
||||
(self.dirs.media_dir / "outside-logo.jpg").write_bytes(b"not a logo")
|
||||
|
||||
response = self.client.get("/logo/outside-logo.jpg")
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
|
||||
|
||||
def test_api_strips_exif_data_from_uploaded_logo(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
@@ -874,49 +828,6 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
|
||||
config.refresh_from_db()
|
||||
self.assertEqual(config.llm_api_key, None)
|
||||
|
||||
def test_update_remote_ocr_api_key(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- Existing config with remote_ocr_api_key specified
|
||||
WHEN:
|
||||
- API to update remote_ocr_api_key is called with all *s
|
||||
- API to update remote_ocr_api_key is called with empty string
|
||||
THEN:
|
||||
- remote_ocr_api_key is unchanged
|
||||
- remote_ocr_api_key is set to None
|
||||
"""
|
||||
config = ApplicationConfiguration.objects.first()
|
||||
assert config is not None
|
||||
config.remote_ocr_api_key = "1234567890"
|
||||
config.save()
|
||||
|
||||
# Test with all *
|
||||
response = self.client.patch(
|
||||
f"{self.ENDPOINT}1/",
|
||||
json.dumps(
|
||||
{
|
||||
"remote_ocr_api_key": "*" * 32,
|
||||
},
|
||||
),
|
||||
content_type="application/json",
|
||||
)
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
config.refresh_from_db()
|
||||
self.assertEqual(config.remote_ocr_api_key, "1234567890")
|
||||
# Test with empty string
|
||||
response = self.client.patch(
|
||||
f"{self.ENDPOINT}1/",
|
||||
json.dumps(
|
||||
{
|
||||
"remote_ocr_api_key": "",
|
||||
},
|
||||
),
|
||||
content_type="application/json",
|
||||
)
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
config.refresh_from_db()
|
||||
self.assertEqual(config.remote_ocr_api_key, None)
|
||||
|
||||
def test_enable_ai_index_triggers_update(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
|
||||
@@ -532,29 +532,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
|
||||
m.assert_called_once()
|
||||
args, kwargs = m.call_args
|
||||
self.assertEqual(args[0], [self.doc1.id])
|
||||
self.assertEqual(kwargs, {"remote_ocr": False})
|
||||
|
||||
@mock.patch("documents.views.bulk_edit.reprocess")
|
||||
def test_reprocess_documents_endpoint_remote_ocr(self, m) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- API data to reprocess a document with remote OCR requested
|
||||
WHEN:
|
||||
- API is called
|
||||
THEN:
|
||||
- reprocess is called with remote_ocr=True
|
||||
"""
|
||||
self.setup_mock(m, "reprocess")
|
||||
response = self.client.post(
|
||||
"/api/documents/reprocess/",
|
||||
json.dumps({"documents": [self.doc1.id], "remote_ocr": True}),
|
||||
content_type="application/json",
|
||||
)
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
m.assert_called_once()
|
||||
args, kwargs = m.call_args
|
||||
self.assertEqual(args[0], [self.doc1.id])
|
||||
self.assertEqual(kwargs, {"remote_ocr": True})
|
||||
self.assertEqual(len(kwargs), 0)
|
||||
|
||||
@mock.patch("documents.serialisers.bulk_edit.set_storage_path")
|
||||
def test_api_set_storage_path(self, m) -> None:
|
||||
@@ -1090,30 +1068,6 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
|
||||
self.assertCountEqual(args[0], [self.doc2.id, self.doc3.id])
|
||||
self.assertEqual(len(kwargs["set_permissions"]["view"]["users"]), 2)
|
||||
|
||||
@mock.patch("documents.serialisers.bulk_edit.set_permissions")
|
||||
def test_set_permissions_requires_set_permissions_parameter(self, m) -> None:
|
||||
self.setup_mock(m, "set_permissions")
|
||||
|
||||
response = self.client.post(
|
||||
"/api/documents/bulk_edit/",
|
||||
json.dumps(
|
||||
{
|
||||
"documents": [self.doc2.id],
|
||||
"method": "set_permissions",
|
||||
"parameters": {
|
||||
"owner": self.user.id,
|
||||
"merge": True,
|
||||
"permissions": {"view": {"users": [self.user.id]}},
|
||||
},
|
||||
},
|
||||
),
|
||||
content_type="application/json",
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
|
||||
self.assertIn(b"set_permissions not specified", response.content)
|
||||
m.assert_not_called()
|
||||
|
||||
@mock.patch("documents.serialisers.bulk_edit.set_permissions")
|
||||
def test_set_permissions_merge(self, m) -> None:
|
||||
self.setup_mock(m, "set_permissions")
|
||||
@@ -1575,29 +1529,6 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
|
||||
),
|
||||
)
|
||||
|
||||
def test_legacy_bulk_edit_reprocess_invalid_remote_ocr(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- The deprecated bulk_edit endpoint with a non-boolean remote_ocr
|
||||
WHEN:
|
||||
- API is called
|
||||
THEN:
|
||||
- The request is rejected rather than passed through to the task
|
||||
"""
|
||||
response = self.client.post(
|
||||
"/api/documents/bulk_edit/",
|
||||
json.dumps(
|
||||
{
|
||||
"documents": [self.doc1.id],
|
||||
"method": "reprocess",
|
||||
"parameters": {"remote_ocr": "yes please"},
|
||||
},
|
||||
),
|
||||
content_type="application/json",
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
|
||||
|
||||
@mock.patch("documents.views.bulk_edit.edit_pdf")
|
||||
def test_edit_pdf(self, m) -> None:
|
||||
self.setup_mock(m, "edit_pdf")
|
||||
|
||||
@@ -2905,18 +2905,20 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
|
||||
|
||||
v1 = SavedView.objects.get(name="test")
|
||||
self.assertEqual(v1.sort_field, "created2")
|
||||
self.assertEqual(v1.icon, SavedView.Icon.FUNNEL)
|
||||
self.assertEqual(v1.filter_rules.count(), 1)
|
||||
self.assertEqual(v1.owner, self.user)
|
||||
|
||||
response = self.client.patch(
|
||||
f"/api/saved_views/{v1.id}/",
|
||||
{"sort_reverse": True},
|
||||
{"sort_reverse": True, "icon": SavedView.Icon.RECEIPT},
|
||||
format="json",
|
||||
)
|
||||
|
||||
v1 = SavedView.objects.get(id=v1.id)
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
self.assertTrue(v1.sort_reverse)
|
||||
self.assertEqual(v1.icon, SavedView.Icon.RECEIPT)
|
||||
self.assertEqual(v1.filter_rules.count(), 1)
|
||||
|
||||
view["filter_rules"] = [{"rule_type": 12, "value": "secret"}]
|
||||
@@ -2936,6 +2938,13 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
|
||||
v1 = SavedView.objects.get(id=v1.id)
|
||||
self.assertEqual(v1.filter_rules.count(), 0)
|
||||
|
||||
response = self.client.patch(
|
||||
f"/api/saved_views/{v1.id}/",
|
||||
{"icon": "not-an-icon"},
|
||||
format="json",
|
||||
)
|
||||
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
|
||||
|
||||
def test_saved_view_display_options(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
|
||||
@@ -1057,52 +1057,33 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
|
||||
THEN:
|
||||
- The similar documents are returned from the API request
|
||||
"""
|
||||
# Distinct created/added/modified dates: documents sharing a timestamp
|
||||
# term (down to the second) would be matched on it by more_like_this
|
||||
# (which cannot be scoped to content fields), surfacing unrelated
|
||||
# documents. `modified` is auto_now, so it can't be set via factory
|
||||
# kwargs like created/added - freeze time per document instead so all
|
||||
# three date fields land on distinct seconds.
|
||||
with time_machine.travel(
|
||||
timezone.make_aware(datetime.datetime(2018, 1, 1)),
|
||||
tick=False,
|
||||
):
|
||||
d1 = DocumentFactory(
|
||||
title="invoice",
|
||||
content="the thing i bought at a shop and paid with bank account",
|
||||
created=datetime.date(2018, 1, 1),
|
||||
added=timezone.make_aware(datetime.datetime(2018, 1, 1)),
|
||||
)
|
||||
with time_machine.travel(
|
||||
timezone.make_aware(datetime.datetime(2019, 3, 4)),
|
||||
tick=False,
|
||||
):
|
||||
d2 = DocumentFactory(
|
||||
title="bank statement 1",
|
||||
content="things i paid for in august",
|
||||
created=datetime.date(2019, 3, 4),
|
||||
added=timezone.make_aware(datetime.datetime(2019, 3, 4)),
|
||||
)
|
||||
with time_machine.travel(
|
||||
timezone.make_aware(datetime.datetime(2020, 7, 9)),
|
||||
tick=False,
|
||||
):
|
||||
d3 = DocumentFactory(
|
||||
title="bank statement 3",
|
||||
content="things i paid for in september",
|
||||
created=datetime.date(2020, 7, 9),
|
||||
added=timezone.make_aware(datetime.datetime(2020, 7, 9)),
|
||||
)
|
||||
with time_machine.travel(
|
||||
timezone.make_aware(datetime.datetime(2021, 11, 30)),
|
||||
tick=False,
|
||||
):
|
||||
d4 = DocumentFactory(
|
||||
title="Quarterly Report",
|
||||
content="quarterly revenue profit margin earnings growth",
|
||||
created=datetime.date(2021, 11, 30),
|
||||
added=timezone.make_aware(datetime.datetime(2021, 11, 30)),
|
||||
)
|
||||
# Distinct created/added dates: documents created at the same instant
|
||||
# share a timestamp term, and more_like_this (which cannot be scoped to
|
||||
# content fields) would then match on it, surfacing unrelated documents.
|
||||
d1 = DocumentFactory(
|
||||
title="invoice",
|
||||
content="the thing i bought at a shop and paid with bank account",
|
||||
created=datetime.date(2018, 1, 1),
|
||||
added=timezone.make_aware(datetime.datetime(2018, 1, 1)),
|
||||
)
|
||||
d2 = DocumentFactory(
|
||||
title="bank statement 1",
|
||||
content="things i paid for in august",
|
||||
created=datetime.date(2019, 3, 4),
|
||||
added=timezone.make_aware(datetime.datetime(2019, 3, 4)),
|
||||
)
|
||||
d3 = DocumentFactory(
|
||||
title="bank statement 3",
|
||||
content="things i paid for in september",
|
||||
created=datetime.date(2020, 7, 9),
|
||||
added=timezone.make_aware(datetime.datetime(2020, 7, 9)),
|
||||
)
|
||||
d4 = DocumentFactory(
|
||||
title="Quarterly Report",
|
||||
content="quarterly revenue profit margin earnings growth",
|
||||
created=datetime.date(2021, 11, 30),
|
||||
added=timezone.make_aware(datetime.datetime(2021, 11, 30)),
|
||||
)
|
||||
backend = get_backend()
|
||||
backend.add_or_update(d1)
|
||||
backend.add_or_update(d2)
|
||||
|
||||
@@ -60,10 +60,6 @@ class TestApiUiSettings(DirectoriesMixin, APITestCase):
|
||||
},
|
||||
"email_enabled": False,
|
||||
"ai_enabled": False,
|
||||
"remote_ocr": {
|
||||
"configured": False,
|
||||
"mode": "always",
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
@@ -158,50 +154,6 @@ class TestApiUiSettings(DirectoriesMixin, APITestCase):
|
||||
str(response.data["settings"]),
|
||||
)
|
||||
|
||||
@override_settings(
|
||||
REMOTE_OCR_ENGINE="azureai",
|
||||
REMOTE_OCR_API_KEY="somekey",
|
||||
REMOTE_OCR_ENDPOINT="https://example.cognitiveservices.azure.com",
|
||||
REMOTE_OCR_MODE="workflow_only",
|
||||
)
|
||||
def test_settings_reports_remote_ocr_when_configured(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A fully configured remote OCR engine in workflow_only mode
|
||||
WHEN:
|
||||
- The ui_settings endpoint is called
|
||||
THEN:
|
||||
- The UI is told remote OCR is available and selective, so it can
|
||||
offer it where it would actually change something
|
||||
"""
|
||||
response = self.client.get(self.ENDPOINT, format="json")
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
self.assertEqual(
|
||||
response.data["settings"]["remote_ocr"],
|
||||
{"configured": True, "mode": "workflow_only"},
|
||||
)
|
||||
|
||||
@override_settings(
|
||||
REMOTE_OCR_ENGINE="azureai",
|
||||
REMOTE_OCR_API_KEY=None,
|
||||
REMOTE_OCR_ENDPOINT=None,
|
||||
)
|
||||
def test_settings_reports_remote_ocr_incompletely_configured(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- An engine named but missing its endpoint and API key
|
||||
WHEN:
|
||||
- The ui_settings endpoint is called
|
||||
THEN:
|
||||
- It is reported as not configured, matching what the parser
|
||||
registry will actually do
|
||||
"""
|
||||
response = self.client.get(self.ENDPOINT, format="json")
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
self.assertFalse(response.data["settings"]["remote_ocr"]["configured"])
|
||||
|
||||
@override_settings(
|
||||
OAUTH_CALLBACK_BASE_URL="http://localhost:8000",
|
||||
GMAIL_OAUTH_CLIENT_ID="abc123",
|
||||
|
||||
@@ -390,221 +390,6 @@ class TestApiWorkflows(DirectoriesMixin, APITestCase):
|
||||
|
||||
self.assertEqual(Workflow.objects.count(), 1)
|
||||
|
||||
def test_api_create_remote_ocr_action_requires_consumption_trigger(
|
||||
self,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- API request to create a workflow with a remote OCR action
|
||||
- No consumption started trigger, so the action could never run
|
||||
WHEN:
|
||||
- API is called
|
||||
THEN:
|
||||
- Correct HTTP 400 response
|
||||
- No objects are created
|
||||
"""
|
||||
existing_count = Workflow.objects.count()
|
||||
|
||||
response = self.client.post(
|
||||
self.ENDPOINT,
|
||||
json.dumps(
|
||||
{
|
||||
"name": "Remote OCR too late",
|
||||
"order": 1,
|
||||
"triggers": [
|
||||
{
|
||||
"type": WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED,
|
||||
},
|
||||
],
|
||||
"actions": [
|
||||
{
|
||||
"type": WorkflowAction.WorkflowActionType.REMOTE_OCR,
|
||||
},
|
||||
],
|
||||
},
|
||||
),
|
||||
content_type="application/json",
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
|
||||
self.assertEqual(Workflow.objects.count(), existing_count)
|
||||
|
||||
def test_api_create_remote_ocr_action_with_consumption_trigger(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- API request to create a workflow with a remote OCR action
|
||||
- A consumption started trigger alongside another trigger type
|
||||
WHEN:
|
||||
- API is called
|
||||
THEN:
|
||||
- The workflow is created, the action applies to consumption only
|
||||
"""
|
||||
response = self.client.post(
|
||||
self.ENDPOINT,
|
||||
json.dumps(
|
||||
{
|
||||
"name": "Remote OCR on consume",
|
||||
"order": 1,
|
||||
"triggers": [
|
||||
{
|
||||
"type": WorkflowTrigger.WorkflowTriggerType.CONSUMPTION,
|
||||
"filter_filename": "*.pdf",
|
||||
},
|
||||
{
|
||||
"type": WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED,
|
||||
},
|
||||
],
|
||||
"actions": [
|
||||
{
|
||||
"type": WorkflowAction.WorkflowActionType.REMOTE_OCR,
|
||||
},
|
||||
],
|
||||
},
|
||||
),
|
||||
content_type="application/json",
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
|
||||
|
||||
def _post_ai_suggestions_workflow(self, *, trigger_types, action: dict):
|
||||
def trigger(trigger_type):
|
||||
# consumption triggers require a filter of their own
|
||||
if trigger_type == WorkflowTrigger.WorkflowTriggerType.CONSUMPTION:
|
||||
return {"type": trigger_type, "filter_filename": "*.pdf"}
|
||||
return {"type": trigger_type}
|
||||
|
||||
return self.client.post(
|
||||
self.ENDPOINT,
|
||||
json.dumps(
|
||||
{
|
||||
"name": "Apply AI suggestions",
|
||||
"order": 1,
|
||||
"triggers": [trigger(t) for t in trigger_types],
|
||||
"actions": [
|
||||
{
|
||||
"type": WorkflowAction.WorkflowActionType.APPLY_AI_SUGGESTIONS,
|
||||
**action,
|
||||
},
|
||||
],
|
||||
},
|
||||
),
|
||||
content_type="application/json",
|
||||
)
|
||||
|
||||
def test_api_create_apply_ai_suggestions_action(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- API request to create a workflow with an apply AI suggestions
|
||||
action and a valid set of fields
|
||||
WHEN:
|
||||
- API is called
|
||||
THEN:
|
||||
- The workflow is created with the chosen options
|
||||
"""
|
||||
response = self._post_ai_suggestions_workflow(
|
||||
trigger_types=[WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED],
|
||||
action={
|
||||
"ai_suggestion_fields": ["title", "tags", "correspondent"],
|
||||
"ai_create_missing": True,
|
||||
"ai_overwrite_existing": True,
|
||||
},
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
|
||||
action = Workflow.objects.get(name="Apply AI suggestions").actions.first()
|
||||
self.assertEqual(
|
||||
action.ai_suggestion_fields,
|
||||
["title", "tags", "correspondent"],
|
||||
)
|
||||
self.assertTrue(action.ai_create_missing)
|
||||
self.assertTrue(action.ai_overwrite_existing)
|
||||
|
||||
def test_api_create_apply_ai_suggestions_action_requires_fields(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- API request to create an apply AI suggestions action with no
|
||||
fields selected, which could never do anything
|
||||
WHEN:
|
||||
- API is called
|
||||
THEN:
|
||||
- Correct HTTP 400 response
|
||||
- No objects are created
|
||||
"""
|
||||
existing_count = Workflow.objects.count()
|
||||
|
||||
response = self._post_ai_suggestions_workflow(
|
||||
trigger_types=[WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED],
|
||||
action={"ai_suggestion_fields": []},
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
|
||||
self.assertEqual(Workflow.objects.count(), existing_count)
|
||||
|
||||
def test_api_create_apply_ai_suggestions_action_rejects_unknown_field(
|
||||
self,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- API request to create an apply AI suggestions action naming a
|
||||
field that does not exist
|
||||
WHEN:
|
||||
- API is called
|
||||
THEN:
|
||||
- Correct HTTP 400 response
|
||||
"""
|
||||
response = self._post_ai_suggestions_workflow(
|
||||
trigger_types=[WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED],
|
||||
action={"ai_suggestion_fields": ["title", "not_a_field"]},
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
|
||||
|
||||
def test_api_create_apply_ai_suggestions_action_rejects_consumption_only(
|
||||
self,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- API request to create an apply AI suggestions action whose only
|
||||
trigger is consumption started, so there is no document content
|
||||
to make suggestions from yet
|
||||
WHEN:
|
||||
- API is called
|
||||
THEN:
|
||||
- Correct HTTP 400 response
|
||||
- No objects are created
|
||||
"""
|
||||
existing_count = Workflow.objects.count()
|
||||
|
||||
response = self._post_ai_suggestions_workflow(
|
||||
trigger_types=[WorkflowTrigger.WorkflowTriggerType.CONSUMPTION],
|
||||
action={"ai_suggestion_fields": ["title"]},
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
|
||||
self.assertEqual(Workflow.objects.count(), existing_count)
|
||||
|
||||
def test_api_create_apply_ai_suggestions_action_allows_extra_consumption_trigger(
|
||||
self,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- API request to create an apply AI suggestions action with a
|
||||
consumption trigger alongside a usable one
|
||||
WHEN:
|
||||
- API is called
|
||||
THEN:
|
||||
- The workflow is created, the action applies to the other trigger
|
||||
"""
|
||||
response = self._post_ai_suggestions_workflow(
|
||||
trigger_types=[
|
||||
WorkflowTrigger.WorkflowTriggerType.CONSUMPTION,
|
||||
WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED,
|
||||
],
|
||||
action={"ai_suggestion_fields": ["title"]},
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
|
||||
|
||||
def test_api_create_workflow_trigger_action_empty_fields(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
|
||||
@@ -1782,56 +1782,3 @@ class TestPDFActions(DirectoriesMixin, TestCase):
|
||||
|
||||
self.assertIn("wrong password", str(exc.exception))
|
||||
self.assertIn("Error removing password from document", cm.output[0])
|
||||
|
||||
|
||||
class TestBulkEditReprocess(DirectoriesMixin, TestCase):
|
||||
def setUp(self) -> None:
|
||||
super().setUp()
|
||||
|
||||
self.doc = Document.objects.create(
|
||||
title="test",
|
||||
checksum="A",
|
||||
mime_type="application/pdf",
|
||||
)
|
||||
|
||||
@mock.patch("documents.bulk_edit.update_document_content_maybe_archive_file")
|
||||
def test_reprocess_defaults_to_local(self, mock_task: mock.Mock) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A reprocess request that says nothing about remote OCR
|
||||
WHEN:
|
||||
- reprocess is called
|
||||
THEN:
|
||||
- The task is queued without asking for the remote engine
|
||||
"""
|
||||
result = bulk_edit.reprocess([self.doc.id])
|
||||
|
||||
self.assertEqual(result, "OK")
|
||||
mock_task.apply_async.assert_called_once()
|
||||
_, kwargs = mock_task.apply_async.call_args
|
||||
self.assertEqual(
|
||||
kwargs["kwargs"],
|
||||
{"document_id": self.doc.id, "remote_ocr": False},
|
||||
)
|
||||
|
||||
@mock.patch("documents.bulk_edit.update_document_content_maybe_archive_file")
|
||||
def test_reprocess_passes_remote_ocr(self, mock_task: mock.Mock) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A reprocess request that explicitly asks for remote OCR
|
||||
WHEN:
|
||||
- reprocess is called
|
||||
THEN:
|
||||
- The request is forwarded to the task for every document
|
||||
"""
|
||||
other = Document.objects.create(
|
||||
title="test2",
|
||||
checksum="B",
|
||||
mime_type="application/pdf",
|
||||
)
|
||||
|
||||
bulk_edit.reprocess([self.doc.id, other.id], remote_ocr=True)
|
||||
|
||||
self.assertEqual(mock_task.apply_async.call_count, 2)
|
||||
for call in mock_task.apply_async.call_args_list:
|
||||
self.assertTrue(call.kwargs["kwargs"]["remote_ocr"])
|
||||
|
||||
@@ -1559,72 +1559,6 @@ class PostConsumeTestCase(DirectoriesMixin, GetConsumerMixin, TestCase):
|
||||
consumer.run_post_consume_script(doc)
|
||||
|
||||
|
||||
class TestConsumerRemoteOCR(
|
||||
DirectoriesMixin,
|
||||
FileSystemAssertsMixin,
|
||||
GetConsumerMixin,
|
||||
TestCase,
|
||||
):
|
||||
"""
|
||||
The consumer resolves the remote OCR mode and the per-document request from
|
||||
workflows into the allow_remote flag it hands to the parser registry.
|
||||
"""
|
||||
|
||||
def setUp(self) -> None:
|
||||
super().setUp()
|
||||
|
||||
patcher = mock.patch("documents.consumer.get_parser_registry")
|
||||
self.mock_registry = patcher.start()
|
||||
self.mock_registry.return_value.get_parser_for_file.return_value = DummyParser
|
||||
self.addCleanup(patcher.stop)
|
||||
|
||||
def _consume(self, *, overrides: DocumentMetadataOverrides | None = None) -> bool:
|
||||
src = (
|
||||
Path(__file__).parent
|
||||
/ "samples"
|
||||
/ "documents"
|
||||
/ "originals"
|
||||
/ "0000001.pdf"
|
||||
)
|
||||
dst = self.dirs.scratch_dir / "sample.pdf"
|
||||
shutil.copy(src, dst)
|
||||
|
||||
with self.get_consumer(dst, overrides=overrides) as consumer:
|
||||
consumer.run()
|
||||
|
||||
_, kwargs = self.mock_registry.return_value.get_parser_for_file.call_args
|
||||
return kwargs["allow_remote"]
|
||||
|
||||
@override_settings(REMOTE_OCR_MODE="always")
|
||||
def test_always_mode_allows_remote(self) -> None:
|
||||
"""
|
||||
GIVEN: Remote OCR mode is 'always'.
|
||||
WHEN: A document is consumed without any workflow asking for it.
|
||||
THEN: The registry is allowed to pick the remote parser.
|
||||
"""
|
||||
self.assertTrue(self._consume())
|
||||
|
||||
@override_settings(REMOTE_OCR_MODE="workflow_only")
|
||||
def test_workflow_only_mode_denies_remote_by_default(self) -> None:
|
||||
"""
|
||||
GIVEN: Remote OCR mode is 'workflow_only'.
|
||||
WHEN: A document is consumed and nothing asked for remote OCR.
|
||||
THEN: The remote parser is excluded.
|
||||
"""
|
||||
self.assertFalse(self._consume())
|
||||
|
||||
@override_settings(REMOTE_OCR_MODE="workflow_only")
|
||||
def test_workflow_only_mode_allows_remote_when_requested(self) -> None:
|
||||
"""
|
||||
GIVEN: Remote OCR mode is 'workflow_only'.
|
||||
WHEN: A workflow set remote_ocr on the metadata overrides.
|
||||
THEN: The registry is allowed to pick the remote parser.
|
||||
"""
|
||||
self.assertTrue(
|
||||
self._consume(overrides=DocumentMetadataOverrides(remote_ocr=True)),
|
||||
)
|
||||
|
||||
|
||||
class TestMetadataOverrides(TestCase):
|
||||
def test_update_skip_asn_if_exists(self) -> None:
|
||||
base = DocumentMetadataOverrides()
|
||||
@@ -1632,20 +1566,6 @@ class TestMetadataOverrides(TestCase):
|
||||
base.update(incoming)
|
||||
self.assertTrue(base.skip_asn_if_exists)
|
||||
|
||||
def test_update_remote_ocr(self) -> None:
|
||||
base = DocumentMetadataOverrides()
|
||||
base.update(DocumentMetadataOverrides(remote_ocr=True))
|
||||
self.assertTrue(base.remote_ocr)
|
||||
|
||||
def test_update_remote_ocr_is_not_unset(self) -> None:
|
||||
"""
|
||||
A later workflow that says nothing must not undo an earlier one that
|
||||
asked for remote OCR.
|
||||
"""
|
||||
base = DocumentMetadataOverrides(remote_ocr=True)
|
||||
base.update(DocumentMetadataOverrides())
|
||||
self.assertTrue(base.remote_ocr)
|
||||
|
||||
def test_update_actor_and_version_label(self) -> None:
|
||||
base = DocumentMetadataOverrides(
|
||||
actor_id=1,
|
||||
|
||||
@@ -426,7 +426,7 @@ class TestExportImport(
|
||||
st_mtime_1 = (self.target / "manifest.json").stat().st_mtime
|
||||
|
||||
with mock.patch(
|
||||
"documents.export.sinks.copy_file_with_basic_stats",
|
||||
"documents.management.commands.document_exporter.copy_file_with_basic_stats",
|
||||
) as m:
|
||||
self._do_export()
|
||||
m.assert_not_called()
|
||||
@@ -437,7 +437,7 @@ class TestExportImport(
|
||||
Path(self.d1.source_path).touch()
|
||||
|
||||
with mock.patch(
|
||||
"documents.export.sinks.copy_file_with_basic_stats",
|
||||
"documents.management.commands.document_exporter.copy_file_with_basic_stats",
|
||||
) as m:
|
||||
self._do_export()
|
||||
self.assertEqual(m.call_count, 1)
|
||||
@@ -464,7 +464,7 @@ class TestExportImport(
|
||||
self.assertIsFile(self.target / "manifest.json")
|
||||
|
||||
with mock.patch(
|
||||
"documents.export.sinks.copy_file_with_basic_stats",
|
||||
"documents.management.commands.document_exporter.copy_file_with_basic_stats",
|
||||
) as m:
|
||||
self._do_export()
|
||||
m.assert_not_called()
|
||||
@@ -475,7 +475,7 @@ class TestExportImport(
|
||||
self.d2.save()
|
||||
|
||||
with mock.patch(
|
||||
"documents.export.sinks.copy_file_with_basic_stats",
|
||||
"documents.management.commands.document_exporter.copy_file_with_basic_stats",
|
||||
) as m:
|
||||
self._do_export(compare_checksums=True)
|
||||
self.assertEqual(m.call_count, 1)
|
||||
@@ -1058,26 +1058,6 @@ class TestExportImport(
|
||||
|
||||
self.assertEqual(Document.objects.all().count(), 4)
|
||||
|
||||
def test_zip_with_compare_flags_raises(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A request to export to a zip file
|
||||
WHEN:
|
||||
- --compare-checksums or --compare-json is also passed
|
||||
THEN:
|
||||
- A CommandError is raised (the flags are no-ops in zip mode)
|
||||
"""
|
||||
for flag in ("--compare-checksums", "--compare-json"):
|
||||
with self.subTest(flag=flag):
|
||||
with self.assertRaises(CommandError):
|
||||
call_command(
|
||||
"document_exporter",
|
||||
self.target,
|
||||
"--zip",
|
||||
flag,
|
||||
skip_checks=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.management
|
||||
class TestCryptExportImport(
|
||||
|
||||
@@ -12,22 +12,9 @@ from django.test import override_settings
|
||||
from guardian.shortcuts import assign_perm
|
||||
from rest_framework.test import APIClient
|
||||
|
||||
from documents.matching import match_correspondents
|
||||
from documents.matching import match_document_types
|
||||
from documents.matching import match_storage_paths
|
||||
from documents.matching import match_tags
|
||||
from documents.models import Correspondent
|
||||
from documents.models import DocumentType
|
||||
from documents.models import StoragePath
|
||||
from documents.models import Tag
|
||||
from documents.permissions import permitted_document_ids
|
||||
from documents.permissions import permitted_object_ids
|
||||
from documents.serialisers import _get_viewable_duplicates
|
||||
from documents.tests.factories import CorrespondentFactory
|
||||
from documents.tests.factories import DocumentFactory
|
||||
from documents.tests.factories import DocumentTypeFactory
|
||||
from documents.tests.factories import StoragePathFactory
|
||||
from documents.tests.factories import TagFactory
|
||||
|
||||
|
||||
def assert_visible_document_ids(actual_ids, *, expected_visible, expected_hidden):
|
||||
@@ -444,342 +431,3 @@ class TestTrashRestorePermissionBoundary:
|
||||
format="json",
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestTrashViewExcludesExplicitlyGrantedDocuments:
|
||||
"""
|
||||
Regression test pinning TrashView's use of
|
||||
``_TrashPermittedObjectsFilter`` (``include_granted = False``). If that
|
||||
flag were ever flipped to the default ``True``, or the subclass removed
|
||||
in favor of the base ``PermittedObjectsFilter``, a trashed document
|
||||
would leak into ``/api/trash/`` results for any user holding an
|
||||
explicit guardian grant on it, even though they are neither the owner
|
||||
nor a superuser.
|
||||
"""
|
||||
|
||||
def test_explicit_grant_does_not_leak_trashed_document(self, rest_api_client):
|
||||
owner = User.objects.create_user(username="trash_owner")
|
||||
grantee = User.objects.create_user(username="trash_grantee")
|
||||
doc = DocumentFactory(owner=owner)
|
||||
doc.delete() # soft delete
|
||||
assign_perm("view_document", grantee, doc)
|
||||
|
||||
rest_api_client.force_authenticate(user=grantee)
|
||||
response = rest_api_client.get("/api/trash/")
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
result_ids = {result["id"] for result in response.data["results"]}
|
||||
assert doc.pk not in result_ids
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@pytest.mark.parametrize(
|
||||
("model", "factory", "perm"),
|
||||
[
|
||||
(Tag, TagFactory, "view_tag"),
|
||||
(Correspondent, CorrespondentFactory, "view_correspondent"),
|
||||
(DocumentType, DocumentTypeFactory, "view_documenttype"),
|
||||
(StoragePath, StoragePathFactory, "view_storagepath"),
|
||||
],
|
||||
)
|
||||
class TestPermittedObjectIdsGenericModels:
|
||||
def test_owner_sees_own_object(self, model, factory, perm):
|
||||
owner = User.objects.create_user(username=f"owner_{model.__name__}")
|
||||
stranger = User.objects.create_user(username=f"stranger_{model.__name__}")
|
||||
owned = factory(owner=owner)
|
||||
strangers = factory(owner=stranger)
|
||||
|
||||
assert_visible_document_ids(
|
||||
permitted_object_ids(owner, model, perm),
|
||||
expected_visible=[owned.pk],
|
||||
expected_hidden=[strangers.pk],
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("is_superuser", [False, True])
|
||||
def test_inactive_user_sees_nothing(self, model, factory, perm, is_superuser):
|
||||
suffix = f"{model.__name__}_{is_superuser}"
|
||||
user = User.objects.create_user(
|
||||
username=f"inactive_{suffix}",
|
||||
is_active=False,
|
||||
is_superuser=is_superuser,
|
||||
)
|
||||
other = User.objects.create_user(username=f"other_{suffix}")
|
||||
granted = factory(owner=other)
|
||||
assign_perm(perm, user, granted)
|
||||
|
||||
assert_visible_document_ids(
|
||||
permitted_object_ids(user, model, perm),
|
||||
expected_visible=[],
|
||||
expected_hidden=[
|
||||
factory(owner=None).pk,
|
||||
factory(owner=user).pk,
|
||||
granted.pk,
|
||||
],
|
||||
)
|
||||
|
||||
def test_unowned_object_visible_to_everyone(self, model, factory, perm):
|
||||
user = User.objects.create_user(username=f"user_{model.__name__}")
|
||||
unowned = factory(owner=None)
|
||||
|
||||
assert_visible_document_ids(
|
||||
permitted_object_ids(user, model, perm),
|
||||
expected_visible=[unowned.pk],
|
||||
expected_hidden=[],
|
||||
)
|
||||
|
||||
def test_explicit_permission_grants_visibility(self, model, factory, perm):
|
||||
owner = User.objects.create_user(username=f"owner2_{model.__name__}")
|
||||
grantee = User.objects.create_user(username=f"grantee_{model.__name__}")
|
||||
stranger = User.objects.create_user(username=f"stranger2_{model.__name__}")
|
||||
shared = factory(owner=owner)
|
||||
not_shared = factory(owner=owner)
|
||||
assign_perm(perm, grantee, shared)
|
||||
|
||||
assert_visible_document_ids(
|
||||
permitted_object_ids(grantee, model, perm),
|
||||
expected_visible=[shared.pk],
|
||||
expected_hidden=[not_shared.pk],
|
||||
)
|
||||
assert_visible_document_ids(
|
||||
permitted_object_ids(stranger, model, perm),
|
||||
expected_visible=[],
|
||||
expected_hidden=[shared.pk, not_shared.pk],
|
||||
)
|
||||
|
||||
def test_group_permission_grants_visibility_to_members_only(
|
||||
self,
|
||||
model,
|
||||
factory,
|
||||
perm,
|
||||
):
|
||||
owner = User.objects.create_user(username=f"owner3_{model.__name__}")
|
||||
member = User.objects.create_user(username=f"member_{model.__name__}")
|
||||
non_member = User.objects.create_user(username=f"nonmember_{model.__name__}")
|
||||
group = Group.objects.create(name=f"group_{model.__name__}")
|
||||
member.groups.add(group)
|
||||
shared = factory(owner=owner)
|
||||
assign_perm(perm, group, shared)
|
||||
|
||||
assert_visible_document_ids(
|
||||
permitted_object_ids(member, model, perm),
|
||||
expected_visible=[shared.pk],
|
||||
expected_hidden=[],
|
||||
)
|
||||
assert_visible_document_ids(
|
||||
permitted_object_ids(non_member, model, perm),
|
||||
expected_visible=[],
|
||||
expected_hidden=[shared.pk],
|
||||
)
|
||||
|
||||
def test_superuser_sees_everything(self, model, factory, perm):
|
||||
superuser = User.objects.create_superuser(username=f"root_{model.__name__}")
|
||||
owner = User.objects.create_user(username=f"owner4_{model.__name__}")
|
||||
obj = factory(owner=owner)
|
||||
|
||||
assert_visible_document_ids(
|
||||
permitted_object_ids(superuser, model, perm),
|
||||
expected_visible=[obj.pk],
|
||||
expected_hidden=[],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestMatchingRespectsObjectPermissions:
|
||||
def test_match_tags_only_considers_tags_visible_to_user(self):
|
||||
owner = User.objects.create_user(username="tag_owner")
|
||||
classifying_user = User.objects.create_user(username="classifier_user")
|
||||
visible_tag = TagFactory(
|
||||
owner=owner,
|
||||
match="invoice",
|
||||
matching_algorithm=Tag.MATCH_LITERAL,
|
||||
)
|
||||
hidden_tag = TagFactory(
|
||||
owner=owner,
|
||||
match="invoice",
|
||||
matching_algorithm=Tag.MATCH_LITERAL,
|
||||
)
|
||||
assign_perm("view_tag", classifying_user, visible_tag)
|
||||
doc = DocumentFactory(owner=classifying_user, content="an invoice document")
|
||||
|
||||
matched = match_tags(doc, classifier=None, user=classifying_user)
|
||||
matched_ids = {t.pk for t in matched}
|
||||
assert visible_tag.pk in matched_ids
|
||||
assert hidden_tag.pk not in matched_ids
|
||||
|
||||
def test_match_correspondents_only_considers_correspondents_visible_to_user(self):
|
||||
owner = User.objects.create_user(username="correspondent_owner")
|
||||
classifying_user = User.objects.create_user(username="classifier_user2")
|
||||
visible_correspondent = CorrespondentFactory(
|
||||
owner=owner,
|
||||
match="invoice",
|
||||
matching_algorithm=Correspondent.MATCH_LITERAL,
|
||||
)
|
||||
hidden_correspondent = CorrespondentFactory(
|
||||
owner=owner,
|
||||
match="invoice",
|
||||
matching_algorithm=Correspondent.MATCH_LITERAL,
|
||||
)
|
||||
assign_perm("view_correspondent", classifying_user, visible_correspondent)
|
||||
doc = DocumentFactory(owner=classifying_user, content="an invoice document")
|
||||
|
||||
matched = match_correspondents(doc, classifier=None, user=classifying_user)
|
||||
matched_ids = {c.pk for c in matched}
|
||||
assert visible_correspondent.pk in matched_ids
|
||||
assert hidden_correspondent.pk not in matched_ids
|
||||
|
||||
def test_match_document_types_only_considers_document_types_visible_to_user(self):
|
||||
owner = User.objects.create_user(username="document_type_owner")
|
||||
classifying_user = User.objects.create_user(username="classifier_user3")
|
||||
visible_document_type = DocumentTypeFactory(
|
||||
owner=owner,
|
||||
match="invoice",
|
||||
matching_algorithm=DocumentType.MATCH_LITERAL,
|
||||
)
|
||||
hidden_document_type = DocumentTypeFactory(
|
||||
owner=owner,
|
||||
match="invoice",
|
||||
matching_algorithm=DocumentType.MATCH_LITERAL,
|
||||
)
|
||||
assign_perm("view_documenttype", classifying_user, visible_document_type)
|
||||
doc = DocumentFactory(owner=classifying_user, content="an invoice document")
|
||||
|
||||
matched = match_document_types(doc, classifier=None, user=classifying_user)
|
||||
matched_ids = {dt.pk for dt in matched}
|
||||
assert visible_document_type.pk in matched_ids
|
||||
assert hidden_document_type.pk not in matched_ids
|
||||
|
||||
def test_match_storage_paths_only_considers_storage_paths_visible_to_user(self):
|
||||
owner = User.objects.create_user(username="storage_path_owner")
|
||||
classifying_user = User.objects.create_user(username="classifier_user4")
|
||||
visible_storage_path = StoragePathFactory(
|
||||
owner=owner,
|
||||
match="invoice",
|
||||
matching_algorithm=StoragePath.MATCH_LITERAL,
|
||||
)
|
||||
hidden_storage_path = StoragePathFactory(
|
||||
owner=owner,
|
||||
match="invoice",
|
||||
matching_algorithm=StoragePath.MATCH_LITERAL,
|
||||
)
|
||||
assign_perm("view_storagepath", classifying_user, visible_storage_path)
|
||||
doc = DocumentFactory(owner=classifying_user, content="an invoice document")
|
||||
|
||||
matched = match_storage_paths(doc, classifier=None, user=classifying_user)
|
||||
matched_ids = {sp.pk for sp in matched}
|
||||
assert visible_storage_path.pk in matched_ids
|
||||
assert hidden_storage_path.pk not in matched_ids
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestBulkEditObjectsApplyToAllPermissionBoundary:
|
||||
def test_apply_to_all_tags_excludes_unpermitted_tag(self, rest_api_client):
|
||||
owner = User.objects.create_user(username="tags_owner")
|
||||
requester = User.objects.create_user(username="tags_requester")
|
||||
# grant the global change_tag permission so the object-level
|
||||
# filtering (not the global has_perm check) is what's under test
|
||||
requester.user_permissions.add(
|
||||
Permission.objects.get(codename="change_tag"),
|
||||
)
|
||||
rest_api_client.force_authenticate(user=requester)
|
||||
visible = TagFactory(owner=owner)
|
||||
hidden = TagFactory(owner=owner)
|
||||
assign_perm("view_tag", requester, visible)
|
||||
assign_perm("change_tag", requester, visible)
|
||||
|
||||
response = rest_api_client.post(
|
||||
"/api/bulk_edit_objects/",
|
||||
{
|
||||
"object_type": "tags",
|
||||
"operation": "set_permissions",
|
||||
"all": True,
|
||||
"filters": {},
|
||||
"owner": requester.pk,
|
||||
},
|
||||
format="json",
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
|
||||
# The apply_to_all dispatch must resolve permitted objects up front:
|
||||
# the visible tag (object-level change_tag granted) gets its owner
|
||||
# reassigned, while the hidden tag (no object-level grant) is
|
||||
# excluded entirely and keeps its original owner.
|
||||
visible.refresh_from_db()
|
||||
hidden.refresh_from_db()
|
||||
assert visible.owner == requester
|
||||
assert hidden.owner == owner
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestBulkEditObjectsTagDescendantPartialPermission:
|
||||
def test_apply_to_all_descendant_expansion_respects_per_object_permissions(
|
||||
self,
|
||||
rest_api_client,
|
||||
):
|
||||
"""
|
||||
GIVEN:
|
||||
- A tag hierarchy (parent -> permitted_child, unpermitted_child)
|
||||
- A non-superuser requester with object-level change_tag granted
|
||||
on the parent and on only ONE of the two children
|
||||
WHEN:
|
||||
- bulk_edit_objects is called with all=True and a filter that
|
||||
matches only the root (parent) tag, engaging the
|
||||
tag-descendant-expansion logic in BulkEditObjectsView.post
|
||||
THEN:
|
||||
- The descendant expansion only pulls in descendants the
|
||||
requester actually has permission on: the permitted child's
|
||||
owner is reassigned alongside the parent's, while the
|
||||
unpermitted child keeps its original owner. This pins that the
|
||||
expansion checks per-object permissions (editable_ids), not
|
||||
merely "is a descendant of a filter match".
|
||||
|
||||
NOTE: this uses ``set_permissions`` (owner reassignment) rather than
|
||||
``delete`` as the operation, because Tag.tn_parent (django-treenode)
|
||||
cascades deletes to descendants at the database/ORM level regardless
|
||||
of which tags the view resolved into ``objs`` -- a delete-based test
|
||||
would pass/fail based on FK cascade behavior, not on whether the
|
||||
descendant-expansion logic itself respected per-object permissions.
|
||||
"""
|
||||
owner = User.objects.create_user(username="tag_hierarchy_owner")
|
||||
requester = User.objects.create_user(username="tag_hierarchy_requester")
|
||||
# global change_tag permission so the has_perm() gate passes and the
|
||||
# object-level permitted_object_ids filtering is what's under test
|
||||
requester.user_permissions.add(
|
||||
Permission.objects.get(codename="change_tag"),
|
||||
)
|
||||
rest_api_client.force_authenticate(user=requester)
|
||||
|
||||
parent = TagFactory(owner=owner, name="parent-tag")
|
||||
permitted_child = TagFactory(
|
||||
owner=owner,
|
||||
name="permitted-child-tag",
|
||||
tn_parent=parent,
|
||||
)
|
||||
unpermitted_child = TagFactory(
|
||||
owner=owner,
|
||||
name="unpermitted-child-tag",
|
||||
tn_parent=parent,
|
||||
)
|
||||
assign_perm("change_tag", requester, parent)
|
||||
assign_perm("change_tag", requester, permitted_child)
|
||||
# unpermitted_child is intentionally NOT granted change_tag
|
||||
|
||||
response = rest_api_client.post(
|
||||
"/api/bulk_edit_objects/",
|
||||
{
|
||||
"object_type": "tags",
|
||||
"operation": "set_permissions",
|
||||
"all": True,
|
||||
"filters": {"is_root": True},
|
||||
"owner": requester.pk,
|
||||
},
|
||||
format="json",
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
|
||||
parent.refresh_from_db()
|
||||
permitted_child.refresh_from_db()
|
||||
unpermitted_child.refresh_from_db()
|
||||
assert parent.owner == requester
|
||||
assert permitted_child.owner == requester
|
||||
assert unpermitted_child.owner == owner
|
||||
|
||||
@@ -1,111 +0,0 @@
|
||||
import pytest
|
||||
from django.contrib.auth.models import User
|
||||
from guardian.shortcuts import assign_perm
|
||||
from rest_framework.test import APIRequestFactory
|
||||
|
||||
from documents.filters import PermittedObjectsFilter
|
||||
from documents.models import Tag
|
||||
from documents.tests.factories import TagFactory
|
||||
|
||||
|
||||
class _DummyView:
|
||||
queryset = Tag.objects.all()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestPermittedObjectsFilter:
|
||||
def test_superuser_bypasses_filtering_entirely(self):
|
||||
superuser = User.objects.create_superuser(username="root")
|
||||
owner = User.objects.create_user(username="owner")
|
||||
TagFactory(owner=owner)
|
||||
request = APIRequestFactory().get("/")
|
||||
request.user = superuser
|
||||
|
||||
result = PermittedObjectsFilter().filter_queryset(
|
||||
request,
|
||||
Tag.objects.all(),
|
||||
_DummyView(),
|
||||
)
|
||||
assert result.count() == Tag.objects.count()
|
||||
|
||||
def test_non_superuser_sees_only_owned_unowned_and_granted(self):
|
||||
owner = User.objects.create_user(username="owner")
|
||||
grantee = User.objects.create_user(username="grantee")
|
||||
owned = TagFactory(owner=grantee)
|
||||
unowned = TagFactory(owner=None)
|
||||
granted = TagFactory(owner=owner)
|
||||
hidden = TagFactory(owner=owner)
|
||||
assign_perm("view_tag", grantee, granted)
|
||||
request = APIRequestFactory().get("/")
|
||||
request.user = grantee
|
||||
|
||||
result = PermittedObjectsFilter().filter_queryset(
|
||||
request,
|
||||
Tag.objects.all(),
|
||||
_DummyView(),
|
||||
)
|
||||
visible_ids = set(result.values_list("id", flat=True))
|
||||
assert visible_ids == {owned.pk, unowned.pk, granted.pk}
|
||||
assert hidden.pk not in visible_ids
|
||||
|
||||
def test_include_granted_false_excludes_explicitly_shared_objects(self):
|
||||
owner = User.objects.create_user(username="owner2")
|
||||
grantee = User.objects.create_user(username="grantee2")
|
||||
owned = TagFactory(owner=grantee)
|
||||
granted = TagFactory(owner=owner)
|
||||
assign_perm("view_tag", grantee, granted)
|
||||
request = APIRequestFactory().get("/")
|
||||
request.user = grantee
|
||||
|
||||
class _OwnerOnlyFilter(PermittedObjectsFilter):
|
||||
include_granted = False
|
||||
|
||||
result = _OwnerOnlyFilter().filter_queryset(
|
||||
request,
|
||||
Tag.objects.all(),
|
||||
_DummyView(),
|
||||
)
|
||||
visible_ids = set(result.values_list("id", flat=True))
|
||||
assert visible_ids == {owned.pk}
|
||||
assert granted.pk not in visible_ids
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("username", "is_superuser"),
|
||||
[("inactive", False), ("inactive_super", True)],
|
||||
)
|
||||
def test_inactive_user_sees_nothing(self, username: str, *, is_superuser: bool):
|
||||
user = User.objects.create_user(
|
||||
username=username,
|
||||
is_active=False,
|
||||
is_superuser=is_superuser,
|
||||
)
|
||||
TagFactory(owner=None)
|
||||
TagFactory(owner=user)
|
||||
granted = TagFactory(owner=User.objects.create_user(username=f"o_{username}"))
|
||||
assign_perm("view_tag", user, granted)
|
||||
request = APIRequestFactory().get("/")
|
||||
request.user = user
|
||||
|
||||
result = PermittedObjectsFilter().filter_queryset(
|
||||
request,
|
||||
Tag.objects.all(),
|
||||
_DummyView(),
|
||||
)
|
||||
assert result.count() == 0
|
||||
|
||||
def test_inactive_user_sees_nothing_with_include_granted_false(self):
|
||||
user = User.objects.create_user(username="inactive_owner", is_active=False)
|
||||
TagFactory(owner=user)
|
||||
TagFactory(owner=None)
|
||||
request = APIRequestFactory().get("/")
|
||||
request.user = user
|
||||
|
||||
class _OwnerOnlyFilter(PermittedObjectsFilter):
|
||||
include_granted = False
|
||||
|
||||
result = _OwnerOnlyFilter().filter_queryset(
|
||||
request,
|
||||
Tag.objects.all(),
|
||||
_DummyView(),
|
||||
)
|
||||
assert result.count() == 0
|
||||
@@ -385,25 +385,6 @@ class TestTaskFailureHandler:
|
||||
task_failure_handler(task_id=None, exception=ValueError("x"), traceback=None)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestApplyAiSuggestionsTracking:
|
||||
def test_records_the_document_it_is_for(self) -> None:
|
||||
"""
|
||||
The action queues one task per document, so the tracked record notes
|
||||
which document it is for -- otherwise a bulk run is an indistinguishable
|
||||
wall of identical entries in the tasks list.
|
||||
"""
|
||||
task_id = send_publish(
|
||||
"documents.tasks.apply_ai_suggestions",
|
||||
(),
|
||||
{"action_id": 1, "document_id": 42},
|
||||
)
|
||||
|
||||
task = PaperlessTask.objects.get(task_id=task_id)
|
||||
assert task.task_type == PaperlessTask.TaskType.APPLY_AI_SUGGESTIONS
|
||||
assert task.input_data == {"document_id": 42}
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestTaskRevokedHandler:
|
||||
def test_marks_task_revoked(self, mocker: pytest_mock.MockerFixture) -> None:
|
||||
|
||||
@@ -14,7 +14,6 @@ from documents.models import Correspondent
|
||||
from documents.models import Document
|
||||
from documents.models import DocumentType
|
||||
from documents.models import Tag
|
||||
from documents.models import WorkflowAction
|
||||
from documents.sanity_checker import SanityCheckFailedException
|
||||
from documents.sanity_checker import SanityCheckMessages
|
||||
from documents.tests.test_classifier import dummy_preprocess
|
||||
@@ -288,45 +287,6 @@ class TestUpdateContent(DirectoriesMixin, TestCase):
|
||||
self.assertNotEqual(Document.objects.get(pk=doc.pk).content, "test")
|
||||
|
||||
|
||||
class TestUpdateContentRemoteOCR(DirectoriesMixin, TestCase):
|
||||
"""
|
||||
Consumption workflows do not run on reprocess, so the remote parser is
|
||||
used only in 'always' mode or when the caller explicitly asks for it.
|
||||
"""
|
||||
|
||||
def setUp(self) -> None:
|
||||
super().setUp()
|
||||
|
||||
patcher = mock.patch("documents.tasks.get_parser_registry")
|
||||
self.mock_registry = patcher.start()
|
||||
self.mock_registry.return_value.get_parser_for_file.return_value = None
|
||||
self.addCleanup(patcher.stop)
|
||||
|
||||
self.doc = Document.objects.create(
|
||||
title="test",
|
||||
content="my document",
|
||||
checksum="wow",
|
||||
mime_type="application/pdf",
|
||||
)
|
||||
|
||||
def _allow_remote(self, **kwargs) -> bool:
|
||||
tasks.update_document_content_maybe_archive_file(self.doc.pk, **kwargs)
|
||||
_, call_kwargs = self.mock_registry.return_value.get_parser_for_file.call_args
|
||||
return call_kwargs["allow_remote"]
|
||||
|
||||
@override_settings(REMOTE_OCR_MODE="always")
|
||||
def test_always_mode_allows_remote(self) -> None:
|
||||
self.assertTrue(self._allow_remote())
|
||||
|
||||
@override_settings(REMOTE_OCR_MODE="workflow_only")
|
||||
def test_workflow_only_mode_denies_remote_by_default(self) -> None:
|
||||
self.assertFalse(self._allow_remote())
|
||||
|
||||
@override_settings(REMOTE_OCR_MODE="workflow_only")
|
||||
def test_workflow_only_mode_allows_remote_when_requested(self) -> None:
|
||||
self.assertTrue(self._allow_remote(remote_ocr=True))
|
||||
|
||||
|
||||
class TestAIIndex(DirectoriesMixin, TestCase):
|
||||
@override_settings(
|
||||
AI_ENABLED=True,
|
||||
@@ -448,110 +408,3 @@ class TestAIIndex(DirectoriesMixin, TestCase):
|
||||
rebuild=False,
|
||||
document_ids=doc_ids,
|
||||
)
|
||||
|
||||
|
||||
class TestApplyAISuggestionsTask(DirectoriesMixin, TestCase):
|
||||
def setUp(self) -> None:
|
||||
super().setUp()
|
||||
self.doc = Document.objects.create(
|
||||
title="doc",
|
||||
content="content",
|
||||
checksum="apply-ai-suggestions",
|
||||
)
|
||||
self.action = WorkflowAction.objects.create(
|
||||
type=WorkflowAction.WorkflowActionType.APPLY_AI_SUGGESTIONS,
|
||||
ai_suggestion_fields=[WorkflowAction.AISuggestionField.TITLE],
|
||||
)
|
||||
|
||||
def test_reindexes_without_sending_document_updated(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- An apply AI suggestions action that changes the document
|
||||
WHEN:
|
||||
- The task runs
|
||||
THEN:
|
||||
- The search index and caches are refreshed directly, deliberately
|
||||
not via the document_updated signal: that re-runs updated
|
||||
workflows, which for this action means queueing another LLM
|
||||
query for a document it just changed, forever
|
||||
"""
|
||||
with (
|
||||
mock.patch(
|
||||
"documents.workflows.ai.apply_ai_suggestions_to_document",
|
||||
return_value=["title"],
|
||||
),
|
||||
mock.patch("documents.tasks.index_document") as index_document,
|
||||
mock.patch("documents.tasks.clear_document_caches") as clear_caches,
|
||||
mock.patch("documents.tasks.document_updated") as document_updated,
|
||||
):
|
||||
tasks.apply_ai_suggestions(self.action.pk, self.doc.pk)
|
||||
|
||||
index_document.delay.assert_called_once_with(self.doc.pk)
|
||||
clear_caches.assert_called_once_with(self.doc.pk)
|
||||
document_updated.send.assert_not_called()
|
||||
|
||||
def test_no_changes_skips_reindex(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- An apply AI suggestions action that changes nothing
|
||||
WHEN:
|
||||
- The task runs
|
||||
THEN:
|
||||
- No reindexing work is queued
|
||||
"""
|
||||
with (
|
||||
mock.patch(
|
||||
"documents.workflows.ai.apply_ai_suggestions_to_document",
|
||||
return_value=[],
|
||||
),
|
||||
mock.patch("documents.tasks.index_document") as index_document,
|
||||
):
|
||||
tasks.apply_ai_suggestions(self.action.pk, self.doc.pk)
|
||||
|
||||
index_document.delay.assert_not_called()
|
||||
|
||||
@override_settings(AI_ENABLED=True, LLM_EMBEDDING_BACKEND="huggingface")
|
||||
def test_updates_llm_index_when_enabled(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- An apply AI suggestions action that changes the document
|
||||
- The LLM index is enabled
|
||||
WHEN:
|
||||
- The task runs
|
||||
THEN:
|
||||
- The document is updated in the LLM index too
|
||||
"""
|
||||
with (
|
||||
mock.patch(
|
||||
"documents.workflows.ai.apply_ai_suggestions_to_document",
|
||||
return_value=["title"],
|
||||
),
|
||||
mock.patch("documents.tasks.index_document"),
|
||||
mock.patch(
|
||||
"documents.tasks.update_document_in_llm_index",
|
||||
) as update_in_llm_index,
|
||||
):
|
||||
tasks.apply_ai_suggestions(self.action.pk, self.doc.pk)
|
||||
|
||||
update_in_llm_index.apply_async.assert_called_once()
|
||||
|
||||
def test_deleted_document_is_a_noop(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A document that was deleted between the workflow running and the
|
||||
queued task starting
|
||||
WHEN:
|
||||
- The task runs
|
||||
THEN:
|
||||
- It logs and exits rather than raising
|
||||
"""
|
||||
with (
|
||||
mock.patch(
|
||||
"documents.workflows.ai.apply_ai_suggestions_to_document",
|
||||
) as apply_suggestions,
|
||||
self.assertLogs("paperless.tasks", level="WARNING") as cm,
|
||||
):
|
||||
tasks.apply_ai_suggestions(self.action.pk, self.doc.pk + 1000)
|
||||
|
||||
apply_suggestions.assert_not_called()
|
||||
self.assertIn("no longer exists", "".join(cm.output))
|
||||
|
||||
@@ -78,6 +78,10 @@ class TestViews(DirectoriesMixin, TestCase):
|
||||
response.context_data["styles_css"],
|
||||
f"frontend/{language_actual}/styles.css",
|
||||
)
|
||||
self.assertEqual(
|
||||
response.context_data["runtime_js"],
|
||||
f"frontend/{language_actual}/runtime.js",
|
||||
)
|
||||
self.assertEqual(
|
||||
response.context_data["polyfills_js"],
|
||||
f"frontend/{language_actual}/polyfills.js",
|
||||
|
||||
@@ -31,9 +31,7 @@ from documents.file_handling import create_source_path_directory
|
||||
from documents.file_handling import generate_filename
|
||||
from documents.file_handling import generate_unique_filename
|
||||
from documents.signals.handlers import run_workflows
|
||||
from documents.workflows.ai import apply_ai_suggestions_to_document
|
||||
from documents.workflows.webhooks import send_webhook
|
||||
from paperless_ai.exceptions import LLMTimeoutError
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from django.db.models import QuerySet
|
||||
@@ -5362,493 +5360,3 @@ class TestDateWorkflowLocalization(
|
||||
document = Document.objects.first()
|
||||
assert document is not None
|
||||
assert document.title == expected_title
|
||||
|
||||
|
||||
class TestRemoteOCRWorkflowAction(DirectoriesMixin, SampleDirMixin, APITestCase):
|
||||
def _make_workflow(self, trigger_type) -> None:
|
||||
trigger = WorkflowTrigger.objects.create(type=trigger_type)
|
||||
action = WorkflowAction.objects.create(
|
||||
type=WorkflowAction.WorkflowActionType.REMOTE_OCR,
|
||||
)
|
||||
w = Workflow.objects.create(name="Remote OCR", order=0)
|
||||
w.triggers.add(trigger)
|
||||
w.actions.add(action)
|
||||
w.save()
|
||||
|
||||
def test_consumption_trigger_requests_remote_ocr(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A consumption workflow with a remote OCR action
|
||||
WHEN:
|
||||
- A matching document is consumed
|
||||
THEN:
|
||||
- The overrides ask for remote OCR, which is what the consumer
|
||||
reads when choosing a parser
|
||||
"""
|
||||
self._make_workflow(WorkflowTrigger.WorkflowTriggerType.CONSUMPTION)
|
||||
|
||||
test_file = shutil.copy(
|
||||
self.SAMPLE_DIR / "simple.pdf",
|
||||
self.dirs.scratch_dir / "simple.pdf",
|
||||
)
|
||||
overrides = DocumentMetadataOverrides()
|
||||
|
||||
run_workflows(
|
||||
WorkflowTrigger.WorkflowTriggerType.CONSUMPTION,
|
||||
ConsumableDocument(
|
||||
source=DocumentSource.ConsumeFolder,
|
||||
original_file=test_file,
|
||||
),
|
||||
overrides=overrides,
|
||||
)
|
||||
|
||||
self.assertTrue(overrides.remote_ocr)
|
||||
|
||||
def test_other_trigger_types_are_ignored(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A workflow with a remote OCR action that also has a
|
||||
non-consumption trigger, which is a valid combination
|
||||
WHEN:
|
||||
- The non-consumption trigger fires
|
||||
THEN:
|
||||
- The action is skipped, since the document has already been
|
||||
parsed by this point
|
||||
"""
|
||||
trigger = WorkflowTrigger.objects.create(
|
||||
type=WorkflowTrigger.WorkflowTriggerType.CONSUMPTION,
|
||||
)
|
||||
updated_trigger = WorkflowTrigger.objects.create(
|
||||
type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_UPDATED,
|
||||
)
|
||||
action = WorkflowAction.objects.create(
|
||||
type=WorkflowAction.WorkflowActionType.REMOTE_OCR,
|
||||
)
|
||||
w = Workflow.objects.create(name="Remote OCR", order=0)
|
||||
w.triggers.add(trigger, updated_trigger)
|
||||
w.actions.add(action)
|
||||
w.save()
|
||||
|
||||
doc = Document.objects.create(
|
||||
title="sample test",
|
||||
original_filename="sample.pdf",
|
||||
)
|
||||
|
||||
with self.assertLogs("paperless.handlers", level="DEBUG") as cm:
|
||||
run_workflows(
|
||||
WorkflowTrigger.WorkflowTriggerType.DOCUMENT_UPDATED,
|
||||
doc,
|
||||
)
|
||||
|
||||
self.assertIn("only applies to consumption triggers", "".join(cm.output))
|
||||
|
||||
|
||||
SUGGESTIONS = {
|
||||
"title": "Suggested Title",
|
||||
"tags": ["Existing Tag", "Suggested Tag"],
|
||||
"correspondents": ["Existing Correspondent", "Suggested Correspondent"],
|
||||
"document_types": ["Suggested Document Type"],
|
||||
"storage_paths": ["Suggested Storage Path"],
|
||||
"dates": ["2024-03-05"],
|
||||
}
|
||||
|
||||
ALL_SUGGESTION_FIELDS = [
|
||||
WorkflowAction.AISuggestionField.TITLE,
|
||||
WorkflowAction.AISuggestionField.TAGS,
|
||||
WorkflowAction.AISuggestionField.CORRESPONDENT,
|
||||
WorkflowAction.AISuggestionField.DOCUMENT_TYPE,
|
||||
WorkflowAction.AISuggestionField.STORAGE_PATH,
|
||||
WorkflowAction.AISuggestionField.CREATED,
|
||||
]
|
||||
|
||||
|
||||
@override_settings(AI_ENABLED=True)
|
||||
class TestApplyAISuggestionsWorkflowAction(
|
||||
DirectoriesMixin,
|
||||
SampleDirMixin,
|
||||
APITestCase,
|
||||
):
|
||||
def setUp(self) -> None:
|
||||
super().setUp()
|
||||
self.user = User.objects.create(username="ai-user")
|
||||
self.doc = Document.objects.create(
|
||||
title="original.pdf",
|
||||
content="the document content",
|
||||
checksum="ai-suggestions-checksum",
|
||||
mime_type="application/pdf",
|
||||
created=datetime.date(2020, 1, 1),
|
||||
owner=self.user,
|
||||
)
|
||||
|
||||
def make_action(self, **kwargs) -> WorkflowAction:
|
||||
return WorkflowAction.objects.create(
|
||||
type=WorkflowAction.WorkflowActionType.APPLY_AI_SUGGESTIONS,
|
||||
ai_suggestion_fields=kwargs.pop(
|
||||
"ai_suggestion_fields",
|
||||
ALL_SUGGESTION_FIELDS,
|
||||
),
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def make_workflow(self, action: WorkflowAction, trigger_type) -> Workflow:
|
||||
trigger = WorkflowTrigger.objects.create(type=trigger_type)
|
||||
w = Workflow.objects.create(name="Apply AI suggestions", order=0)
|
||||
w.triggers.add(trigger)
|
||||
w.actions.add(action)
|
||||
w.save()
|
||||
return w
|
||||
|
||||
def apply(self, action: WorkflowAction) -> list[str]:
|
||||
with mock.patch(
|
||||
"documents.workflows.ai.get_ai_document_classification",
|
||||
return_value=SUGGESTIONS,
|
||||
):
|
||||
changed = apply_ai_suggestions_to_document(action, self.doc)
|
||||
self.doc.refresh_from_db()
|
||||
return changed
|
||||
|
||||
def test_document_added_trigger_queues_task(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A document added workflow with an apply AI suggestions action
|
||||
WHEN:
|
||||
- A matching document is added
|
||||
THEN:
|
||||
- The work is queued rather than run inline, so a slow LLM query
|
||||
cannot stall the rest of the workflow run
|
||||
"""
|
||||
action = self.make_action()
|
||||
self.make_workflow(action, WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED)
|
||||
|
||||
with mock.patch("documents.tasks.apply_ai_suggestions.delay") as delay:
|
||||
run_workflows(
|
||||
WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED,
|
||||
self.doc,
|
||||
)
|
||||
|
||||
delay.assert_called_once_with(action_id=action.pk, document_id=self.doc.pk)
|
||||
|
||||
def test_consumption_trigger_is_ignored(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A workflow with an apply AI suggestions action and a consumption
|
||||
trigger alongside a valid one
|
||||
WHEN:
|
||||
- The consumption trigger fires
|
||||
THEN:
|
||||
- The action is skipped, since the document has not been parsed
|
||||
yet and so has no content to make suggestions from
|
||||
"""
|
||||
action = self.make_action()
|
||||
w = self.make_workflow(
|
||||
action,
|
||||
WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED,
|
||||
)
|
||||
w.triggers.add(
|
||||
WorkflowTrigger.objects.create(
|
||||
type=WorkflowTrigger.WorkflowTriggerType.CONSUMPTION,
|
||||
),
|
||||
)
|
||||
|
||||
test_file = shutil.copy(
|
||||
self.SAMPLE_DIR / "simple.pdf",
|
||||
self.dirs.scratch_dir / "simple.pdf",
|
||||
)
|
||||
|
||||
with (
|
||||
mock.patch("documents.tasks.apply_ai_suggestions.delay") as delay,
|
||||
self.assertLogs("paperless.handlers", level="DEBUG") as cm,
|
||||
):
|
||||
run_workflows(
|
||||
WorkflowTrigger.WorkflowTriggerType.CONSUMPTION,
|
||||
ConsumableDocument(
|
||||
source=DocumentSource.ConsumeFolder,
|
||||
original_file=test_file,
|
||||
),
|
||||
overrides=DocumentMetadataOverrides(),
|
||||
)
|
||||
|
||||
delay.assert_not_called()
|
||||
self.assertIn("does not apply to consumption triggers", "".join(cm.output))
|
||||
|
||||
def test_no_selected_fields_does_nothing(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- An action with no suggestion fields selected
|
||||
WHEN:
|
||||
- The action is applied
|
||||
THEN:
|
||||
- Nothing is changed and it is logged
|
||||
"""
|
||||
action = self.make_action(ai_suggestion_fields=[])
|
||||
|
||||
with self.assertLogs("paperless.workflows.ai", level="WARNING") as cm:
|
||||
changed = self.apply(action)
|
||||
|
||||
self.assertEqual(changed, [])
|
||||
self.assertIn("no AI suggestion fields selected", "".join(cm.output))
|
||||
|
||||
@override_settings(AI_ENABLED=False)
|
||||
def test_ai_disabled_does_nothing(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- An action on an install where AI has since been disabled
|
||||
WHEN:
|
||||
- The action is applied
|
||||
THEN:
|
||||
- Nothing is changed and it is logged
|
||||
"""
|
||||
action = self.make_action()
|
||||
|
||||
with self.assertLogs("paperless.workflows.ai", level="ERROR") as cm:
|
||||
changed = self.apply(action)
|
||||
|
||||
self.assertEqual(changed, [])
|
||||
self.assertIn("AI is not enabled", "".join(cm.output))
|
||||
|
||||
def test_invalid_configuration_leaves_document_untouched(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- An AI backend that is misconfigured
|
||||
WHEN:
|
||||
- The action is applied
|
||||
THEN:
|
||||
- The failure is logged and the document is left alone. It is not
|
||||
re-raised, because retrying will not fix a bad configuration
|
||||
"""
|
||||
action = self.make_action()
|
||||
|
||||
with (
|
||||
mock.patch(
|
||||
"documents.workflows.ai.get_ai_document_classification",
|
||||
side_effect=ValueError("nope"),
|
||||
),
|
||||
self.assertLogs("paperless.workflows.ai", level="ERROR") as cm,
|
||||
):
|
||||
changed = apply_ai_suggestions_to_document(action, self.doc)
|
||||
|
||||
self.assertEqual(changed, [])
|
||||
self.doc.refresh_from_db()
|
||||
self.assertEqual(self.doc.title, "original.pdf")
|
||||
self.assertIn("Invalid AI configuration", "".join(cm.output))
|
||||
|
||||
def test_transient_llm_failure_is_raised_for_retry(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- An LLM backend that times out, or rate limits the request
|
||||
WHEN:
|
||||
- The action is applied
|
||||
THEN:
|
||||
- The error propagates so the queued task can back off and retry,
|
||||
rather than silently dropping this document's suggestions
|
||||
"""
|
||||
action = self.make_action()
|
||||
|
||||
with (
|
||||
mock.patch(
|
||||
"documents.workflows.ai.get_ai_document_classification",
|
||||
side_effect=LLMTimeoutError(),
|
||||
),
|
||||
self.assertRaises(LLMTimeoutError),
|
||||
):
|
||||
apply_ai_suggestions_to_document(action, self.doc)
|
||||
|
||||
self.doc.refresh_from_db()
|
||||
self.assertEqual(self.doc.title, "original.pdf")
|
||||
|
||||
def test_only_matching_objects_are_applied(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- An action without create missing, and only some of the suggested
|
||||
objects existing
|
||||
WHEN:
|
||||
- The action is applied
|
||||
THEN:
|
||||
- Only the existing objects are assigned, unmatched suggestions are
|
||||
dropped rather than creating anything
|
||||
"""
|
||||
tag = Tag.objects.create(name="Existing Tag", owner=self.user)
|
||||
correspondent = Correspondent.objects.create(
|
||||
name="Existing Correspondent",
|
||||
owner=self.user,
|
||||
)
|
||||
action = self.make_action(ai_overwrite_existing=True)
|
||||
|
||||
changed = self.apply(action)
|
||||
|
||||
self.assertEqual(self.doc.correspondent, correspondent)
|
||||
self.assertEqual(list(self.doc.tags.all()), [tag])
|
||||
# Nothing matched for these and create missing is off
|
||||
self.assertIsNone(self.doc.document_type)
|
||||
self.assertIsNone(self.doc.storage_path)
|
||||
self.assertNotIn("document_type", changed)
|
||||
self.assertEqual(Tag.objects.count(), 1)
|
||||
self.assertEqual(Correspondent.objects.count(), 1)
|
||||
|
||||
def test_create_missing_creates_objects_owned_by_document_owner(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- An action with create missing enabled
|
||||
WHEN:
|
||||
- The action is applied and suggestions match nothing
|
||||
THEN:
|
||||
- Tags, correspondents and document types are created, owned by the
|
||||
document owner so they stay private to them
|
||||
- Storage paths are never created, since a path template cannot be
|
||||
inferred from a name
|
||||
"""
|
||||
action = self.make_action(
|
||||
ai_create_missing=True,
|
||||
ai_overwrite_existing=True,
|
||||
)
|
||||
|
||||
changed = self.apply(action)
|
||||
|
||||
self.assertEqual(
|
||||
sorted(t.name for t in self.doc.tags.all()),
|
||||
["Existing Tag", "Suggested Tag"],
|
||||
)
|
||||
self.assertEqual(self.doc.correspondent.name, "Existing Correspondent")
|
||||
self.assertEqual(self.doc.correspondent.owner, self.user)
|
||||
self.assertEqual(self.doc.document_type.name, "Suggested Document Type")
|
||||
self.assertEqual(self.doc.document_type.owner, self.user)
|
||||
|
||||
self.assertIsNone(self.doc.storage_path)
|
||||
self.assertFalse(StoragePath.objects.exists())
|
||||
self.assertNotIn("storage_path", changed)
|
||||
|
||||
def test_overwrite_disabled_keeps_existing_values(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- An action without overwrite existing
|
||||
- A document that already has a title, created date and
|
||||
correspondent
|
||||
WHEN:
|
||||
- The action is applied
|
||||
THEN:
|
||||
- The existing values are kept, only the empty document type is
|
||||
filled in
|
||||
"""
|
||||
existing = Correspondent.objects.create(name="Mine", owner=self.user)
|
||||
self.doc.correspondent = existing
|
||||
self.doc.save()
|
||||
action = self.make_action(ai_create_missing=True)
|
||||
|
||||
changed = self.apply(action)
|
||||
|
||||
self.assertEqual(self.doc.title, "original.pdf")
|
||||
self.assertEqual(self.doc.created, datetime.date(2020, 1, 1))
|
||||
self.assertEqual(self.doc.correspondent, existing)
|
||||
self.assertEqual(self.doc.document_type.name, "Suggested Document Type")
|
||||
self.assertNotIn("title", changed)
|
||||
self.assertNotIn("correspondent", changed)
|
||||
|
||||
def test_overwrite_enabled_replaces_existing_values(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- An action with overwrite existing
|
||||
- A document that already has a title and created date
|
||||
WHEN:
|
||||
- The action is applied
|
||||
THEN:
|
||||
- The suggested values replace them
|
||||
"""
|
||||
action = self.make_action(
|
||||
ai_create_missing=True,
|
||||
ai_overwrite_existing=True,
|
||||
)
|
||||
|
||||
changed = self.apply(action)
|
||||
|
||||
self.assertEqual(self.doc.title, "Suggested Title")
|
||||
self.assertEqual(self.doc.created, datetime.date(2024, 3, 5))
|
||||
self.assertIn("title", changed)
|
||||
self.assertIn("created", changed)
|
||||
|
||||
def test_tags_are_added_not_replaced(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A document that already has a tag unrelated to the suggestions
|
||||
WHEN:
|
||||
- The action is applied with overwrite existing enabled
|
||||
THEN:
|
||||
- The existing tag is kept, since suggested tags are always
|
||||
additive regardless of the overwrite setting
|
||||
"""
|
||||
kept = Tag.objects.create(name="Do Not Remove", owner=self.user)
|
||||
self.doc.tags.add(kept)
|
||||
Tag.objects.create(name="Existing Tag", owner=self.user)
|
||||
action = self.make_action(ai_overwrite_existing=True)
|
||||
|
||||
self.apply(action)
|
||||
|
||||
self.assertEqual(
|
||||
sorted(t.name for t in self.doc.tags.all()),
|
||||
["Do Not Remove", "Existing Tag"],
|
||||
)
|
||||
|
||||
def test_unselected_fields_are_untouched(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- An action that only selects the title
|
||||
WHEN:
|
||||
- The action is applied
|
||||
THEN:
|
||||
- Only the title changes, even though the LLM suggested everything
|
||||
"""
|
||||
action = self.make_action(
|
||||
ai_suggestion_fields=[WorkflowAction.AISuggestionField.TITLE],
|
||||
ai_create_missing=True,
|
||||
ai_overwrite_existing=True,
|
||||
)
|
||||
|
||||
changed = self.apply(action)
|
||||
|
||||
self.assertEqual(changed, ["title"])
|
||||
self.assertEqual(self.doc.title, "Suggested Title")
|
||||
self.assertEqual(self.doc.tags.count(), 0)
|
||||
self.assertIsNone(self.doc.correspondent)
|
||||
self.assertEqual(self.doc.created, datetime.date(2020, 1, 1))
|
||||
|
||||
def test_another_users_private_objects_are_not_matched(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A suggested tag name that exists, but is owned by someone else
|
||||
WHEN:
|
||||
- The action is applied
|
||||
THEN:
|
||||
- It is not assigned, because the document owner cannot see it
|
||||
"""
|
||||
other = User.objects.create(username="someone-else")
|
||||
Tag.objects.create(name="Existing Tag", owner=other)
|
||||
action = self.make_action(
|
||||
ai_suggestion_fields=[WorkflowAction.AISuggestionField.TAGS],
|
||||
)
|
||||
|
||||
self.apply(action)
|
||||
|
||||
self.assertEqual(self.doc.tags.count(), 0)
|
||||
|
||||
def test_unparsable_dates_are_skipped(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- Suggested dates that are not all valid
|
||||
WHEN:
|
||||
- The action is applied
|
||||
THEN:
|
||||
- The first usable date is applied and the rest ignored
|
||||
"""
|
||||
action = self.make_action(
|
||||
ai_suggestion_fields=[WorkflowAction.AISuggestionField.CREATED],
|
||||
ai_overwrite_existing=True,
|
||||
)
|
||||
|
||||
with mock.patch(
|
||||
"documents.workflows.ai.get_ai_document_classification",
|
||||
return_value={**SUGGESTIONS, "dates": ["not a date", "2019-07-04"]},
|
||||
):
|
||||
changed = apply_ai_suggestions_to_document(action, self.doc)
|
||||
|
||||
self.doc.refresh_from_db()
|
||||
self.assertEqual(changed, ["created"])
|
||||
self.assertEqual(self.doc.created, datetime.date(2019, 7, 4))
|
||||
|
||||
+40
-54
@@ -133,10 +133,12 @@ from documents.file_handling import format_filename
|
||||
from documents.filters import CorrespondentFilterSet
|
||||
from documents.filters import CustomFieldFilterSet
|
||||
from documents.filters import DocumentFilterSet
|
||||
from documents.filters import DocumentPermissionsFilter
|
||||
from documents.filters import DocumentsOrderingFilter
|
||||
from documents.filters import DocumentTypeFilterSet
|
||||
from documents.filters import ObjectOwnedOrGrantedPermissionsFilter
|
||||
from documents.filters import ObjectOwnedPermissionsFilter
|
||||
from documents.filters import PaperlessTaskFilterSet
|
||||
from documents.filters import PermittedObjectsFilter
|
||||
from documents.filters import ShareLinkBundleFilterSet
|
||||
from documents.filters import ShareLinkFilterSet
|
||||
from documents.filters import StoragePathFilterSet
|
||||
@@ -176,7 +178,6 @@ from documents.permissions import has_global_statistics_permission
|
||||
from documents.permissions import has_perms_owner_aware
|
||||
from documents.permissions import has_system_status_permission
|
||||
from documents.permissions import permitted_document_ids
|
||||
from documents.permissions import permitted_object_ids
|
||||
from documents.permissions import set_permissions_for_object
|
||||
from documents.plugins.date_parsing import get_date_parser
|
||||
from documents.schema import generate_object_with_permissions_schema
|
||||
@@ -236,15 +237,12 @@ from paperless import version
|
||||
from paperless.celery import app as celery_app
|
||||
from paperless.config import AIConfig
|
||||
from paperless.config import GeneralConfig
|
||||
from paperless.config import RemoteOCRConfig
|
||||
from paperless.models import ApplicationConfiguration
|
||||
from paperless.parsers.registry import get_parser_registry
|
||||
from paperless.parsers.remote import RemoteEngineConfig
|
||||
from paperless.serialisers import GroupSerializer
|
||||
from paperless.serialisers import UserSerializer
|
||||
from paperless.views import StandardPagination
|
||||
from paperless_ai.ai_classifier import get_ai_document_classification
|
||||
from paperless_ai.ai_classifier import get_llm_output_language
|
||||
from paperless_ai.chat import stream_chat_with_documents
|
||||
from paperless_ai.exceptions import LLMTimeoutError
|
||||
from paperless_ai.matching import extract_unmatched_names
|
||||
@@ -350,6 +348,7 @@ class IndexView(TemplateView):
|
||||
context["username"] = self.request.user.username
|
||||
context["full_name"] = self.request.user.get_full_name()
|
||||
context["styles_css"] = f"frontend/{self.get_frontend_language()}/styles.css"
|
||||
context["runtime_js"] = f"frontend/{self.get_frontend_language()}/runtime.js"
|
||||
context["polyfills_js"] = (
|
||||
f"frontend/{self.get_frontend_language()}/polyfills.js"
|
||||
)
|
||||
@@ -552,7 +551,7 @@ class CorrespondentViewSet(
|
||||
filter_backends = (
|
||||
DjangoFilterBackend,
|
||||
OrderingFilter,
|
||||
PermittedObjectsFilter,
|
||||
ObjectOwnedOrGrantedPermissionsFilter,
|
||||
)
|
||||
filterset_class = CorrespondentFilterSet
|
||||
ordering_fields = (
|
||||
@@ -593,7 +592,7 @@ class TagViewSet(PermissionsAwareDocumentCountMixin, ModelViewSet[Tag]):
|
||||
filter_backends = (
|
||||
DjangoFilterBackend,
|
||||
OrderingFilter,
|
||||
PermittedObjectsFilter,
|
||||
ObjectOwnedOrGrantedPermissionsFilter,
|
||||
)
|
||||
filterset_class = TagFilterSet
|
||||
ordering_fields = ("color", "name", "matching_algorithm", "match", "document_count")
|
||||
@@ -656,6 +655,20 @@ class TagViewSet(PermissionsAwareDocumentCountMixin, ModelViewSet[Tag]):
|
||||
update_document_parent_tags(tag, new_parent)
|
||||
|
||||
|
||||
def _get_llm_output_language(ai_config: AIConfig, request) -> str | None:
|
||||
output_language = ai_config.llm_output_language
|
||||
if (
|
||||
not output_language
|
||||
and hasattr(request.user, "ui_settings")
|
||||
and isinstance(
|
||||
request.user.ui_settings.settings,
|
||||
dict,
|
||||
)
|
||||
):
|
||||
output_language = request.user.ui_settings.settings.get("language")
|
||||
return output_language
|
||||
|
||||
|
||||
@extend_schema_view(**generate_object_with_permissions_schema(DocumentTypeSerializer))
|
||||
class DocumentTypeViewSet(
|
||||
PermissionsAwareDocumentCountMixin,
|
||||
@@ -671,7 +684,7 @@ class DocumentTypeViewSet(
|
||||
filter_backends = (
|
||||
DjangoFilterBackend,
|
||||
OrderingFilter,
|
||||
PermittedObjectsFilter,
|
||||
ObjectOwnedOrGrantedPermissionsFilter,
|
||||
)
|
||||
filterset_class = DocumentTypeFilterSet
|
||||
ordering_fields = ("name", "matching_algorithm", "match", "document_count")
|
||||
@@ -975,7 +988,7 @@ class DocumentViewSet(
|
||||
DjangoFilterBackend,
|
||||
SearchFilter,
|
||||
DocumentsOrderingFilter,
|
||||
PermittedObjectsFilter,
|
||||
DocumentPermissionsFilter,
|
||||
)
|
||||
filterset_class = DocumentFilterSet
|
||||
search_fields = ("title", "correspondent__name", "effective_content")
|
||||
@@ -1517,10 +1530,7 @@ class DocumentViewSet(
|
||||
if not ai_config.ai_enabled:
|
||||
return HttpResponseBadRequest("AI is required for this feature")
|
||||
|
||||
output_language = get_llm_output_language(
|
||||
ai_config=ai_config,
|
||||
user=request.user,
|
||||
)
|
||||
output_language = _get_llm_output_language(ai_config=ai_config, request=request)
|
||||
llm_cache_backend = ":".join(
|
||||
part
|
||||
for part in (
|
||||
@@ -2265,10 +2275,7 @@ class ChatStreamingView(GenericAPIView[Any]):
|
||||
id__in=permitted_document_ids(request.user),
|
||||
)
|
||||
|
||||
output_language = get_llm_output_language(
|
||||
ai_config=ai_config,
|
||||
user=request.user,
|
||||
)
|
||||
output_language = _get_llm_output_language(ai_config=ai_config, request=request)
|
||||
|
||||
response = StreamingHttpResponse(
|
||||
stream_chat_with_documents(
|
||||
@@ -2667,7 +2674,7 @@ class SavedViewViewSet(BulkPermissionMixin, PassUserMixin, ModelViewSet[SavedVie
|
||||
permission_classes = (IsAuthenticated, PaperlessObjectPermissions)
|
||||
filter_backends = (
|
||||
OrderingFilter,
|
||||
PermittedObjectsFilter,
|
||||
ObjectOwnedOrGrantedPermissionsFilter,
|
||||
)
|
||||
ordering_fields = ("name",)
|
||||
|
||||
@@ -3914,7 +3921,7 @@ class StoragePathViewSet(PermissionsAwareDocumentCountMixin, ModelViewSet[Storag
|
||||
filter_backends = (
|
||||
DjangoFilterBackend,
|
||||
OrderingFilter,
|
||||
PermittedObjectsFilter,
|
||||
ObjectOwnedOrGrantedPermissionsFilter,
|
||||
)
|
||||
filterset_class = StoragePathFilterSet
|
||||
ordering_fields = ("name", "path", "matching_algorithm", "match", "document_count")
|
||||
@@ -4005,11 +4012,6 @@ class UiSettingsView(GenericAPIView[Any]):
|
||||
|
||||
ui_settings["auditlog_enabled"] = settings.AUDIT_LOG_ENABLED
|
||||
|
||||
ui_settings["remote_ocr"] = {
|
||||
"configured": RemoteEngineConfig.from_app_config().engine_is_valid(),
|
||||
"mode": RemoteOCRConfig().remote_ocr_mode,
|
||||
}
|
||||
|
||||
if settings.GMAIL_OAUTH_ENABLED or settings.OUTLOOK_OAUTH_ENABLED:
|
||||
manager = PaperlessMailOAuth2Manager()
|
||||
if settings.GMAIL_OAUTH_ENABLED:
|
||||
@@ -4450,7 +4452,7 @@ class ShareLinkViewSet(
|
||||
filter_backends = (
|
||||
DjangoFilterBackend,
|
||||
OrderingFilter,
|
||||
PermittedObjectsFilter,
|
||||
ObjectOwnedOrGrantedPermissionsFilter,
|
||||
)
|
||||
filterset_class = ShareLinkFilterSet
|
||||
ordering_fields = ("created", "expiration", "document")
|
||||
@@ -4480,7 +4482,7 @@ class ShareLinkBundleViewSet(PassUserMixin, ModelViewSet[ShareLinkBundle]):
|
||||
filter_backends = (
|
||||
DjangoFilterBackend,
|
||||
OrderingFilter,
|
||||
PermittedObjectsFilter,
|
||||
ObjectOwnedOrGrantedPermissionsFilter,
|
||||
)
|
||||
filterset_class = ShareLinkBundleFilterSet
|
||||
ordering_fields = ("created", "expiration", "status")
|
||||
@@ -4763,8 +4765,10 @@ class BulkEditObjectsView(PassUserMixin):
|
||||
"document_types": DocumentTypeFilterSet,
|
||||
"storage_paths": StoragePathFilterSet,
|
||||
}[object_type]
|
||||
user_permitted_objects = object_class.objects.filter(
|
||||
id__in=permitted_object_ids(user, object_class, perm_codename),
|
||||
user_permitted_objects = get_objects_for_user_owner_aware(
|
||||
user,
|
||||
perm_codename,
|
||||
object_class,
|
||||
)
|
||||
objs = filterset_class(
|
||||
data=filters,
|
||||
@@ -4789,11 +4793,8 @@ class BulkEditObjectsView(PassUserMixin):
|
||||
|
||||
if not user.is_superuser:
|
||||
perm = f"documents.{perm_codename}"
|
||||
has_perms = (
|
||||
user.has_perm(perm)
|
||||
and not objs.exclude(
|
||||
pk__in=permitted_object_ids(user, object_class, perm_codename),
|
||||
).exists()
|
||||
has_perms = user.has_perm(perm) and all(
|
||||
has_perms_owner_aware(user, perm_codename, obj) for obj in objs
|
||||
)
|
||||
|
||||
if not has_perms:
|
||||
@@ -5294,11 +5295,7 @@ class SystemStatusView(PassUserMixin):
|
||||
class TrashView(ListModelMixin, PassUserMixin):
|
||||
permission_classes = (IsAuthenticated,)
|
||||
serializer_class = TrashSerializer
|
||||
|
||||
class _TrashPermittedObjectsFilter(PermittedObjectsFilter):
|
||||
include_granted = False
|
||||
|
||||
filter_backends = (_TrashPermittedObjectsFilter,)
|
||||
filter_backends = (ObjectOwnedPermissionsFilter,)
|
||||
pagination_class = StandardPagination
|
||||
|
||||
model = Document
|
||||
@@ -5351,26 +5348,15 @@ def serve_logo(request: HttpRequest, filename: str | None = None) -> FileRespons
|
||||
config = ApplicationConfiguration.objects.first()
|
||||
app_logo = config.app_logo
|
||||
|
||||
if app_logo:
|
||||
path = Path(app_logo.path)
|
||||
logo_name = app_logo.name
|
||||
else:
|
||||
if not settings.APP_LOGO:
|
||||
raise Http404("No logo configured")
|
||||
|
||||
logo_root = (Path(settings.MEDIA_ROOT) / "logo").resolve()
|
||||
path = (Path(settings.MEDIA_ROOT) / settings.APP_LOGO.lstrip("/")).resolve()
|
||||
if not path.is_relative_to(logo_root) or not path.is_file():
|
||||
raise Http404("Configured logo not found")
|
||||
|
||||
logo_name = path.name
|
||||
if not app_logo:
|
||||
raise Http404("No logo configured")
|
||||
|
||||
path = app_logo.path
|
||||
content_type = magic.from_file(path, mime=True) or "application/octet-stream"
|
||||
logo_file = app_logo.open("rb") if app_logo else path.open("rb")
|
||||
|
||||
return FileResponse(
|
||||
logo_file,
|
||||
app_logo.open("rb"),
|
||||
content_type=content_type,
|
||||
filename=logo_name,
|
||||
filename=app_logo.name,
|
||||
as_attachment=True,
|
||||
)
|
||||
|
||||
@@ -1,241 +0,0 @@
|
||||
import logging
|
||||
from datetime import date
|
||||
from datetime import datetime
|
||||
|
||||
from django.contrib.auth.models import User
|
||||
|
||||
from documents.models import Correspondent
|
||||
from documents.models import Document
|
||||
from documents.models import DocumentType
|
||||
from documents.models import StoragePath
|
||||
from documents.models import Tag
|
||||
from documents.models import WorkflowAction
|
||||
from paperless.config import AIConfig
|
||||
from paperless_ai.ai_classifier import get_ai_document_classification
|
||||
from paperless_ai.ai_classifier import get_llm_output_language
|
||||
from paperless_ai.matching import extract_unmatched_names
|
||||
from paperless_ai.matching import match_correspondents_by_name
|
||||
from paperless_ai.matching import match_document_types_by_name
|
||||
from paperless_ai.matching import match_storage_paths_by_name
|
||||
from paperless_ai.matching import match_tags_by_name
|
||||
|
||||
logger = logging.getLogger("paperless.workflows.ai")
|
||||
|
||||
AISuggestionField = WorkflowAction.AISuggestionField
|
||||
|
||||
# Tags use m2m relation instead
|
||||
DIRECT_FIELDS: dict[str, str] = {
|
||||
AISuggestionField.TITLE: "title",
|
||||
AISuggestionField.CORRESPONDENT: "correspondent",
|
||||
AISuggestionField.DOCUMENT_TYPE: "document_type",
|
||||
AISuggestionField.STORAGE_PATH: "storage_path",
|
||||
AISuggestionField.CREATED: "created",
|
||||
}
|
||||
|
||||
|
||||
def resolve_date(dates: list[str]) -> date | None:
|
||||
"""
|
||||
First usable date out of the suggestions, which are expected as
|
||||
YYYY-MM-DD. Document.created is a DateField, so only one can be applied.
|
||||
"""
|
||||
for value in dates:
|
||||
try:
|
||||
return datetime.strptime(value, "%Y-%m-%d").date()
|
||||
except (TypeError, ValueError):
|
||||
logger.debug("Ignoring unparsable suggested date %s", value)
|
||||
return None
|
||||
|
||||
|
||||
def resolve_object(
|
||||
model,
|
||||
names: list[str],
|
||||
matched: list,
|
||||
*,
|
||||
create_missing: bool,
|
||||
owner: User | None,
|
||||
):
|
||||
"""
|
||||
Single object from a suggestion list. The best match if there was one, else
|
||||
optionally a newly-created object. StoragePaths are excluded.
|
||||
"""
|
||||
if matched:
|
||||
return matched[0]
|
||||
|
||||
if not create_missing or model is StoragePath:
|
||||
return None
|
||||
|
||||
unmatched = extract_unmatched_names(names, matched)
|
||||
if not unmatched:
|
||||
return None
|
||||
|
||||
# (name, owner) is what MatchingModel is unique on
|
||||
obj, created = model.objects.get_or_create(
|
||||
name=unmatched[0][:128],
|
||||
owner=owner,
|
||||
)
|
||||
if created:
|
||||
logger.info("Created %s '%s' from AI suggestion", model.__name__, obj.name)
|
||||
return obj
|
||||
|
||||
|
||||
def resolve_tags(
|
||||
names: list[str],
|
||||
matched: list[Tag],
|
||||
*,
|
||||
create_missing: bool,
|
||||
owner: User | None,
|
||||
) -> list[Tag]:
|
||||
"""
|
||||
Matched tags, plus newly created ones if create_missing is set.
|
||||
"""
|
||||
tags = list(matched)
|
||||
if not create_missing:
|
||||
return tags
|
||||
|
||||
for name in extract_unmatched_names(names, matched):
|
||||
tag, created = Tag.objects.get_or_create(
|
||||
name=name[:128],
|
||||
owner=owner,
|
||||
)
|
||||
if created:
|
||||
logger.info("Created tag '%s' from AI suggestion", tag.name)
|
||||
tags.append(tag)
|
||||
return tags
|
||||
|
||||
|
||||
def apply_ai_suggestions_to_document(
|
||||
action: WorkflowAction,
|
||||
document: Document,
|
||||
logging_group=None,
|
||||
) -> list[str]:
|
||||
"""
|
||||
Get suggestions about `document` and write the chosen fields.
|
||||
|
||||
Returns the names of the fields that were actually changed.
|
||||
"""
|
||||
selected = set(action.ai_suggestion_fields or [])
|
||||
if not selected:
|
||||
logger.warning(
|
||||
"Workflow action %s has no AI suggestion fields selected, skipping",
|
||||
action.pk,
|
||||
extra={"group": logging_group},
|
||||
)
|
||||
return []
|
||||
|
||||
ai_config = AIConfig()
|
||||
if not ai_config.ai_enabled:
|
||||
logger.error(
|
||||
"AI is not enabled, cannot apply AI suggestions for document %s",
|
||||
document.pk,
|
||||
extra={"group": logging_group},
|
||||
)
|
||||
return []
|
||||
|
||||
# Workflows run without a user, so we use the document owner
|
||||
owner = document.owner
|
||||
|
||||
try:
|
||||
suggestions = get_ai_document_classification(
|
||||
document,
|
||||
owner,
|
||||
get_llm_output_language(ai_config, owner),
|
||||
)
|
||||
except ValueError:
|
||||
# A bad AI config will not fix itself, so swallow it rather than
|
||||
# letting the caller retry. Timeouts, rate limits, network errors etc
|
||||
# propagate so the queued task can back off and try again.
|
||||
logger.exception(
|
||||
"Invalid AI configuration, cannot get suggestions for document %s",
|
||||
document.pk,
|
||||
extra={"group": logging_group},
|
||||
)
|
||||
return []
|
||||
|
||||
overwrite = action.ai_overwrite_existing
|
||||
create_missing = action.ai_create_missing
|
||||
updated_fields: list[str] = []
|
||||
|
||||
def should_set(field: str) -> bool:
|
||||
# The field is selected and (overwrite or it's empty)
|
||||
return field in selected and (
|
||||
overwrite or getattr(document, DIRECT_FIELDS[field]) in (None, "")
|
||||
)
|
||||
|
||||
if should_set(AISuggestionField.TITLE):
|
||||
title = (suggestions.get("title") or "").strip()
|
||||
if title:
|
||||
# title is capped at 128 characters
|
||||
document.title = title[:128]
|
||||
updated_fields.append("title")
|
||||
|
||||
if should_set(AISuggestionField.CORRESPONDENT):
|
||||
names = suggestions.get("correspondents", [])
|
||||
correspondent = resolve_object(
|
||||
Correspondent,
|
||||
names,
|
||||
match_correspondents_by_name(names, owner),
|
||||
create_missing=create_missing,
|
||||
owner=owner,
|
||||
)
|
||||
if correspondent:
|
||||
document.correspondent = correspondent
|
||||
updated_fields.append("correspondent")
|
||||
|
||||
if should_set(AISuggestionField.DOCUMENT_TYPE):
|
||||
names = suggestions.get("document_types", [])
|
||||
document_type = resolve_object(
|
||||
DocumentType,
|
||||
names,
|
||||
match_document_types_by_name(names, owner),
|
||||
create_missing=create_missing,
|
||||
owner=owner,
|
||||
)
|
||||
if document_type:
|
||||
document.document_type = document_type
|
||||
updated_fields.append("document_type")
|
||||
|
||||
if should_set(AISuggestionField.STORAGE_PATH):
|
||||
names = suggestions.get("storage_paths", [])
|
||||
storage_path = resolve_object(
|
||||
StoragePath,
|
||||
names,
|
||||
match_storage_paths_by_name(names, owner),
|
||||
create_missing=create_missing,
|
||||
owner=owner,
|
||||
)
|
||||
if storage_path:
|
||||
document.storage_path = storage_path
|
||||
updated_fields.append("storage_path")
|
||||
|
||||
if should_set(AISuggestionField.CREATED):
|
||||
created = resolve_date(suggestions.get("dates", []))
|
||||
if created:
|
||||
document.created = created
|
||||
updated_fields.append("created")
|
||||
|
||||
if updated_fields:
|
||||
# save fields and update modified
|
||||
document.save(update_fields=[*updated_fields, "modified"])
|
||||
|
||||
if AISuggestionField.TAGS in selected:
|
||||
names = suggestions.get("tags", [])
|
||||
tags = resolve_tags(
|
||||
names,
|
||||
match_tags_by_name(names, owner),
|
||||
create_missing=create_missing,
|
||||
owner=owner,
|
||||
)
|
||||
if tags:
|
||||
# Suggested tags are always added, so overwrite_existing
|
||||
# does not really apply here
|
||||
document.add_nested_tags(tags)
|
||||
updated_fields.append("tags")
|
||||
|
||||
logger.info(
|
||||
"Applied AI suggestions %s to document %s",
|
||||
updated_fields or "(none)",
|
||||
document.pk,
|
||||
extra={"group": logging_group},
|
||||
)
|
||||
|
||||
return updated_fields
|
||||
@@ -2,7 +2,7 @@ msgid ""
|
||||
msgstr ""
|
||||
"Project-Id-Version: paperless-ngx\n"
|
||||
"Report-Msgid-Bugs-To: \n"
|
||||
"POT-Creation-Date: 2026-08-10 02:25+0000\n"
|
||||
"POT-Creation-Date: 2026-08-04 15:02+0000\n"
|
||||
"PO-Revision-Date: 2022-02-17 04:17\n"
|
||||
"Last-Translator: \n"
|
||||
"Language-Team: English\n"
|
||||
@@ -21,39 +21,39 @@ msgstr ""
|
||||
msgid "Documents"
|
||||
msgstr ""
|
||||
|
||||
#: documents/filters.py:471
|
||||
#: documents/filters.py:472
|
||||
msgid "Value must be valid JSON."
|
||||
msgstr ""
|
||||
|
||||
#: documents/filters.py:490
|
||||
#: documents/filters.py:491
|
||||
msgid "Invalid custom field query expression"
|
||||
msgstr ""
|
||||
|
||||
#: documents/filters.py:500
|
||||
#: documents/filters.py:501
|
||||
msgid "Invalid expression list. Must be nonempty."
|
||||
msgstr ""
|
||||
|
||||
#: documents/filters.py:521
|
||||
#: documents/filters.py:522
|
||||
msgid "Invalid logical operator {op!r}"
|
||||
msgstr ""
|
||||
|
||||
#: documents/filters.py:535
|
||||
#: documents/filters.py:536
|
||||
msgid "Maximum number of query conditions exceeded."
|
||||
msgstr ""
|
||||
|
||||
#: documents/filters.py:599
|
||||
#: documents/filters.py:600
|
||||
msgid "{name!r} is not a valid custom field."
|
||||
msgstr ""
|
||||
|
||||
#: documents/filters.py:636
|
||||
#: documents/filters.py:637
|
||||
msgid "{data_type} does not support query expr {expr!r}."
|
||||
msgstr ""
|
||||
|
||||
#: documents/filters.py:755 documents/models.py:136
|
||||
#: documents/filters.py:756 documents/models.py:136
|
||||
msgid "Maximum nesting depth exceeded."
|
||||
msgstr ""
|
||||
|
||||
#: documents/filters.py:1079
|
||||
#: documents/filters.py:1098
|
||||
msgid "Custom field not found"
|
||||
msgstr ""
|
||||
|
||||
@@ -1352,7 +1352,7 @@ msgid "workflow runs"
|
||||
msgstr ""
|
||||
|
||||
#: documents/serialisers.py:521 documents/serialisers.py:873
|
||||
#: documents/serialisers.py:2767 documents/views.py:299 documents/views.py:2555
|
||||
#: documents/serialisers.py:2765 documents/views.py:300 documents/views.py:2557
|
||||
#: paperless_mail/serialisers.py:155
|
||||
msgid "Insufficient permissions."
|
||||
msgstr ""
|
||||
@@ -1361,39 +1361,39 @@ msgstr ""
|
||||
msgid "Invalid color."
|
||||
msgstr ""
|
||||
|
||||
#: documents/serialisers.py:2244
|
||||
#: documents/serialisers.py:2242
|
||||
#, python-format
|
||||
msgid "File type %(type)s not supported"
|
||||
msgstr ""
|
||||
|
||||
#: documents/serialisers.py:2288
|
||||
#: documents/serialisers.py:2286
|
||||
#, python-format
|
||||
msgid "Custom field id must be an integer: %(id)s"
|
||||
msgstr ""
|
||||
|
||||
#: documents/serialisers.py:2295
|
||||
#: documents/serialisers.py:2293
|
||||
#, python-format
|
||||
msgid "Custom field with id %(id)s does not exist"
|
||||
msgstr ""
|
||||
|
||||
#: documents/serialisers.py:2312 documents/serialisers.py:2322
|
||||
#: documents/serialisers.py:2310 documents/serialisers.py:2320
|
||||
msgid ""
|
||||
"Custom fields must be a list of integers or an object mapping ids to values."
|
||||
msgstr ""
|
||||
|
||||
#: documents/serialisers.py:2317
|
||||
#: documents/serialisers.py:2315
|
||||
msgid "Some custom fields don't exist or were specified twice."
|
||||
msgstr ""
|
||||
|
||||
#: documents/serialisers.py:2464
|
||||
#: documents/serialisers.py:2462
|
||||
msgid "Invalid variable detected."
|
||||
msgstr ""
|
||||
|
||||
#: documents/serialisers.py:2823
|
||||
#: documents/serialisers.py:2821
|
||||
msgid "Duplicate document identifiers are not allowed."
|
||||
msgstr ""
|
||||
|
||||
#: documents/serialisers.py:2853 documents/views.py:4509
|
||||
#: documents/serialisers.py:2851 documents/views.py:4511
|
||||
#, python-format
|
||||
msgid "Documents not found: %(ids)s"
|
||||
msgstr ""
|
||||
@@ -1661,36 +1661,36 @@ msgstr ""
|
||||
msgid "Unable to parse URI {value}"
|
||||
msgstr ""
|
||||
|
||||
#: documents/views.py:292 documents/views.py:2552
|
||||
#: documents/views.py:293 documents/views.py:2554
|
||||
msgid "Invalid more_like_id"
|
||||
msgstr ""
|
||||
|
||||
#: documents/views.py:1566
|
||||
#: documents/views.py:1568
|
||||
msgid "Invalid AI configuration."
|
||||
msgstr ""
|
||||
|
||||
#: documents/views.py:1575
|
||||
#: documents/views.py:1577
|
||||
msgid "AI backend request timed out."
|
||||
msgstr ""
|
||||
|
||||
#: documents/views.py:2377 documents/views.py:2698
|
||||
#: documents/views.py:2379 documents/views.py:2700
|
||||
msgid "Specify only one of text, title_search, query, or more_like_id."
|
||||
msgstr ""
|
||||
|
||||
#: documents/views.py:4522
|
||||
#: documents/views.py:4524
|
||||
#, python-format
|
||||
msgid "Insufficient permissions to share document %(id)s."
|
||||
msgstr ""
|
||||
|
||||
#: documents/views.py:4568
|
||||
#: documents/views.py:4570
|
||||
msgid "Bundle is already being processed."
|
||||
msgstr ""
|
||||
|
||||
#: documents/views.py:4629
|
||||
#: documents/views.py:4631
|
||||
msgid "The share link bundle is still being prepared. Please try again later."
|
||||
msgstr ""
|
||||
|
||||
#: documents/views.py:4639
|
||||
#: documents/views.py:4641
|
||||
msgid "The share link bundle is unavailable."
|
||||
msgstr ""
|
||||
|
||||
|
||||
@@ -19,10 +19,7 @@ class AutoLoginMiddleware(MiddlewareMixin):
|
||||
if request.path.startswith("/api/token/") and request.method == "POST":
|
||||
return None
|
||||
try:
|
||||
request.user = User.objects.get(
|
||||
username=settings.AUTO_LOGIN_USERNAME,
|
||||
is_active=True,
|
||||
)
|
||||
request.user = User.objects.get(username=settings.AUTO_LOGIN_USERNAME)
|
||||
auth.login(
|
||||
request=request,
|
||||
user=request.user,
|
||||
|
||||
+3
-17
@@ -339,30 +339,16 @@ def check_deprecated_v2_ocr_env_vars(
|
||||
|
||||
@register()
|
||||
def check_remote_parser_configured(app_configs: Any, **kwargs: Any) -> list[Error]:
|
||||
# Import here because checks.py runs before the app registry is ready
|
||||
from paperless.models import RemoteOCRMode
|
||||
|
||||
errors = []
|
||||
|
||||
if settings.REMOTE_OCR_ENGINE == "azureai" and not (
|
||||
settings.REMOTE_OCR_ENDPOINT and settings.REMOTE_OCR_API_KEY
|
||||
):
|
||||
errors.append(
|
||||
return [
|
||||
Error(
|
||||
"Azure AI remote parser requires endpoint and API key to be configured.",
|
||||
),
|
||||
)
|
||||
]
|
||||
|
||||
valid_modes = {mode.value for mode in RemoteOCRMode}
|
||||
if settings.REMOTE_OCR_MODE not in valid_modes:
|
||||
errors.append(
|
||||
Error(
|
||||
f"PAPERLESS_REMOTE_OCR_MODE is set to {settings.REMOTE_OCR_MODE!r}, "
|
||||
f"expected one of {sorted(valid_modes)}.",
|
||||
),
|
||||
)
|
||||
|
||||
return errors
|
||||
return []
|
||||
|
||||
|
||||
def get_tesseract_langs():
|
||||
|
||||
@@ -9,7 +9,6 @@ from paperless.models import CleanChoices
|
||||
from paperless.models import ColorConvertChoices
|
||||
from paperless.models import ModeChoices
|
||||
from paperless.models import OutputTypeChoices
|
||||
from paperless.models import RemoteOCRMode
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
@@ -186,45 +185,6 @@ class GeneralConfig(BaseConfig):
|
||||
self.app_logo = app_config.app_logo.url if app_config.app_logo else None
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class RemoteOCRConfig(BaseConfig):
|
||||
"""
|
||||
Settings for the remote (cloud) OCR parser
|
||||
"""
|
||||
|
||||
remote_ocr_engine: str | None = dataclasses.field(init=False)
|
||||
remote_ocr_api_key: str | None = dataclasses.field(init=False)
|
||||
remote_ocr_endpoint: str | None = dataclasses.field(init=False)
|
||||
remote_ocr_mode: RemoteOCRMode = dataclasses.field(init=False)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
app_config = self._get_config_instance()
|
||||
|
||||
self.remote_ocr_engine = (
|
||||
app_config.remote_ocr_engine or settings.REMOTE_OCR_ENGINE
|
||||
)
|
||||
self.remote_ocr_api_key = (
|
||||
app_config.remote_ocr_api_key or settings.REMOTE_OCR_API_KEY
|
||||
)
|
||||
self.remote_ocr_endpoint = (
|
||||
app_config.remote_ocr_endpoint or settings.REMOTE_OCR_ENDPOINT
|
||||
)
|
||||
self.remote_ocr_mode = app_config.remote_ocr_mode or RemoteOCRMode(
|
||||
settings.REMOTE_OCR_MODE,
|
||||
)
|
||||
|
||||
@property
|
||||
def remote_ocr_by_default(self) -> bool:
|
||||
"""
|
||||
Whether every supported document goes to the remote engine.
|
||||
|
||||
When False the remote engine is used only for documents that
|
||||
explicitly asked for it, i.e. a workflow matched during consumption or
|
||||
the user ticked the box when reprocessing.
|
||||
"""
|
||||
return self.remote_ocr_mode == RemoteOCRMode.ALWAYS
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class AIConfig(BaseConfig):
|
||||
"""
|
||||
|
||||
@@ -1,44 +0,0 @@
|
||||
# Generated by Django 5.2.16 on 2026-08-10 14:37
|
||||
|
||||
from django.db import migrations
|
||||
from django.db import models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("paperless", "0013_applicationconfiguration_llm_request_timeout"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name="applicationconfiguration",
|
||||
name="remote_ocr_api_key",
|
||||
field=models.CharField(
|
||||
blank=True,
|
||||
max_length=1024,
|
||||
null=True,
|
||||
verbose_name="Sets the remote OCR API key",
|
||||
),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="applicationconfiguration",
|
||||
name="remote_ocr_endpoint",
|
||||
field=models.CharField(
|
||||
blank=True,
|
||||
max_length=256,
|
||||
null=True,
|
||||
verbose_name="Sets the remote OCR endpoint",
|
||||
),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="applicationconfiguration",
|
||||
name="remote_ocr_engine",
|
||||
field=models.CharField(
|
||||
blank=True,
|
||||
choices=[("azureai", "Azure AI Document Intelligence")],
|
||||
max_length=32,
|
||||
null=True,
|
||||
verbose_name="Sets the remote OCR engine",
|
||||
),
|
||||
),
|
||||
]
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user