mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-09-01 23:47:15 +00:00
Compare commits
38
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ee19e8dcff | ||
|
|
b9cf877029 | ||
|
|
82cc1016ad | ||
|
|
dcedb531c6 | ||
|
|
8bedb07cea | ||
|
|
5e34d566a2 | ||
|
|
601ecce3f1 | ||
|
|
d67beba9f6 | ||
|
|
3bc8ee8425 | ||
|
|
fdef4a99a7 | ||
|
|
d16d05a391 | ||
|
|
10789e63cb | ||
|
|
2197781b39 | ||
|
|
981492bb33 | ||
|
|
0ef5ef5826 | ||
|
|
a6b1763149 | ||
|
|
90531525e2 | ||
|
|
8f00bfa931 | ||
|
|
a0479f1d9b | ||
|
|
9b8bd21044 | ||
|
|
a60172bc6f | ||
|
|
6c5bc1c0ff | ||
|
|
f4a7c478a9 | ||
|
|
bc07c19d9b | ||
|
|
bfcee24572 | ||
|
|
4fec4b0948 | ||
|
|
3e4ffc4132 | ||
|
|
6d61bcee7e | ||
|
|
d78754bff1 | ||
|
|
5c5b1ee6b5 | ||
|
|
08f2f4bfe2 | ||
|
|
f993462973 | ||
|
|
ae70b8d60f | ||
|
|
38db6b51db | ||
|
|
31e9f4272c | ||
|
|
b8659c1af3 | ||
|
|
741115b36b | ||
|
|
1211db5cbb |
@@ -61,7 +61,7 @@ def replace_with_symlinks(
|
|||||||
total_duplicates = 0
|
total_duplicates = 0
|
||||||
space_saved = 0
|
space_saved = 0
|
||||||
|
|
||||||
for file_hash, file_list in duplicate_groups.items():
|
for file_list in duplicate_groups.values():
|
||||||
# Keep the first file as the original, replace others with symlinks
|
# Keep the first file as the original, replace others with symlinks
|
||||||
original_file = file_list[0]
|
original_file = file_list[0]
|
||||||
duplicates = file_list[1:]
|
duplicates = file_list[1:]
|
||||||
|
|||||||
@@ -1,5 +1,26 @@
|
|||||||
# Changelog
|
# Changelog
|
||||||
|
|
||||||
|
## paperless-ngx 3.1.2
|
||||||
|
|
||||||
|
### Bug Fixes
|
||||||
|
|
||||||
|
- Fix: fix dark mode select disabled color, ensure disabled cursor on display mode dropdown [@shamoon](https://github.com/shamoon) ([#13881](https://github.com/paperless-ngx/paperless-ngx/pull/13881))
|
||||||
|
- Fix: add disable to the drag-drop list component [@shamoon](https://github.com/shamoon) ([#13880](https://github.com/paperless-ngx/paperless-ngx/pull/13880))
|
||||||
|
|
||||||
|
### Documentation
|
||||||
|
|
||||||
|
- Chore: update screenshots for v3+ [@shamoon](https://github.com/shamoon) ([#13883](https://github.com/paperless-ngx/paperless-ngx/pull/13883))
|
||||||
|
|
||||||
|
### All App Changes
|
||||||
|
|
||||||
|
<details>
|
||||||
|
<summary>2 changes</summary>
|
||||||
|
|
||||||
|
- Fix: fix dark mode select disabled color, ensure disabled cursor on display mode dropdown [@shamoon](https://github.com/shamoon) ([#13881](https://github.com/paperless-ngx/paperless-ngx/pull/13881))
|
||||||
|
- Fix: add disable to the drag-drop list component [@shamoon](https://github.com/shamoon) ([#13880](https://github.com/paperless-ngx/paperless-ngx/pull/13880))
|
||||||
|
|
||||||
|
</details>
|
||||||
|
|
||||||
## paperless-ngx 3.1.1
|
## paperless-ngx 3.1.1
|
||||||
|
|
||||||
### Bug Fixes
|
### Bug Fixes
|
||||||
|
|||||||
@@ -2088,6 +2088,12 @@ password. All of these options come from their similarly-named [Django settings]
|
|||||||
|
|
||||||
Defaults to "always".
|
Defaults to "always".
|
||||||
|
|
||||||
|
#### [`PAPERLESS_REMOTE_OCR_ALLOW_INTERNAL_ENDPOINTS=<bool>`](#PAPERLESS_REMOTE_OCR_ALLOW_INTERNAL_ENDPOINTS) {#PAPERLESS_REMOTE_OCR_ALLOW_INTERNAL_ENDPOINTS}
|
||||||
|
|
||||||
|
: If set to false, Paperless blocks remote OCR endpoint URLs that resolve to non-public addresses (e.g., localhost, etc).
|
||||||
|
|
||||||
|
Defaults to True.
|
||||||
|
|
||||||
## AI {#ai}
|
## AI {#ai}
|
||||||
|
|
||||||
#### [`PAPERLESS_AI_ENABLED=<bool>`](#PAPERLESS_AI_ENABLED) {#PAPERLESS_AI_ENABLED}
|
#### [`PAPERLESS_AI_ENABLED=<bool>`](#PAPERLESS_AI_ENABLED) {#PAPERLESS_AI_ENABLED}
|
||||||
|
|||||||
+100
-47
@@ -47,7 +47,7 @@ dependencies = [
|
|||||||
"httpx-oauth~=0.17",
|
"httpx-oauth~=0.17",
|
||||||
"ijson>=3.5.1",
|
"ijson>=3.5.1",
|
||||||
"imap-tools~=1.14.0",
|
"imap-tools~=1.14.0",
|
||||||
"jinja2~=3.1.5",
|
"jinja2~=3.1.6",
|
||||||
"langdetect~=1.0.9",
|
"langdetect~=1.0.9",
|
||||||
"llama-index-core>=0.14.23",
|
"llama-index-core>=0.14.23",
|
||||||
"llama-index-embeddings-huggingface>=0.6.1",
|
"llama-index-embeddings-huggingface>=0.6.1",
|
||||||
@@ -186,64 +186,117 @@ line-ending = "lf"
|
|||||||
# https://docs.astral.sh/ruff/rules/
|
# https://docs.astral.sh/ruff/rules/
|
||||||
select = [ "E4", "E7", "E9", "F" ]
|
select = [ "E4", "E7", "E9", "F" ]
|
||||||
extend-select = [
|
extend-select = [
|
||||||
"COM", # https://docs.astral.sh/ruff/rules/#flake8-commas-com
|
"ASYNC", # https://docs.astral.sh/ruff/rules/#flake8-async-async
|
||||||
"DJ", # https://docs.astral.sh/ruff/rules/#flake8-django-dj
|
"B002", # https://docs.astral.sh/ruff/rules/#flake8-bugbear-b
|
||||||
"EXE", # https://docs.astral.sh/ruff/rules/#flake8-executable-exe
|
"B003",
|
||||||
"FBT", # https://docs.astral.sh/ruff/rules/#flake8-boolean-trap-fbt
|
"B004",
|
||||||
"FLY", # https://docs.astral.sh/ruff/rules/#flynt-fly
|
"B005",
|
||||||
"G201", # https://docs.astral.sh/ruff/rules/#flake8-logging-format-g
|
"B006",
|
||||||
"I", # https://docs.astral.sh/ruff/rules/#isort-i
|
"B008",
|
||||||
"ICN", # https://docs.astral.sh/ruff/rules/#flake8-import-conventions-icn
|
"B009",
|
||||||
"INP", # https://docs.astral.sh/ruff/rules/#flake8-no-pep420-inp
|
"B010",
|
||||||
"ISC", # https://docs.astral.sh/ruff/rules/#flake8-implicit-str-concat-isc
|
"B012",
|
||||||
"PIE", # https://docs.astral.sh/ruff/rules/#flake8-pie-pie
|
"B013",
|
||||||
"PLC", # https://docs.astral.sh/ruff/rules/#pylint-pl
|
"B014",
|
||||||
"PLE", # https://docs.astral.sh/ruff/rules/#pylint-pl
|
"B015",
|
||||||
"PTH", # https://docs.astral.sh/ruff/rules/#flake8-use-pathlib-pth
|
"B016",
|
||||||
"Q", # https://docs.astral.sh/ruff/rules/#flake8-quotes-q
|
"B017",
|
||||||
"RSE", # https://docs.astral.sh/ruff/rules/#flake8-raise-rse
|
"B018",
|
||||||
"RUF", # https://docs.astral.sh/ruff/rules/#ruff-specific-rules-ruf
|
"B019",
|
||||||
"SIM", # https://docs.astral.sh/ruff/rules/#flake8-simplify-sim
|
"B020",
|
||||||
"T20", # https://docs.astral.sh/ruff/rules/#flake8-print-t20
|
"B021",
|
||||||
"TC", # https://docs.astral.sh/ruff/rules/#flake8-type-checking-tc
|
"B022",
|
||||||
"TID", # https://docs.astral.sh/ruff/rules/#flake8-tidy-imports-tid
|
"B023",
|
||||||
"UP", # https://docs.astral.sh/ruff/rules/#pyupgrade-up
|
"B025",
|
||||||
"W", # https://docs.astral.sh/ruff/rules/#pycodestyle-e-w
|
"B026",
|
||||||
|
"B029",
|
||||||
|
"B030",
|
||||||
|
"B031",
|
||||||
|
"B032",
|
||||||
|
"B033",
|
||||||
|
"B035",
|
||||||
|
"B039",
|
||||||
|
"C4", # https://docs.astral.sh/ruff/rules/#flake8-comprehensions-c4
|
||||||
|
"COM", # https://docs.astral.sh/ruff/rules/#flake8-commas-com
|
||||||
|
"D419", # https://docs.astral.sh/ruff/rules/#pydocstyle-d
|
||||||
|
"DJ", # https://docs.astral.sh/ruff/rules/#flake8-django-dj
|
||||||
|
"DTZ", # https://docs.astral.sh/ruff/rules/#flake8-datetimez-dtz
|
||||||
|
"EXE", # https://docs.astral.sh/ruff/rules/#flake8-executable-exe
|
||||||
|
"FA", # https://docs.astral.sh/ruff/rules/#flake8-future-annotations-fa
|
||||||
|
"FBT", # https://docs.astral.sh/ruff/rules/#flake8-boolean-trap-fbt
|
||||||
|
"FLY", # https://docs.astral.sh/ruff/rules/#flynt-fly
|
||||||
|
"FURB", # https://docs.astral.sh/ruff/rules/#refurb-furb
|
||||||
|
"G", # https://docs.astral.sh/ruff/rules/#flake8-logging-format-g
|
||||||
|
"I", # https://docs.astral.sh/ruff/rules/#isort-i
|
||||||
|
"ICN", # https://docs.astral.sh/ruff/rules/#flake8-import-conventions-icn
|
||||||
|
"INP", # https://docs.astral.sh/ruff/rules/#flake8-no-pep420-inp
|
||||||
|
"INT", # https://docs.astral.sh/ruff/rules/#flake8-gettext-int
|
||||||
|
"ISC", # https://docs.astral.sh/ruff/rules/#flake8-implicit-str-concat-isc
|
||||||
|
"LOG", # https://docs.astral.sh/ruff/rules/#flake8-logging-log
|
||||||
|
"N999", # https://docs.astral.sh/ruff/rules/#pep8-naming-n
|
||||||
|
"PERF101", # https://docs.astral.sh/ruff/rules/#perflint-perf
|
||||||
|
"PERF102",
|
||||||
|
"PERF402",
|
||||||
|
"PGH005", # https://docs.astral.sh/ruff/rules/#pygrep-hooks-pgh
|
||||||
|
"PIE", # https://docs.astral.sh/ruff/rules/#flake8-pie-pie
|
||||||
|
"PLC", # https://docs.astral.sh/ruff/rules/#pylint-pl
|
||||||
|
"PLE", # https://docs.astral.sh/ruff/rules/#error-ple
|
||||||
|
"PLR0124", # https://docs.astral.sh/ruff/rules/#refactor-plr
|
||||||
|
"PLR0133",
|
||||||
|
"PLR0206",
|
||||||
|
"PLR0402",
|
||||||
|
"PLR1704",
|
||||||
|
"PLR1708",
|
||||||
|
"PLR1711",
|
||||||
|
"PLR1716",
|
||||||
|
"PLR1722",
|
||||||
|
"PLR1730",
|
||||||
|
"PLR1733",
|
||||||
|
"PLR1736",
|
||||||
|
"PLR2044",
|
||||||
|
"PLW", # https://docs.astral.sh/ruff/rules/#warning-plw
|
||||||
|
"PT010", # https://docs.astral.sh/ruff/rules/#flake8-pytest-style-pt
|
||||||
|
"PT014",
|
||||||
|
"PT020",
|
||||||
|
"PT025",
|
||||||
|
"PT026",
|
||||||
|
"PT031",
|
||||||
|
"PTH", # https://docs.astral.sh/ruff/rules/#flake8-use-pathlib-pth
|
||||||
|
"Q", # https://docs.astral.sh/ruff/rules/#flake8-quotes-q
|
||||||
|
"RSE", # https://docs.astral.sh/ruff/rules/#flake8-raise-rse
|
||||||
|
"RUF", # https://docs.astral.sh/ruff/rules/#ruff-specific-rules-ruf
|
||||||
|
"S102", # https://docs.astral.sh/ruff/rules/#flake8-bandit-s
|
||||||
|
"S110",
|
||||||
|
"S112",
|
||||||
|
"S113",
|
||||||
|
"SIM", # https://docs.astral.sh/ruff/rules/#flake8-simplify-sim
|
||||||
|
"T100", # https://docs.astral.sh/ruff/rules/#flake8-debugger-t10
|
||||||
|
"T20", # https://docs.astral.sh/ruff/rules/#flake8-print-t20
|
||||||
|
"TC", # https://docs.astral.sh/ruff/rules/#flake8-type-checking-tc
|
||||||
|
"TID", # https://docs.astral.sh/ruff/rules/#flake8-tidy-imports-tid
|
||||||
|
"TRY002", # https://docs.astral.sh/ruff/rules/#tryceratops-try
|
||||||
|
"TRY004",
|
||||||
|
"TRY201",
|
||||||
|
"TRY203",
|
||||||
|
"TRY401",
|
||||||
|
"UP", # https://docs.astral.sh/ruff/rules/#pyupgrade-up
|
||||||
|
"W", # https://docs.astral.sh/ruff/rules/#pycodestyle-e-w
|
||||||
|
"YTT", # https://docs.astral.sh/ruff/rules/#flake8-2020-ytt
|
||||||
]
|
]
|
||||||
ignore = [
|
ignore = [
|
||||||
"DJ001",
|
"DJ001",
|
||||||
"PLC0415",
|
"PLC0415",
|
||||||
"RUF012",
|
"RUF012",
|
||||||
"SIM105",
|
"SIM105",
|
||||||
|
"G004", # Logging statement uses f-string - good to do, but a large diff
|
||||||
]
|
]
|
||||||
# Migrations
|
# Migrations
|
||||||
per-file-ignores."*/migrations/*.py" = [
|
per-file-ignores."*/migrations/*.py" = []
|
||||||
"E501",
|
|
||||||
"SIM",
|
|
||||||
"T201",
|
|
||||||
]
|
|
||||||
# Testing
|
# Testing
|
||||||
per-file-ignores."*/tests/*.py" = [
|
per-file-ignores."*/tests/*.py" = [
|
||||||
"E501",
|
"DTZ",
|
||||||
"SIM117",
|
"SIM117",
|
||||||
]
|
]
|
||||||
per-file-ignores.".github/scripts/*.py" = [
|
|
||||||
"E501",
|
|
||||||
"INP001",
|
|
||||||
"SIM117",
|
|
||||||
]
|
|
||||||
# Docker specific
|
|
||||||
per-file-ignores."docker/rootfs/usr/local/bin/wait-for-redis.py" = [
|
|
||||||
"INP001",
|
|
||||||
"T201",
|
|
||||||
]
|
|
||||||
per-file-ignores."docker/wait-for-redis.py" = [
|
|
||||||
"INP001",
|
|
||||||
"T201",
|
|
||||||
]
|
|
||||||
per-file-ignores."src/documents/models.py" = [
|
|
||||||
"SIM115",
|
|
||||||
]
|
|
||||||
isort.force-single-line = true
|
isort.force-single-line = true
|
||||||
|
|
||||||
[tool.codespell]
|
[tool.codespell]
|
||||||
|
|||||||
@@ -111,7 +111,7 @@
|
|||||||
</h6>
|
</h6>
|
||||||
<ul class="nav flex-column mb-2" cdkDropList (cdkDropListDropped)="onDrop($event)">
|
<ul class="nav flex-column mb-2" cdkDropList (cdkDropListDropped)="onDrop($event)">
|
||||||
@for (view of savedViewService.sidebarViews; track view.id) {
|
@for (view of savedViewService.sidebarViews; track view.id) {
|
||||||
<li class="nav-item w-100 app-link" cdkDrag [cdkDragDisabled]="!settingsService.organizingSidebarSavedViews() || !canSaveSettings"
|
<li class="nav-item app-link" cdkDrag [cdkDragDisabled]="!settingsService.organizingSidebarSavedViews() || !canSaveSettings"
|
||||||
cdkDragPreviewContainer="parent" cdkDragPreviewClass="navItemDrag" (cdkDragStarted)="onDragStart($event)"
|
cdkDragPreviewContainer="parent" cdkDragPreviewClass="navItemDrag" (cdkDragStarted)="onDragStart($event)"
|
||||||
(cdkDragEnded)="onDragEnd($event)">
|
(cdkDragEnded)="onDragEnd($event)">
|
||||||
<a class="nav-link" routerLink="view/{{view.id}}"
|
<a class="nav-link" routerLink="view/{{view.id}}"
|
||||||
@@ -128,7 +128,7 @@
|
|||||||
}
|
}
|
||||||
</a>
|
</a>
|
||||||
@if (settingsService.organizingSidebarSavedViews() && canSaveSettings) {
|
@if (settingsService.organizingSidebarSavedViews() && canSaveSettings) {
|
||||||
<div class="position-absolute end-0 top-0 px-3 py-2" [class.me-n3]="slimSidebarEnabled" cdkDragHandle>
|
<div class="position-absolute end-0 top-0 px-1 py-2" [class.me-n2]="slimSidebarEnabled" cdkDragHandle>
|
||||||
<i-bs name="grip-vertical"></i-bs>
|
<i-bs name="grip-vertical"></i-bs>
|
||||||
</div>
|
</div>
|
||||||
}
|
}
|
||||||
@@ -332,7 +332,7 @@
|
|||||||
</li>
|
</li>
|
||||||
<li class="nav-item" [class.visually-hidden]="slimSidebarEnabled">
|
<li class="nav-item" [class.visually-hidden]="slimSidebarEnabled">
|
||||||
<div class="text-muted small d-flex align-items-center flex-wrap nav-label">
|
<div class="text-muted small d-flex align-items-center flex-wrap nav-label">
|
||||||
<div class="me-3">
|
<div class="me-2">
|
||||||
<a class="text-muted text-decoration-none" target="_blank" rel="noopener noreferrer"
|
<a class="text-muted text-decoration-none" target="_blank" rel="noopener noreferrer"
|
||||||
href="https://github.com/paperless-ngx/paperless-ngx" ngbPopover="GitHub" i18n-ngbPopover
|
href="https://github.com/paperless-ngx/paperless-ngx" ngbPopover="GitHub" i18n-ngbPopover
|
||||||
[disablePopover]="!slimSidebarPopoversEnabled" placement="end" container="body"
|
[disablePopover]="!slimSidebarPopoversEnabled" placement="end" container="body"
|
||||||
@@ -341,7 +341,7 @@
|
|||||||
</a>
|
</a>
|
||||||
</div>
|
</div>
|
||||||
@if (!settingsService.updateCheckingIsSet || appRemoteVersion()) {
|
@if (!settingsService.updateCheckingIsSet || appRemoteVersion()) {
|
||||||
<div class="version-check">
|
<div class="version-check d-flex align-items-center">
|
||||||
<ng-template #updateAvailablePopContent>
|
<ng-template #updateAvailablePopContent>
|
||||||
<span class="small">Paperless-ngx {{ appRemoteVersion().version }} <ng-container i18n>is
|
<span class="small">Paperless-ngx {{ appRemoteVersion().version }} <ng-container i18n>is
|
||||||
available.</ng-container><br /><ng-container i18n>Click to view.</ng-container></span>
|
available.</ng-container><br /><ng-container i18n>Click to view.</ng-container></span>
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
@if (useDropdown) {
|
@if (useDropdown) {
|
||||||
<div class="btn-group w-100" role="group" ngbDropdown #dropdown="ngbDropdown" (openChange)="onOpenChange($event)" [popperOptions]="popperOptions">
|
<div class="btn-group w-100" role="group" ngbDropdown #dropdown="ngbDropdown" (openChange)="onOpenChange($event)" [popperOptions]="popperOptions">
|
||||||
<button class="btn btn-sm btn-outline-primary" id="dropdown_toggle" ngbDropdownToggle [disabled]="disabled" [aria-label]="title">
|
<button class="btn btn-sm" [ngClass]="!editing && isActive ? 'btn-primary' : 'btn-outline-primary'" id="dropdown_toggle" ngbDropdownToggle [disabled]="disabled" [aria-label]="title">
|
||||||
<i-bs name="{{icon}}"></i-bs><div class="d-none d-sm-inline ms-1">{{title}}</div>
|
<i-bs name="{{icon}}"></i-bs><div class="d-none d-sm-inline ms-1">{{title}}</div>
|
||||||
@if (isActive) {
|
@if (isActive) {
|
||||||
<pngx-clearable-badge [selected]="isActive" (cleared)="reset()"></pngx-clearable-badge>
|
<pngx-clearable-badge [selected]="isActive" (cleared)="reset()"></pngx-clearable-badge>
|
||||||
|
|||||||
+7
-4
@@ -1,5 +1,6 @@
|
|||||||
import {
|
import {
|
||||||
getLocaleNumberSymbol,
|
getLocaleNumberSymbol,
|
||||||
|
NgClass,
|
||||||
NgTemplateOutlet,
|
NgTemplateOutlet,
|
||||||
NumberSymbol,
|
NumberSymbol,
|
||||||
} from '@angular/common'
|
} from '@angular/common'
|
||||||
@@ -48,25 +49,26 @@ import { ClearableBadgeComponent } from '../clearable-badge/clearable-badge.comp
|
|||||||
import { DocumentLinkComponent } from '../input/document-link/document-link.component'
|
import { DocumentLinkComponent } from '../input/document-link/document-link.component'
|
||||||
|
|
||||||
export class CustomFieldQueriesModel {
|
export class CustomFieldQueriesModel {
|
||||||
private _queries: CustomFieldQueryElement[] = []
|
private readonly _queries = signal<CustomFieldQueryElement[]>([])
|
||||||
private rootSubscriptions: Subscription[] = []
|
private rootSubscriptions: Subscription[] = []
|
||||||
|
|
||||||
public readonly changed = new Subject<CustomFieldQueriesModel>()
|
public readonly changed = new Subject<CustomFieldQueriesModel>()
|
||||||
|
|
||||||
public get queries(): CustomFieldQueryElement[] {
|
public get queries(): CustomFieldQueryElement[] {
|
||||||
return this._queries
|
return this._queries()
|
||||||
}
|
}
|
||||||
|
|
||||||
public set queries(value: CustomFieldQueryElement[]) {
|
public set queries(value: CustomFieldQueryElement[]) {
|
||||||
this.teardownRootSubscriptions()
|
this.teardownRootSubscriptions()
|
||||||
this._queries = value ?? []
|
const queries = value ?? []
|
||||||
for (const element of this._queries) {
|
for (const element of queries) {
|
||||||
this.rootSubscriptions.push(
|
this.rootSubscriptions.push(
|
||||||
element.changed.subscribe(() => {
|
element.changed.subscribe(() => {
|
||||||
this.changed.next(this)
|
this.changed.next(this)
|
||||||
})
|
})
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
this._queries.set(queries)
|
||||||
}
|
}
|
||||||
|
|
||||||
public clear(fireEvent = true) {
|
public clear(fireEvent = true) {
|
||||||
@@ -209,6 +211,7 @@ export class CustomFieldQueriesModel {
|
|||||||
DocumentLinkComponent,
|
DocumentLinkComponent,
|
||||||
ReactiveFormsModule,
|
ReactiveFormsModule,
|
||||||
NgbDatepickerModule,
|
NgbDatepickerModule,
|
||||||
|
NgClass,
|
||||||
NgTemplateOutlet,
|
NgTemplateOutlet,
|
||||||
NgSelectModule,
|
NgSelectModule,
|
||||||
NgxBootstrapIconsModule,
|
NgxBootstrapIconsModule,
|
||||||
|
|||||||
@@ -116,7 +116,7 @@
|
|||||||
|
|
||||||
</pngx-page-header>
|
</pngx-page-header>
|
||||||
|
|
||||||
<div class="row sticky-top py-3 mt-n2 mt-md-n3 bg-body">
|
<div class="row sticky-top py-3 mt-n2 mt-md-n3 bg-body rounded shadow-sm">
|
||||||
<pngx-filter-editor [hidden]="isBulkEditing" [disabled]="isBulkEditing" [filterRules]="list.filterRules" (filterRulesChange)="onFilterRulesChange($event)" (resetFilterRules)="onFilterRulesReset($event)" [unmodifiedFilterRules]="unmodifiedFilterRules()" [selectionData]="list.selectionData" #filterEditor></pngx-filter-editor>
|
<pngx-filter-editor [hidden]="isBulkEditing" [disabled]="isBulkEditing" [filterRules]="list.filterRules" (filterRulesChange)="onFilterRulesChange($event)" (resetFilterRules)="onFilterRulesReset($event)" [unmodifiedFilterRules]="unmodifiedFilterRules()" [selectionData]="list.selectionData" #filterEditor></pngx-filter-editor>
|
||||||
<pngx-bulk-editor [hidden]="!isBulkEditing" [disabled]="!isBulkEditing"></pngx-bulk-editor>
|
<pngx-bulk-editor [hidden]="!isBulkEditing" [disabled]="!isBulkEditing"></pngx-bulk-editor>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -1034,6 +1034,49 @@ describe('FilterEditorComponent', () => {
|
|||||||
).toEqual([42, CustomFieldQueryOperator.Exists, 'true'])
|
).toEqual([42, CustomFieldQueryOperator.Exists, 'true'])
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it('should reflect ingested custom field query rules in the dropdown toggle', () => {
|
||||||
|
const dropdown = fixture.debugElement.query(
|
||||||
|
By.css('pngx-custom-fields-query-dropdown')
|
||||||
|
)
|
||||||
|
expect(
|
||||||
|
dropdown.nativeElement.querySelector('pngx-clearable-badge')
|
||||||
|
).toBeNull()
|
||||||
|
|
||||||
|
// switching to a view with a custom field query
|
||||||
|
component.filterRules = [
|
||||||
|
{
|
||||||
|
rule_type: FILTER_CUSTOM_FIELDS_QUERY,
|
||||||
|
value: '["OR",[[42,"exists","true"]]]',
|
||||||
|
},
|
||||||
|
]
|
||||||
|
fixture.detectChanges()
|
||||||
|
expect(
|
||||||
|
dropdown.nativeElement.querySelector('pngx-clearable-badge')
|
||||||
|
).not.toBeNull()
|
||||||
|
expect(
|
||||||
|
dropdown.nativeElement
|
||||||
|
.querySelector('#dropdown_toggle')
|
||||||
|
.classList.contains('btn-primary')
|
||||||
|
).toBeTruthy()
|
||||||
|
|
||||||
|
// and back to a view without one
|
||||||
|
component.filterRules = [
|
||||||
|
{
|
||||||
|
rule_type: FILTER_HAS_TAGS_ALL,
|
||||||
|
value: '19',
|
||||||
|
},
|
||||||
|
]
|
||||||
|
fixture.detectChanges()
|
||||||
|
expect(
|
||||||
|
dropdown.nativeElement.querySelector('pngx-clearable-badge')
|
||||||
|
).toBeNull()
|
||||||
|
expect(
|
||||||
|
dropdown.nativeElement
|
||||||
|
.querySelector('#dropdown_toggle')
|
||||||
|
.classList.contains('btn-primary')
|
||||||
|
).toBeFalsy()
|
||||||
|
})
|
||||||
|
|
||||||
it('should ingest filter rules for owner', () => {
|
it('should ingest filter rules for owner', () => {
|
||||||
expect(component.permissionsSelectionModel.ownerFilter).toEqual(
|
expect(component.permissionsSelectionModel.ownerFilter).toEqual(
|
||||||
OwnerFilterType.NONE
|
OwnerFilterType.NONE
|
||||||
|
|||||||
+5
-16
@@ -47,6 +47,8 @@ $grid-breakpoints: (
|
|||||||
);
|
);
|
||||||
|
|
||||||
:root {
|
:root {
|
||||||
|
--bs-border-radius: #{$border-radius};
|
||||||
|
|
||||||
@each $name, $value in $grid-breakpoints {
|
@each $name, $value in $grid-breakpoints {
|
||||||
--bs-breakpoint-#{$name}: #{$value};
|
--bs-breakpoint-#{$name}: #{$value};
|
||||||
}
|
}
|
||||||
@@ -78,19 +80,12 @@ body {
|
|||||||
}
|
}
|
||||||
|
|
||||||
.btn {
|
.btn {
|
||||||
--bs-btn-border-radius: .425rem;
|
--bs-border-radius-sm: #{$border-radius};
|
||||||
--bs-border-radius-sm: .425rem;
|
|
||||||
font-weight: 500;
|
font-weight: 500;
|
||||||
}
|
}
|
||||||
|
|
||||||
.form-control,
|
|
||||||
.form-select,
|
|
||||||
.input-group-text {
|
|
||||||
border-radius: .425rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.pagination, .input-group {
|
.pagination, .input-group {
|
||||||
--bs-border-radius-sm: .425rem;
|
--bs-border-radius-sm: #{$border-radius};
|
||||||
}
|
}
|
||||||
|
|
||||||
@media(min-width: 768px) {
|
@media(min-width: 768px) {
|
||||||
@@ -689,10 +684,6 @@ table.table {
|
|||||||
--bs-toast-max-width: var(--pngx-toast-max-width);
|
--bs-toast-max-width: var(--pngx-toast-max-width);
|
||||||
}
|
}
|
||||||
|
|
||||||
.alert {
|
|
||||||
--bs-border-radius: .425rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.alert-primary {
|
.alert-primary {
|
||||||
--bs-alert-color: var(--bs-primary);
|
--bs-alert-color: var(--bs-primary);
|
||||||
--bs-alert-bg: var(--pngx-primary-faded);
|
--bs-alert-bg: var(--pngx-primary-faded);
|
||||||
@@ -824,8 +815,6 @@ code {
|
|||||||
--bs-accordion-bg: var(--bs-light);
|
--bs-accordion-bg: var(--bs-light);
|
||||||
--bs-accordion-active-color: var(--bs-primary);
|
--bs-accordion-active-color: var(--bs-primary);
|
||||||
--bs-accordion-active-bg: var(--pngx-bg-alt);
|
--bs-accordion-active-bg: var(--pngx-bg-alt);
|
||||||
--bs-border-radius: .425rem;
|
|
||||||
--bs-accordion-inner-border-radius: calc(.425rem - 1px);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
.accordion-button::after {
|
.accordion-button::after {
|
||||||
@@ -849,7 +838,7 @@ code {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/* Animate items as they're being sorted. */
|
/* Animate items as they're being sorted. */
|
||||||
.cdk-drop-list-dragging .cdk-drag {
|
.cdk-drop-list-dragging .cdk-drag:not(.cdk-drag-preview) {
|
||||||
transition: transform 250ms cubic-bezier(0, 0, 0.2, 1);
|
transition: transform 250ms cubic-bezier(0, 0, 0.2, 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -113,6 +113,7 @@ $form-check-radio-checked-bg-image-dark: url("data:image/svg+xml,%3csvg xmlns='h
|
|||||||
--bs-tertiary-bg: var(--pngx-bg-darker);
|
--bs-tertiary-bg: var(--pngx-bg-darker);
|
||||||
--bs-dark-border-subtle: var(--pngx-bg-darker);
|
--bs-dark-border-subtle: var(--pngx-bg-darker);
|
||||||
--bs-border-color-translucent: rgba(0, 0, 0, .175); // override bs
|
--bs-border-color-translucent: rgba(0, 0, 0, .175); // override bs
|
||||||
|
--bs-box-shadow-sm: 0 0.125rem 0.25rem rgba(0, 0, 0, 0.15); // slightly darker than bs default
|
||||||
|
|
||||||
.text-dark, .text-light {
|
.text-dark, .text-light {
|
||||||
color: var(--bs-body-color) !important;
|
color: var(--bs-body-color) !important;
|
||||||
|
|||||||
+10
-10
@@ -507,8 +507,8 @@ def rotate(
|
|||||||
logger.info(
|
logger.info(
|
||||||
f"Queued new rotated version for document {pair.root_doc.id} by {degrees} degrees",
|
f"Queued new rotated version for document {pair.root_doc.id} by {degrees} degrees",
|
||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception(f"Error rotating document {pair.root_doc.id}: {e}")
|
logger.exception(f"Error rotating document {pair.root_doc.id}")
|
||||||
|
|
||||||
return "OK"
|
return "OK"
|
||||||
|
|
||||||
@@ -554,9 +554,9 @@ def merge(
|
|||||||
affected_docs.append(doc.id)
|
affected_docs.append(doc.id)
|
||||||
if handoff_asn is None and doc.archive_serial_number is not None:
|
if handoff_asn is None and doc.archive_serial_number is not None:
|
||||||
handoff_asn = doc.archive_serial_number
|
handoff_asn = doc.archive_serial_number
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception(
|
logger.exception(
|
||||||
f"Error merging document {doc.id}, it will not be included in the merge: {e}",
|
f"Error merging document {doc.id}, it will not be included in the merge",
|
||||||
)
|
)
|
||||||
if len(affected_docs) == 0:
|
if len(affected_docs) == 0:
|
||||||
logger.warning("No documents were merged")
|
logger.warning("No documents were merged")
|
||||||
@@ -805,8 +805,8 @@ def split(
|
|||||||
else:
|
else:
|
||||||
group(consume_tasks).delay()
|
group(consume_tasks).delay()
|
||||||
|
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception(f"Error splitting document {doc.id}: {e}")
|
logger.exception(f"Error splitting document {doc.id}")
|
||||||
|
|
||||||
return "OK"
|
return "OK"
|
||||||
|
|
||||||
@@ -858,8 +858,8 @@ def delete_pages(
|
|||||||
logger.info(
|
logger.info(
|
||||||
f"Queued new version for document {pair.root_doc.id} after deleting pages {pages}",
|
f"Queued new version for document {pair.root_doc.id} after deleting pages {pages}",
|
||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception(f"Error deleting pages from document {pair.root_doc.id}: {e}")
|
logger.exception(f"Error deleting pages from document {pair.root_doc.id}")
|
||||||
|
|
||||||
return "OK"
|
return "OK"
|
||||||
|
|
||||||
@@ -986,7 +986,7 @@ def edit_pdf(
|
|||||||
group(consume_tasks).delay()
|
group(consume_tasks).delay()
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.exception(f"Error editing document {pair.root_doc.id}: {e}")
|
logger.exception(f"Error editing document {pair.root_doc.id}")
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
f"An error occurred while editing the document: {e}",
|
f"An error occurred while editing the document: {e}",
|
||||||
) from e
|
) from e
|
||||||
@@ -1097,7 +1097,7 @@ def remove_password(
|
|||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.exception(
|
logger.exception(
|
||||||
f"Error removing password from document {pair.root_doc.id}: {e}",
|
f"Error removing password from document {pair.root_doc.id}",
|
||||||
)
|
)
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
f"An error occurred while removing the password: {e}",
|
f"An error occurred while removing the password: {e}",
|
||||||
|
|||||||
@@ -16,6 +16,9 @@ from django.core.cache import cache
|
|||||||
from django.core.cache import caches
|
from django.core.cache import caches
|
||||||
|
|
||||||
from documents.models import Document
|
from documents.models import Document
|
||||||
|
from paperless.signed_pickle import SignedPickleError
|
||||||
|
from paperless.signed_pickle import signed_pickle_dumps
|
||||||
|
from paperless.signed_pickle import signed_pickle_loads
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from django.core.cache.backends.base import BaseCache
|
from django.core.cache.backends.base import BaseCache
|
||||||
@@ -118,9 +121,11 @@ class StoredLRUCache(LRUCache):
|
|||||||
serialized_data = self._backend.get(self._backend_key)
|
serialized_data = self._backend.get(self._backend_key)
|
||||||
try:
|
try:
|
||||||
self._data = (
|
self._data = (
|
||||||
pickle.loads(serialized_data) if serialized_data else OrderedDict()
|
signed_pickle_loads(serialized_data)
|
||||||
|
if serialized_data
|
||||||
|
else OrderedDict()
|
||||||
)
|
)
|
||||||
except pickle.PickleError:
|
except (SignedPickleError, pickle.PickleError):
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"Cache exists in backend but could not be read (possibly invalid format)",
|
"Cache exists in backend but could not be read (possibly invalid format)",
|
||||||
)
|
)
|
||||||
@@ -132,7 +137,7 @@ class StoredLRUCache(LRUCache):
|
|||||||
"""
|
"""
|
||||||
self._backend.set(
|
self._backend.set(
|
||||||
self._backend_key,
|
self._backend_key,
|
||||||
pickle.dumps(self._data),
|
signed_pickle_dumps(self._data),
|
||||||
self.backend_ttl,
|
self.backend_ttl,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
+20
-10
@@ -28,6 +28,9 @@ from documents.caching import CLASSIFIER_VERSION_KEY
|
|||||||
from documents.caching import StoredLRUCache
|
from documents.caching import StoredLRUCache
|
||||||
from documents.models import Document
|
from documents.models import Document
|
||||||
from documents.models import MatchingModel
|
from documents.models import MatchingModel
|
||||||
|
from paperless.signed_pickle import SignedPickleError
|
||||||
|
from paperless.signed_pickle import signed_pickle_dumps
|
||||||
|
from paperless.signed_pickle import signed_pickle_loads
|
||||||
|
|
||||||
logger = logging.getLogger("paperless.classifier")
|
logger = logging.getLogger("paperless.classifier")
|
||||||
|
|
||||||
@@ -69,8 +72,8 @@ def load_classifier(*, raise_exception: bool = False) -> DocumentClassifier | No
|
|||||||
Path(settings.MODEL_FILE).unlink()
|
Path(settings.MODEL_FILE).unlink()
|
||||||
classifier = None
|
classifier = None
|
||||||
if raise_exception:
|
if raise_exception:
|
||||||
raise e
|
raise
|
||||||
except ClassifierModelCorruptError as e:
|
except ClassifierModelCorruptError:
|
||||||
# there's something wrong with the model file.
|
# there's something wrong with the model file.
|
||||||
logger.exception(
|
logger.exception(
|
||||||
"Unrecoverable error while loading document "
|
"Unrecoverable error while loading document "
|
||||||
@@ -79,17 +82,17 @@ def load_classifier(*, raise_exception: bool = False) -> DocumentClassifier | No
|
|||||||
Path(settings.MODEL_FILE).unlink()
|
Path(settings.MODEL_FILE).unlink()
|
||||||
classifier = None
|
classifier = None
|
||||||
if raise_exception:
|
if raise_exception:
|
||||||
raise e
|
raise
|
||||||
except OSError as e:
|
except OSError:
|
||||||
logger.exception("IO error while loading document classification model")
|
logger.exception("IO error while loading document classification model")
|
||||||
classifier = None
|
classifier = None
|
||||||
if raise_exception:
|
if raise_exception:
|
||||||
raise e
|
raise
|
||||||
except Exception as e: # pragma: no cover
|
except Exception: # pragma: no cover
|
||||||
logger.exception("Unknown error while loading document classification model")
|
logger.exception("Unknown error while loading document classification model")
|
||||||
classifier = None
|
classifier = None
|
||||||
if raise_exception:
|
if raise_exception:
|
||||||
raise e
|
raise
|
||||||
|
|
||||||
return classifier
|
return classifier
|
||||||
|
|
||||||
@@ -527,10 +530,17 @@ class DocumentClassifier:
|
|||||||
serialized_result = read_cache.get(key)
|
serialized_result = read_cache.get(key)
|
||||||
if serialized_result is None:
|
if serialized_result is None:
|
||||||
result = self.data_vectorizer.transform([self.preprocess_content(content)])
|
result = self.data_vectorizer.transform([self.preprocess_content(content)])
|
||||||
read_cache.set(key, pickle.dumps(result), CACHE_5_MINUTES)
|
read_cache.set(key, signed_pickle_dumps(result), CACHE_5_MINUTES)
|
||||||
else:
|
else:
|
||||||
read_cache.touch(key, CACHE_5_MINUTES)
|
try:
|
||||||
result = pickle.loads(serialized_result)
|
result = signed_pickle_loads(serialized_result)
|
||||||
|
except SignedPickleError:
|
||||||
|
result = self.data_vectorizer.transform(
|
||||||
|
[self.preprocess_content(content)],
|
||||||
|
)
|
||||||
|
read_cache.set(key, signed_pickle_dumps(result), CACHE_5_MINUTES)
|
||||||
|
else:
|
||||||
|
read_cache.touch(key, CACHE_5_MINUTES)
|
||||||
return result
|
return result
|
||||||
|
|
||||||
def predict_correspondent(self, content: str) -> int | None:
|
def predict_correspondent(self, content: str) -> int | None:
|
||||||
|
|||||||
@@ -217,7 +217,7 @@ class ConsumerPluginMixin:
|
|||||||
current_progress,
|
current_progress,
|
||||||
max_progress,
|
max_progress,
|
||||||
document_id=document_id,
|
document_id=document_id,
|
||||||
owner_id=self.metadata.owner_id if self.metadata.owner_id else None,
|
owner_id=self.metadata.owner_id or None,
|
||||||
users_can_view=(self.metadata.view_users or [])
|
users_can_view=(self.metadata.view_users or [])
|
||||||
+ (self.metadata.change_users or []),
|
+ (self.metadata.change_users or []),
|
||||||
groups_can_view=(self.metadata.view_groups or [])
|
groups_can_view=(self.metadata.view_groups or [])
|
||||||
@@ -675,9 +675,7 @@ class ConsumerPlugin(
|
|||||||
document=document,
|
document=document,
|
||||||
logging_group=self.logging_group,
|
logging_group=self.logging_group,
|
||||||
classifier=classifier,
|
classifier=classifier,
|
||||||
original_file=self.unmodified_original
|
original_file=self.unmodified_original or self.working_copy,
|
||||||
if self.unmodified_original
|
|
||||||
else self.working_copy,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
# After everything is in the database, copy the files into
|
# After everything is in the database, copy the files into
|
||||||
@@ -858,7 +856,7 @@ class ConsumerPlugin(
|
|||||||
else:
|
else:
|
||||||
stats = Path(self.input_doc.original_file).stat()
|
stats = Path(self.input_doc.original_file).stat()
|
||||||
create_date = timezone.make_aware(
|
create_date = timezone.make_aware(
|
||||||
datetime.datetime.fromtimestamp(stats.st_mtime),
|
datetime.datetime.fromtimestamp(stats.st_mtime), # noqa: DTZ006 - make_aware() requires a naive datetime
|
||||||
)
|
)
|
||||||
self.log.debug(f"Creation date from st_mtime: {create_date}")
|
self.log.debug(f"Creation date from st_mtime: {create_date}")
|
||||||
|
|
||||||
@@ -972,7 +970,7 @@ class ConsumerPlugin(
|
|||||||
try:
|
try:
|
||||||
copy_basic_file_stats(source, target)
|
copy_basic_file_stats(source, target)
|
||||||
except Exception: # pragma: no cover
|
except Exception: # pragma: no cover
|
||||||
pass
|
self.log.debug("Unable to copy file stats from %s to %s", source, target)
|
||||||
|
|
||||||
|
|
||||||
class ConsumerPreflightPlugin(
|
class ConsumerPreflightPlugin(
|
||||||
|
|||||||
@@ -78,7 +78,9 @@ class CollatePlugin(NoCleanupPluginMixin, NoSetupPluginMixin, ConsumeTaskPlugin)
|
|||||||
stats = staging.stat()
|
stats = staging.stat()
|
||||||
# if the file is older than the timeout, we don't consider
|
# if the file is older than the timeout, we don't consider
|
||||||
# it valid
|
# it valid
|
||||||
if (dt.datetime.now().timestamp() - stats.st_mtime) > TIMEOUT_SECONDS:
|
if (
|
||||||
|
dt.datetime.now(tz=dt.UTC).timestamp() - stats.st_mtime
|
||||||
|
) > TIMEOUT_SECONDS:
|
||||||
logger.warning("Outdated double sided staging file exists, deleting it")
|
logger.warning("Outdated double sided staging file exists, deleting it")
|
||||||
staging.unlink()
|
staging.unlink()
|
||||||
else:
|
else:
|
||||||
@@ -134,7 +136,7 @@ class CollatePlugin(NoCleanupPluginMixin, NoSetupPluginMixin, ConsumeTaskPlugin)
|
|||||||
shutil.move(pdf_file, staging)
|
shutil.move(pdf_file, staging)
|
||||||
# update access to modification time so we know if the file
|
# update access to modification time so we know if the file
|
||||||
# is outdated when another file gets uploaded
|
# is outdated when another file gets uploaded
|
||||||
timestamp = dt.datetime.now().timestamp()
|
timestamp = dt.datetime.now(tz=dt.UTC).timestamp()
|
||||||
os.utime(staging, (timestamp, timestamp))
|
os.utime(staging, (timestamp, timestamp))
|
||||||
logger.info(
|
logger.info(
|
||||||
"Got scan with odd numbered pages of double-sided scan, moved it to %s",
|
"Got scan with odd numbered pages of double-sided scan, moved it to %s",
|
||||||
|
|||||||
@@ -734,7 +734,7 @@ class CustomFieldQueryParser:
|
|||||||
)
|
)
|
||||||
|
|
||||||
# Check if any of the requested IDs are missing.
|
# Check if any of the requested IDs are missing.
|
||||||
missing_ids = set(value) - set(link.document_id for link in links)
|
missing_ids = set(value) - {link.document_id for link in links}
|
||||||
if missing_ids:
|
if missing_ids:
|
||||||
# The result should be an empty set in this case.
|
# The result should be an empty set in this case.
|
||||||
return Q(id__in=[])
|
return Q(id__in=[])
|
||||||
|
|||||||
@@ -631,23 +631,25 @@ class Command(BaseCommand):
|
|||||||
):
|
):
|
||||||
# Process each change
|
# Process each change
|
||||||
for change_type, path in changes:
|
for change_type, path in changes:
|
||||||
path = Path(path).resolve()
|
resolved_path = Path(path).resolve()
|
||||||
if change_type == Change.deleted:
|
if change_type == Change.deleted:
|
||||||
# Consumed (or otherwise removed); a later file
|
# Consumed (or otherwise removed); a later file
|
||||||
# reusing this name must not be skipped as
|
# reusing this name must not be skipped as
|
||||||
# already-queued.
|
# already-queued.
|
||||||
queued.discard(path)
|
queued.discard(resolved_path)
|
||||||
if not path.is_file():
|
if not resolved_path.is_file():
|
||||||
continue
|
continue
|
||||||
if path in queued:
|
if resolved_path in queued:
|
||||||
# Already queued and awaiting consumption; a stray
|
# Already queued and awaiting consumption; a stray
|
||||||
# event (NAS metadata touch, AV scan, etc.) while
|
# event (NAS metadata touch, AV scan, etc.) while
|
||||||
# the file sits on disk mid-consumption must not
|
# the file sits on disk mid-consumption must not
|
||||||
# cause it to be queued a second time (GH #13511).
|
# cause it to be queued a second time (GH #13511).
|
||||||
logger.debug(f"Ignoring event for queued file: {path}")
|
logger.debug(
|
||||||
|
f"Ignoring event for queued file: {resolved_path}",
|
||||||
|
)
|
||||||
continue
|
continue
|
||||||
logger.debug(f"Event: {change_type.name} for {path}")
|
logger.debug(f"Event: {change_type.name} for {resolved_path}")
|
||||||
tracker.track(path, change_type)
|
tracker.track(resolved_path, change_type)
|
||||||
|
|
||||||
# Check for stable files
|
# Check for stable files
|
||||||
for stable_path in tracker.get_stable_files():
|
for stable_path in tracker.get_stable_files():
|
||||||
|
|||||||
@@ -30,6 +30,10 @@ if TYPE_CHECKING:
|
|||||||
logger = logging.getLogger("paperless.matching")
|
logger = logging.getLogger("paperless.matching")
|
||||||
|
|
||||||
|
|
||||||
|
class UnsupportedWorkflowTriggerTypeError(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
def log_reason(
|
def log_reason(
|
||||||
matching_model: MatchingModel | WorkflowTrigger,
|
matching_model: MatchingModel | WorkflowTrigger,
|
||||||
document: Document,
|
document: Document,
|
||||||
@@ -691,7 +695,9 @@ def document_matches_workflow(
|
|||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
# New trigger types need to be explicitly checked above
|
# New trigger types need to be explicitly checked above
|
||||||
raise Exception(f"Trigger type {trigger_type} not yet supported")
|
raise UnsupportedWorkflowTriggerTypeError(
|
||||||
|
f"Trigger type {trigger_type} not yet supported",
|
||||||
|
)
|
||||||
|
|
||||||
if trigger_matched:
|
if trigger_matched:
|
||||||
logger.info(f"Document matched {trigger} from {workflow}")
|
logger.info(f"Document matched {trigger} from {workflow}")
|
||||||
|
|||||||
@@ -75,7 +75,7 @@ def recompute_checksums(apps, schema_editor):
|
|||||||
if updated_fields:
|
if updated_fields:
|
||||||
batch.append(doc)
|
batch.append(doc)
|
||||||
|
|
||||||
processed += 1
|
processed += 1 # noqa: SIM113
|
||||||
|
|
||||||
if len(batch) >= _BATCH_SIZE:
|
if len(batch) >= _BATCH_SIZE:
|
||||||
Document.objects.bulk_update(batch, ["checksum", "archive_checksum"])
|
Document.objects.bulk_update(batch, ["checksum", "archive_checksum"])
|
||||||
|
|||||||
@@ -377,7 +377,7 @@ class Document(SoftDeleteModel, ModelWithOwner): # type: ignore[django-manager-
|
|||||||
from documents.versioning import versions_newest_first
|
from documents.versioning import versions_newest_first
|
||||||
|
|
||||||
if hasattr(self, "effective_content"):
|
if hasattr(self, "effective_content"):
|
||||||
return getattr(self, "effective_content")
|
return self.effective_content
|
||||||
|
|
||||||
if self.root_document_id is not None or self.pk is None:
|
if self.root_document_id is not None or self.pk is None:
|
||||||
return self.content
|
return self.content
|
||||||
@@ -462,7 +462,11 @@ class Document(SoftDeleteModel, ModelWithOwner): # type: ignore[django-manager-
|
|||||||
"""
|
"""
|
||||||
Returns a sanitized filename for the document, not including any paths.
|
Returns a sanitized filename for the document, not including any paths.
|
||||||
"""
|
"""
|
||||||
result = str(self)
|
# Root owns metadata for all versions
|
||||||
|
context_document = (
|
||||||
|
self.root_document if self.root_document_id is not None else self
|
||||||
|
)
|
||||||
|
result = str(context_document)
|
||||||
|
|
||||||
if counter:
|
if counter:
|
||||||
result += f"_{counter:02}"
|
result += f"_{counter:02}"
|
||||||
|
|||||||
@@ -41,7 +41,7 @@ def get_default_file_extension(mime_type: str) -> str:
|
|||||||
return supported[mime_type]
|
return supported[mime_type]
|
||||||
|
|
||||||
ext = mimetypes.guess_extension(mime_type)
|
ext = mimetypes.guess_extension(mime_type)
|
||||||
return ext if ext else ""
|
return ext or ""
|
||||||
|
|
||||||
|
|
||||||
def is_file_ext_supported(ext: str) -> bool:
|
def is_file_ext_supported(ext: str) -> bool:
|
||||||
@@ -110,7 +110,7 @@ def run_convert(
|
|||||||
args += ["-define", "pdf:use-cropbox=true"] if use_cropbox else []
|
args += ["-define", "pdf:use-cropbox=true"] if use_cropbox else []
|
||||||
args += [str(input_file), str(output_file)]
|
args += [str(input_file), str(output_file)]
|
||||||
|
|
||||||
logger.debug("Execute: " + " ".join(args), extra={"group": logging_group})
|
logger.debug("Execute: %s", " ".join(args), extra={"group": logging_group})
|
||||||
|
|
||||||
try:
|
try:
|
||||||
run_subprocess(args, environment, logger)
|
run_subprocess(args, environment, logger)
|
||||||
|
|||||||
@@ -43,8 +43,8 @@ def _discover_parser_class() -> type[DateParserPluginBase]:
|
|||||||
valid_plugins.append(ep)
|
valid_plugins.append(ep)
|
||||||
else:
|
else:
|
||||||
logger.warning(f"Plugin {ep.name} does not subclass DateParser.")
|
logger.warning(f"Plugin {ep.name} does not subclass DateParser.")
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception(f"Unable to load date parser plugin {ep.name}: {e}")
|
logger.exception(f"Unable to load date parser plugin {ep.name}")
|
||||||
|
|
||||||
if not valid_plugins:
|
if not valid_plugins:
|
||||||
return RegexDateParserPlugin
|
return RegexDateParserPlugin
|
||||||
|
|||||||
@@ -91,8 +91,8 @@ class DateParserPluginBase(ABC):
|
|||||||
},
|
},
|
||||||
locales=self.config.languages,
|
locales=self.config.languages,
|
||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception(f"Error while parsing date string '{date_string}': {e}")
|
logger.exception(f"Error while parsing date string '{date_string}'")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def _filter_date(
|
def _filter_date(
|
||||||
|
|||||||
@@ -59,11 +59,10 @@ def safe_regex_match(pattern: str, text: str, *, flags: int = 0):
|
|||||||
try:
|
try:
|
||||||
validate_regex_pattern(pattern)
|
validate_regex_pattern(pattern)
|
||||||
compiled = regex.compile(pattern, flags=flags)
|
compiled = regex.compile(pattern, flags=flags)
|
||||||
except (regex.error, ValueError) as exc:
|
except (regex.error, ValueError):
|
||||||
logger.exception(
|
logger.exception(
|
||||||
"Error while processing regular expression %s: %s",
|
"Error while processing regular expression %s",
|
||||||
textwrap.shorten(pattern, width=80, placeholder="…"),
|
textwrap.shorten(pattern, width=80, placeholder="…"),
|
||||||
exc,
|
|
||||||
)
|
)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
@@ -86,11 +85,10 @@ def safe_regex_sub(pattern: str, repl: str, text: str, *, flags: int = 0) -> str
|
|||||||
try:
|
try:
|
||||||
validate_regex_pattern(pattern)
|
validate_regex_pattern(pattern)
|
||||||
compiled = regex.compile(pattern, flags=flags)
|
compiled = regex.compile(pattern, flags=flags)
|
||||||
except (regex.error, ValueError) as exc:
|
except (regex.error, ValueError):
|
||||||
logger.exception(
|
logger.exception(
|
||||||
"Error while processing regular expression %s: %s",
|
"Error while processing regular expression %s",
|
||||||
textwrap.shorten(pattern, width=80, placeholder="…"),
|
textwrap.shorten(pattern, width=80, placeholder="…"),
|
||||||
exc,
|
|
||||||
)
|
)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|||||||
@@ -1142,7 +1142,7 @@ def get_backend() -> TantivyBackend:
|
|||||||
Returns:
|
Returns:
|
||||||
Thread-safe singleton TantivyBackend instance
|
Thread-safe singleton TantivyBackend instance
|
||||||
"""
|
"""
|
||||||
global _backend, _backend_path
|
global _backend, _backend_path # noqa: PLW0603 - module-level singleton, no class to hold this state
|
||||||
|
|
||||||
current_path: Path = settings.INDEX_DIR
|
current_path: Path = settings.INDEX_DIR
|
||||||
|
|
||||||
@@ -1173,7 +1173,7 @@ def reset_backend() -> None:
|
|||||||
Forces creation of a new backend instance on the next get_backend() call.
|
Forces creation of a new backend instance on the next get_backend() call.
|
||||||
Used for test isolation and when switching between different index directories.
|
Used for test isolation and when switching between different index directories.
|
||||||
"""
|
"""
|
||||||
global _backend, _backend_path
|
global _backend, _backend_path # noqa: PLW0603 - module-level singleton, no class to hold this state
|
||||||
|
|
||||||
with _backend_lock:
|
with _backend_lock:
|
||||||
if _backend is not None:
|
if _backend is not None:
|
||||||
|
|||||||
@@ -240,7 +240,7 @@ def parse_user_query(
|
|||||||
DEFAULT_SEARCH_FIELDS,
|
DEFAULT_SEARCH_FIELDS,
|
||||||
field_boosts=_FIELD_BOOSTS,
|
field_boosts=_FIELD_BOOSTS,
|
||||||
# (prefix=True, distance=1, transposition_cost_one=True) — edit-distance fuzziness
|
# (prefix=True, distance=1, transposition_cost_one=True) — edit-distance fuzziness
|
||||||
fuzzy_fields={f: (True, 1, True) for f in DEFAULT_SEARCH_FIELDS},
|
fuzzy_fields=dict.fromkeys(DEFAULT_SEARCH_FIELDS, (True, 1, True)),
|
||||||
)
|
)
|
||||||
# 0.1 boost keeps fuzzy hits ranked below exact matches (intentional)
|
# 0.1 boost keeps fuzzy hits ranked below exact matches (intentional)
|
||||||
clauses.append((tantivy.Occur.Should, tantivy.Query.boost_query(fuzzy, 0.1)))
|
clauses.append((tantivy.Occur.Should, tantivy.Query.boost_query(fuzzy, 0.1)))
|
||||||
|
|||||||
@@ -434,7 +434,7 @@ class OwnedObjectSerializer(
|
|||||||
return set()
|
return set()
|
||||||
|
|
||||||
ctype = ContentType.objects.get_for_model(first_obj)
|
ctype = ContentType.objects.get_for_model(first_obj)
|
||||||
object_pks = list(obj.pk for obj in objects)
|
object_pks = [obj.pk for obj in objects]
|
||||||
pk_type = type(first_obj.pk)
|
pk_type = type(first_obj.pk)
|
||||||
|
|
||||||
def get_pks_for_permission_type(model):
|
def get_pks_for_permission_type(model):
|
||||||
@@ -730,7 +730,7 @@ class TagSerializer(MatchingModelSerializer, OwnedObjectSerializer):
|
|||||||
self.instance.clean()
|
self.instance.clean()
|
||||||
except ValidationError as e:
|
except ValidationError as e:
|
||||||
logger.debug("Tag parent validation failed: %s", e)
|
logger.debug("Tag parent validation failed: %s", e)
|
||||||
raise e
|
raise
|
||||||
finally:
|
finally:
|
||||||
self.instance.tn_parent = original_parent
|
self.instance.tn_parent = original_parent
|
||||||
else:
|
else:
|
||||||
@@ -740,7 +740,7 @@ class TagSerializer(MatchingModelSerializer, OwnedObjectSerializer):
|
|||||||
temp.clean()
|
temp.clean()
|
||||||
except ValidationError as e:
|
except ValidationError as e:
|
||||||
logger.debug("Tag parent validation failed: %s", e)
|
logger.debug("Tag parent validation failed: %s", e)
|
||||||
raise e
|
raise
|
||||||
|
|
||||||
return super().validate(attrs)
|
return super().validate(attrs)
|
||||||
|
|
||||||
@@ -1150,7 +1150,7 @@ class DocumentSerializer(
|
|||||||
def to_representation(self, instance):
|
def to_representation(self, instance):
|
||||||
doc = super().to_representation(instance)
|
doc = super().to_representation(instance)
|
||||||
if "content" in self.fields and hasattr(instance, "effective_content"):
|
if "content" in self.fields and hasattr(instance, "effective_content"):
|
||||||
doc["content"] = getattr(instance, "effective_content") or ""
|
doc["content"] = instance.effective_content or ""
|
||||||
if self.truncate_content and "content" in self.fields:
|
if self.truncate_content and "content" in self.fields:
|
||||||
doc["content"] = doc.get("content")[0:550]
|
doc["content"] = doc.get("content")[0:550]
|
||||||
return doc
|
return doc
|
||||||
@@ -1860,8 +1860,8 @@ class BulkEditSerializer(
|
|||||||
if isinstance(custom_fields, dict):
|
if isinstance(custom_fields, dict):
|
||||||
try:
|
try:
|
||||||
ids = [int(i[0]) for i in custom_fields.items()]
|
ids = [int(i[0]) for i in custom_fields.items()]
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception(f"Error validating custom fields: {e}")
|
logger.exception("Error validating custom fields")
|
||||||
raise serializers.ValidationError(
|
raise serializers.ValidationError(
|
||||||
f"{name} must be a list of integers or a dict of id:value pairs, see the log for details",
|
f"{name} must be a list of integers or a dict of id:value pairs, see the log for details",
|
||||||
)
|
)
|
||||||
@@ -2059,13 +2059,12 @@ class BulkEditSerializer(
|
|||||||
for doc in docs:
|
for doc in docs:
|
||||||
if "-" in doc:
|
if "-" in doc:
|
||||||
pages.append(
|
pages.append(
|
||||||
[
|
list(
|
||||||
x
|
range(
|
||||||
for x in range(
|
|
||||||
int(doc.split("-")[0]),
|
int(doc.split("-")[0]),
|
||||||
int(doc.split("-")[1]) + 1,
|
int(doc.split("-")[1]) + 1,
|
||||||
)
|
),
|
||||||
],
|
),
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
pages.append([int(doc)])
|
pages.append([int(doc)])
|
||||||
@@ -2926,7 +2925,7 @@ class ShareLinkBundleSerializer(OwnedObjectSerializer):
|
|||||||
return share_link_bundle
|
return share_link_bundle
|
||||||
|
|
||||||
def get_document_count(self, obj: ShareLinkBundle) -> int:
|
def get_document_count(self, obj: ShareLinkBundle) -> int:
|
||||||
return getattr(obj, "document_total") or obj.documents.count()
|
return obj.document_total or obj.documents.count()
|
||||||
|
|
||||||
|
|
||||||
class BulkEditObjectsSerializer(SerializerWithPerms, SetPermissionsMixin):
|
class BulkEditObjectsSerializer(SerializerWithPerms, SetPermissionsMixin):
|
||||||
|
|||||||
@@ -637,7 +637,7 @@ def update_filename_and_move_files(
|
|||||||
# so this is not the end of the world.
|
# so this is not the end of the world.
|
||||||
# B: if moving the original file failed, nothing has changed
|
# B: if moving the original file failed, nothing has changed
|
||||||
# anyway.
|
# anyway.
|
||||||
pass
|
logger.exception("Error reverting document changes")
|
||||||
|
|
||||||
# restore old values on the instance
|
# restore old values on the instance
|
||||||
instance.filename = old_filename
|
instance.filename = old_filename
|
||||||
@@ -1102,10 +1102,11 @@ def _extract_input_data(
|
|||||||
if v is None or k.startswith("_"):
|
if v is None or k.startswith("_"):
|
||||||
continue
|
continue
|
||||||
if isinstance(v, datetime.date):
|
if isinstance(v, datetime.date):
|
||||||
v = v.isoformat()
|
override_dict[k] = v.isoformat()
|
||||||
elif isinstance(v, Path):
|
elif isinstance(v, Path):
|
||||||
v = str(v)
|
override_dict[k] = str(v)
|
||||||
override_dict[k] = v
|
else:
|
||||||
|
override_dict[k] = v
|
||||||
if override_dict:
|
if override_dict:
|
||||||
data["overrides"] = override_dict
|
data["overrides"] = override_dict
|
||||||
return data
|
return data
|
||||||
|
|||||||
@@ -217,9 +217,9 @@ def consume_file(
|
|||||||
overrides.filename or input_doc.original_file.name,
|
overrides.filename or input_doc.original_file.name,
|
||||||
self.request.id,
|
self.request.id,
|
||||||
) as status_mgr,
|
) as status_mgr,
|
||||||
TemporaryDirectory(dir=settings.SCRATCH_DIR) as tmp_dir,
|
TemporaryDirectory(dir=settings.SCRATCH_DIR) as tmp_dir_name,
|
||||||
):
|
):
|
||||||
tmp_dir = Path(tmp_dir)
|
tmp_dir = Path(tmp_dir_name)
|
||||||
msg = None
|
msg = None
|
||||||
for plugin_class in plugins:
|
for plugin_class in plugins:
|
||||||
plugin_name = plugin_class.NAME
|
plugin_name = plugin_class.NAME
|
||||||
@@ -261,7 +261,7 @@ def consume_file(
|
|||||||
)
|
)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.exception(f"{plugin_name} failed: {e}")
|
logger.exception(f"{plugin_name} failed")
|
||||||
status_mgr.send_progress(
|
status_mgr.send_progress(
|
||||||
ProgressStatusOptions.FAILED,
|
ProgressStatusOptions.FAILED,
|
||||||
f"{e}",
|
f"{e}",
|
||||||
@@ -495,8 +495,8 @@ def empty_trash(doc_ids=None) -> None:
|
|||||||
content_type=ContentType.objects.get_for_model(Document),
|
content_type=ContentType.objects.get_for_model(Document),
|
||||||
object_id__in=deleted_document_ids,
|
object_id__in=deleted_document_ids,
|
||||||
).delete()
|
).delete()
|
||||||
except Exception as e: # pragma: no cover
|
except Exception: # pragma: no cover
|
||||||
logger.exception(f"Error while emptying trash: {e}")
|
logger.exception("Error while emptying trash")
|
||||||
finally:
|
finally:
|
||||||
models.signals.post_delete.disconnect(
|
models.signals.post_delete.disconnect(
|
||||||
cleanup_document_deletion,
|
cleanup_document_deletion,
|
||||||
@@ -832,9 +832,8 @@ def build_share_link_bundle(bundle_id: int) -> None:
|
|||||||
logger.info("Built share link bundle %s", bundle.pk)
|
logger.info("Built share link bundle %s", bundle.pk)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
logger.exception(
|
logger.exception(
|
||||||
"Failed to build share link bundle %s: %s",
|
"Failed to build share link bundle %s",
|
||||||
bundle_id,
|
bundle_id,
|
||||||
exc,
|
|
||||||
)
|
)
|
||||||
bundle.status = ShareLinkBundle.Status.FAILED
|
bundle.status = ShareLinkBundle.Status.FAILED
|
||||||
bundle.last_error = {
|
bundle.last_error = {
|
||||||
|
|||||||
@@ -78,6 +78,10 @@ class PlaceholderString(str):
|
|||||||
def __ne__(self, other) -> bool:
|
def __ne__(self, other) -> bool:
|
||||||
return not self.__eq__(other)
|
return not self.__eq__(other)
|
||||||
|
|
||||||
|
def __hash__(self) -> int:
|
||||||
|
# Equal to both "-none-" and "none", so hash to a single canonical value
|
||||||
|
return hash("-none-")
|
||||||
|
|
||||||
|
|
||||||
NO_VALUE_PLACEHOLDER = PlaceholderString("-none-")
|
NO_VALUE_PLACEHOLDER = PlaceholderString("-none-")
|
||||||
|
|
||||||
|
|||||||
@@ -138,9 +138,9 @@ def parse_w_workflow_placeholders(
|
|||||||
|
|
||||||
# We're good!
|
# We're good!
|
||||||
return rendered_template
|
return rendered_template
|
||||||
except UndefinedError as e:
|
except UndefinedError:
|
||||||
# The undefined class logs this already for us
|
# The undefined class logs this already for us
|
||||||
raise e
|
raise
|
||||||
except TemplateSyntaxError as e:
|
except TemplateSyntaxError as e:
|
||||||
logger.warning(f"Template syntax error in title generation: {e}")
|
logger.warning(f"Template syntax error in title generation: {e}")
|
||||||
except SecurityError as e:
|
except SecurityError as e:
|
||||||
@@ -150,5 +150,5 @@ def parse_w_workflow_placeholders(
|
|||||||
logger.warning(
|
logger.warning(
|
||||||
f"Invalid title format '{text}', workflow not applied: {e}",
|
f"Invalid title format '{text}', workflow not applied: {e}",
|
||||||
)
|
)
|
||||||
raise e
|
raise
|
||||||
return None
|
return None
|
||||||
|
|||||||
@@ -296,7 +296,7 @@ class TestRegexDateParser:
|
|||||||
|
|
||||||
# simulate parse failure for malformed input
|
# simulate parse failure for malformed input
|
||||||
if "99/99/9999" in date_string or "bad date" in date_string:
|
if "99/99/9999" in date_string or "bad date" in date_string:
|
||||||
raise Exception("parse failed for malformed date")
|
raise Exception("parse failed for malformed date") # noqa: TRY002 - simulates a generic parser failure
|
||||||
|
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|||||||
@@ -57,13 +57,13 @@ class MultiprocessCommand(PaperlessCommand):
|
|||||||
|
|
||||||
def handle(self, *args, **options):
|
def handle(self, *args, **options):
|
||||||
items = list(range(5))
|
items = list(range(5))
|
||||||
results = []
|
results = list(
|
||||||
for result in self.process_parallel(
|
self.process_parallel(
|
||||||
_double_value,
|
_double_value,
|
||||||
items,
|
items,
|
||||||
description="Processing...",
|
description="Processing...",
|
||||||
):
|
),
|
||||||
results.append(result)
|
)
|
||||||
successes = sum(1 for r in results if r.success)
|
successes = sum(1 for r in results if r.success)
|
||||||
self.stdout.write(f"Successes: {successes}")
|
self.stdout.write(f"Successes: {successes}")
|
||||||
|
|
||||||
|
|||||||
@@ -66,7 +66,7 @@ class TestWriteBatchLockRetry:
|
|||||||
)
|
)
|
||||||
mock_sleep = mocker.patch(
|
mock_sleep = mocker.patch(
|
||||||
"documents.search._backend.time.sleep",
|
"documents.search._backend.time.sleep",
|
||||||
side_effect=lambda s: sleep_values.append(s),
|
side_effect=sleep_values.append,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Should not raise — 4th attempt succeeds
|
# Should not raise — 4th attempt succeeds
|
||||||
@@ -111,7 +111,7 @@ class TestWriteBatchLockRetry:
|
|||||||
sleep_values: list[float] = []
|
sleep_values: list[float] = []
|
||||||
mocker.patch(
|
mocker.patch(
|
||||||
"documents.search._backend.time.sleep",
|
"documents.search._backend.time.sleep",
|
||||||
side_effect=lambda s: sleep_values.append(s),
|
side_effect=sleep_values.append,
|
||||||
)
|
)
|
||||||
for _ in range(50):
|
for _ in range(50):
|
||||||
sleep_values.clear()
|
sleep_values.clear()
|
||||||
|
|||||||
@@ -1063,3 +1063,79 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
|
|||||||
)
|
)
|
||||||
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
|
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
|
||||||
self.assertIn("non-public address", str(response.data).lower())
|
self.assertIn("non-public address", str(response.data).lower())
|
||||||
|
|
||||||
|
@override_settings(REMOTE_OCR_ALLOW_INTERNAL_ENDPOINTS=False)
|
||||||
|
def test_update_remote_ocr_endpoint_blocks_internal_endpoint_when_disallowed(
|
||||||
|
self,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- Internal remote OCR endpoints are disallowed
|
||||||
|
WHEN:
|
||||||
|
- The config is updated with a remote OCR endpoint resolving internally
|
||||||
|
THEN:
|
||||||
|
- The request is rejected
|
||||||
|
"""
|
||||||
|
response = self.client.patch(
|
||||||
|
f"{self.ENDPOINT}1/",
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"remote_ocr_endpoint": "http://127.0.0.1:5000",
|
||||||
|
},
|
||||||
|
),
|
||||||
|
content_type="application/json",
|
||||||
|
)
|
||||||
|
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
|
||||||
|
self.assertIn("non-public address", str(response.data).lower())
|
||||||
|
|
||||||
|
@override_settings(REMOTE_OCR_ALLOW_INTERNAL_ENDPOINTS=True)
|
||||||
|
def test_update_remote_ocr_endpoint_allows_internal_endpoint_by_default(
|
||||||
|
self,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- Internal remote OCR endpoints are allowed (the default)
|
||||||
|
WHEN:
|
||||||
|
- The config is updated with a remote OCR endpoint resolving internally
|
||||||
|
THEN:
|
||||||
|
- The request is accepted, preserving existing self-hosted deployments
|
||||||
|
"""
|
||||||
|
response = self.client.patch(
|
||||||
|
f"{self.ENDPOINT}1/",
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"remote_ocr_endpoint": "http://127.0.0.1:5000",
|
||||||
|
},
|
||||||
|
),
|
||||||
|
content_type="application/json",
|
||||||
|
)
|
||||||
|
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||||
|
self.assertEqual(
|
||||||
|
response.data["remote_ocr_endpoint"],
|
||||||
|
"http://127.0.0.1:5000",
|
||||||
|
)
|
||||||
|
|
||||||
|
@override_settings(REMOTE_OCR_ALLOW_INTERNAL_ENDPOINTS=False)
|
||||||
|
def test_update_remote_ocr_endpoint_empty_value_skips_validation(
|
||||||
|
self,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- Internal remote OCR endpoints are disallowed
|
||||||
|
WHEN:
|
||||||
|
- The config is updated with an empty remote OCR endpoint
|
||||||
|
THEN:
|
||||||
|
- The request is accepted; clearing the field never needs
|
||||||
|
outbound URL validation
|
||||||
|
"""
|
||||||
|
response = self.client.patch(
|
||||||
|
f"{self.ENDPOINT}1/",
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"remote_ocr_endpoint": "",
|
||||||
|
},
|
||||||
|
),
|
||||||
|
content_type="application/json",
|
||||||
|
)
|
||||||
|
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||||
|
self.assertEqual(response.data["remote_ocr_endpoint"], "")
|
||||||
|
|||||||
@@ -1003,8 +1003,8 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
|
|||||||
for correspondent in response.data[field]:
|
for correspondent in response.data[field]:
|
||||||
self.assertEqual(correspondent["document_count"], 0)
|
self.assertEqual(correspondent["document_count"], 0)
|
||||||
self.assertCountEqual(
|
self.assertCountEqual(
|
||||||
map(lambda c: c["id"], response.data[field]),
|
(c["id"] for c in response.data[field]),
|
||||||
map(lambda c: c["id"], Entity.objects.values("id")),
|
(c["id"] for c in Entity.objects.values("id")),
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_api_selection_data(self) -> None:
|
def test_api_selection_data(self) -> None:
|
||||||
|
|||||||
@@ -102,6 +102,7 @@ class TestApiObjects(DirectoriesMixin, APITestCase):
|
|||||||
- API is called
|
- API is called
|
||||||
THEN:
|
THEN:
|
||||||
- Last correspondence date is returned only if requested for list, and for detail
|
- Last correspondence date is returned only if requested for list, and for detail
|
||||||
|
- The date is scoped to documents the requesting user may view
|
||||||
"""
|
"""
|
||||||
|
|
||||||
Document.objects.create(
|
Document.objects.create(
|
||||||
@@ -145,6 +146,32 @@ class TestApiObjects(DirectoriesMixin, APITestCase):
|
|||||||
response.data["last_correspondence"],
|
response.data["last_correspondence"],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# A newer document owned by another user must not leak through the
|
||||||
|
# aggregate for a non-superuser who cannot view it
|
||||||
|
other = User.objects.create_user(username="other")
|
||||||
|
Document.objects.create(
|
||||||
|
mime_type="application/pdf",
|
||||||
|
correspondent=self.c1,
|
||||||
|
created=datetime.date(2023, 6, 1),
|
||||||
|
checksum="hidden",
|
||||||
|
owner=other,
|
||||||
|
)
|
||||||
|
|
||||||
|
user = User.objects.create_user(username="regular")
|
||||||
|
user.user_permissions.add(
|
||||||
|
Permission.objects.get(codename="view_correspondent"),
|
||||||
|
)
|
||||||
|
self.client.force_authenticate(user=user)
|
||||||
|
|
||||||
|
response = self.client.get("/api/correspondents/?last_correspondence=true")
|
||||||
|
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||||
|
result = next(r for r in response.data["results"] if r["id"] == self.c1.id)
|
||||||
|
self.assertIn("2022-01-02", result["last_correspondence"])
|
||||||
|
|
||||||
|
response = self.client.get(f"/api/correspondents/{self.c1.id}/")
|
||||||
|
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||||
|
self.assertIn("2022-01-02", response.data["last_correspondence"])
|
||||||
|
|
||||||
def test_paginated_objects_include_all_only_for_legacy_version(self) -> None:
|
def test_paginated_objects_include_all_only_for_legacy_version(self) -> None:
|
||||||
response_v10 = self.client.get("/api/correspondents/")
|
response_v10 = self.client.get("/api/correspondents/")
|
||||||
self.assertEqual(response_v10.status_code, status.HTTP_200_OK)
|
self.assertEqual(response_v10.status_code, status.HTTP_200_OK)
|
||||||
|
|||||||
@@ -18,8 +18,8 @@ class MockOpenIDProvider:
|
|||||||
|
|
||||||
def get_brands(self):
|
def get_brands(self):
|
||||||
default_servers = [
|
default_servers = [
|
||||||
dict(id="yahoo", name="Yahoo", openid_url="http://me.yahoo.com"),
|
{"id": "yahoo", "name": "Yahoo", "openid_url": "http://me.yahoo.com"},
|
||||||
dict(id="hyves", name="Hyves", openid_url="http://hyves.nl"),
|
{"id": "hyves", "name": "Hyves", "openid_url": "http://hyves.nl"},
|
||||||
]
|
]
|
||||||
return default_servers
|
return default_servers
|
||||||
|
|
||||||
|
|||||||
@@ -205,12 +205,12 @@ class TestBarcode(
|
|||||||
- Barcode is detected on page 1 (zero indexed)
|
- Barcode is detected on page 1 (zero indexed)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
for test_file in [
|
for test_filename in [
|
||||||
"patch-code-t-middle-reverse.pdf",
|
"patch-code-t-middle-reverse.pdf",
|
||||||
"patch-code-t-middle-distorted.pdf",
|
"patch-code-t-middle-distorted.pdf",
|
||||||
"patch-code-t-middle-fuzzy.pdf",
|
"patch-code-t-middle-fuzzy.pdf",
|
||||||
]:
|
]:
|
||||||
test_file = self.BARCODE_SAMPLE_DIR / test_file
|
test_file = self.BARCODE_SAMPLE_DIR / test_filename
|
||||||
|
|
||||||
with self.get_reader(test_file) as reader:
|
with self.get_reader(test_file) as reader:
|
||||||
reader.detect()
|
reader.detect()
|
||||||
|
|||||||
@@ -777,7 +777,7 @@ class TestPDFActions(DirectoriesMixin, TestCase):
|
|||||||
sig.set.return_value.apply_async.side_effect = Exception("boom")
|
sig.set.return_value.apply_async.side_effect = Exception("boom")
|
||||||
mock_consume_file.return_value = sig
|
mock_consume_file.return_value = sig
|
||||||
|
|
||||||
with self.assertRaises(Exception):
|
with self.assertRaises(Exception): # noqa: B017 - mock injects a bare Exception
|
||||||
bulk_edit.merge(doc_ids, delete_originals=True)
|
bulk_edit.merge(doc_ids, delete_originals=True)
|
||||||
|
|
||||||
self.doc1.refresh_from_db()
|
self.doc1.refresh_from_db()
|
||||||
@@ -1318,7 +1318,7 @@ class TestPDFActions(DirectoriesMixin, TestCase):
|
|||||||
sig.apply_async.side_effect = Exception("boom")
|
sig.apply_async.side_effect = Exception("boom")
|
||||||
mock_chord.return_value = sig
|
mock_chord.return_value = sig
|
||||||
|
|
||||||
with self.assertRaises(Exception):
|
with self.assertRaises(Exception): # noqa: B017 - mock injects a bare Exception
|
||||||
bulk_edit.edit_pdf(doc_ids, operations, delete_original=True)
|
bulk_edit.edit_pdf(doc_ids, operations, delete_original=True)
|
||||||
|
|
||||||
self.doc2.refresh_from_db()
|
self.doc2.refresh_from_db()
|
||||||
@@ -1430,7 +1430,7 @@ class TestPDFActions(DirectoriesMixin, TestCase):
|
|||||||
{"page": 9999}, # invalid page, forces error during PDF load
|
{"page": 9999}, # invalid page, forces error during PDF load
|
||||||
]
|
]
|
||||||
with self.assertLogs("paperless.bulk_edit", level="ERROR"):
|
with self.assertLogs("paperless.bulk_edit", level="ERROR"):
|
||||||
with self.assertRaises(Exception):
|
with self.assertRaises(ValueError):
|
||||||
bulk_edit.edit_pdf(doc_ids, operations)
|
bulk_edit.edit_pdf(doc_ids, operations)
|
||||||
mock_group.assert_not_called()
|
mock_group.assert_not_called()
|
||||||
mock_consume_file.assert_not_called()
|
mock_consume_file.assert_not_called()
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
import pickle
|
|
||||||
|
|
||||||
from documents.caching import StoredLRUCache
|
from documents.caching import StoredLRUCache
|
||||||
|
from paperless.signed_pickle import HMAC_SIZE
|
||||||
|
from paperless.signed_pickle import signed_pickle_dumps
|
||||||
|
from paperless.signed_pickle import signed_pickle_loads
|
||||||
|
|
||||||
|
|
||||||
def test_lru_cache_entries() -> None:
|
def test_lru_cache_entries() -> None:
|
||||||
@@ -42,4 +43,16 @@ def test_stored_lru_cache_key_ttl(mocker) -> None:
|
|||||||
key, data, timeout = mock_backend.set.call_args[0]
|
key, data, timeout = mock_backend.set.call_args[0]
|
||||||
assert key == "test_key"
|
assert key == "test_key"
|
||||||
assert timeout == 321
|
assert timeout == 321
|
||||||
assert pickle.loads(data) == {"x": "X", "y": "Y"}
|
assert signed_pickle_loads(data) == {"x": "X", "y": "Y"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_stored_lru_cache_rejects_tampered_data(mocker) -> None:
|
||||||
|
serialized_data = bytearray(signed_pickle_dumps({"x": "X"}))
|
||||||
|
serialized_data[HMAC_SIZE] ^= 0xFF
|
||||||
|
mock_backend = mocker.Mock()
|
||||||
|
mock_backend.get.return_value = bytes(serialized_data)
|
||||||
|
cache = StoredLRUCache("test_key", backend=mock_backend)
|
||||||
|
|
||||||
|
cache.load()
|
||||||
|
|
||||||
|
assert cache.get("x") is None
|
||||||
|
|||||||
@@ -19,6 +19,8 @@ from documents.models import MatchingModel
|
|||||||
from documents.models import StoragePath
|
from documents.models import StoragePath
|
||||||
from documents.models import Tag
|
from documents.models import Tag
|
||||||
from documents.tests.utils import DirectoriesMixin
|
from documents.tests.utils import DirectoriesMixin
|
||||||
|
from paperless.signed_pickle import HMAC_SIZE
|
||||||
|
from paperless.signed_pickle import signed_pickle_dumps
|
||||||
|
|
||||||
|
|
||||||
def dummy_preprocess(content: str, **kwargs):
|
def dummy_preprocess(content: str, **kwargs):
|
||||||
@@ -265,6 +267,27 @@ class TestClassifier(DirectoriesMixin, TestCase):
|
|||||||
self.assertEqual(mock_preprocess_content.call_count, 2)
|
self.assertEqual(mock_preprocess_content.call_count, 2)
|
||||||
self.assertEqual(mock_transform.call_count, 2)
|
self.assertEqual(mock_transform.call_count, 2)
|
||||||
|
|
||||||
|
def test_vectorize_recomputes_tampered_cache_entry(self) -> None:
|
||||||
|
cached = bytearray(signed_pickle_dumps(["cached vector"]))
|
||||||
|
cached[HMAC_SIZE] ^= 0xFF
|
||||||
|
self.classifier.data_vectorizer = mock.Mock()
|
||||||
|
self.classifier.data_vectorizer.transform.return_value = ["fresh vector"]
|
||||||
|
|
||||||
|
with (
|
||||||
|
mock.patch(
|
||||||
|
"documents.classifier.read_cache.get",
|
||||||
|
return_value=bytes(cached),
|
||||||
|
),
|
||||||
|
mock.patch("documents.classifier.read_cache.set") as cache_set,
|
||||||
|
mock.patch("documents.classifier.read_cache.touch") as cache_touch,
|
||||||
|
):
|
||||||
|
result = self.classifier._vectorize("content")
|
||||||
|
|
||||||
|
self.assertEqual(result, ["fresh vector"])
|
||||||
|
self.classifier.data_vectorizer.transform.assert_called_once()
|
||||||
|
cache_set.assert_called_once()
|
||||||
|
cache_touch.assert_not_called()
|
||||||
|
|
||||||
def test_no_retrain_if_no_change(self) -> None:
|
def test_no_retrain_if_no_change(self) -> None:
|
||||||
"""
|
"""
|
||||||
GIVEN:
|
GIVEN:
|
||||||
@@ -783,7 +806,7 @@ class TestClassifier(DirectoriesMixin, TestCase):
|
|||||||
|
|
||||||
Path(settings.MODEL_FILE).touch()
|
Path(settings.MODEL_FILE).touch()
|
||||||
mock_load.side_effect = Exception()
|
mock_load.side_effect = Exception()
|
||||||
with self.assertRaises(Exception):
|
with self.assertRaises(Exception): # noqa: B017 - mock injects a bare Exception
|
||||||
load_classifier(raise_exception=True)
|
load_classifier(raise_exception=True)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -137,7 +137,7 @@ class FaultyParser(_BaseNewStyleParser):
|
|||||||
|
|
||||||
class FaultyGenericExceptionParser(_BaseNewStyleParser):
|
class FaultyGenericExceptionParser(_BaseNewStyleParser):
|
||||||
def parse(self, document_path, mime_type, *, produce_archive: bool = True) -> None:
|
def parse(self, document_path, mime_type, *, produce_archive: bool = True) -> None:
|
||||||
raise Exception("Generic exception.")
|
raise Exception("Generic exception.") # noqa: TRY002 - deliberately not a ParseError
|
||||||
|
|
||||||
|
|
||||||
def fake_magic_from_file(file, *, mime=False): # NOSONAR
|
def fake_magic_from_file(file, *, mime=False): # NOSONAR
|
||||||
@@ -1356,7 +1356,7 @@ class PreConsumeTestCase(DirectoriesMixin, GetConsumerMixin, TestCase):
|
|||||||
script_calls = [
|
script_calls = [
|
||||||
call
|
call
|
||||||
for call in m.call_args_list
|
for call in m.call_args_list
|
||||||
if call.args and call.args[0] and call.args[0][0] not in ("pdftotext",)
|
if call.args and call.args[0] and call.args[0][0] != "pdftotext"
|
||||||
]
|
]
|
||||||
self.assertEqual(script_calls, [])
|
self.assertEqual(script_calls, [])
|
||||||
|
|
||||||
|
|||||||
@@ -156,6 +156,40 @@ class TestDocument(TestCase):
|
|||||||
)
|
)
|
||||||
self.assertEqual(doc.get_public_filename(), "2020-12-25 test")
|
self.assertEqual(doc.get_public_filename(), "2020-12-25 test")
|
||||||
|
|
||||||
|
def test_version_file_name_uses_root_document_metadata(self) -> None:
|
||||||
|
root_correspondent = Correspondent.objects.create(name="Root correspondent")
|
||||||
|
version_correspondent = Correspondent.objects.create(
|
||||||
|
name="Version correspondent",
|
||||||
|
)
|
||||||
|
root = Document.objects.create(
|
||||||
|
mime_type="application/pdf",
|
||||||
|
title="Root title",
|
||||||
|
created=date(2020, 12, 25),
|
||||||
|
correspondent=root_correspondent,
|
||||||
|
)
|
||||||
|
version = Document.objects.create(
|
||||||
|
mime_type="application/pdf",
|
||||||
|
title="Version title",
|
||||||
|
created=date(1990, 1, 1),
|
||||||
|
correspondent=version_correspondent,
|
||||||
|
root_document=root,
|
||||||
|
version_index=1,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
version.get_public_filename(),
|
||||||
|
"2020-12-25 Root correspondent Root title.pdf",
|
||||||
|
)
|
||||||
|
|
||||||
|
root.title = "Updated root title"
|
||||||
|
root.save(update_fields=("title",))
|
||||||
|
version.refresh_from_db()
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
version.get_public_filename(),
|
||||||
|
"2020-12-25 Root correspondent Updated root title.pdf",
|
||||||
|
)
|
||||||
|
|
||||||
def test_suggestion_content_uses_latest_version_content_for_root_documents(
|
def test_suggestion_content_uses_latest_version_content_for_root_documents(
|
||||||
self,
|
self,
|
||||||
) -> None:
|
) -> None:
|
||||||
|
|||||||
@@ -192,6 +192,50 @@ class ShareLinkBundleAPITests(DirectoriesMixin, APITestCase):
|
|||||||
self.assertEqual(response.status_code, status.HTTP_302_FOUND)
|
self.assertEqual(response.status_code, status.HTTP_302_FOUND)
|
||||||
self.assertIn("sharelink_notfound=1", response["Location"])
|
self.assertIn("sharelink_notfound=1", response["Location"])
|
||||||
|
|
||||||
|
def test_share_link_missing_file_redirects(self) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A share link whose document file is missing from disk
|
||||||
|
WHEN:
|
||||||
|
- The public share link is requested anonymously
|
||||||
|
THEN:
|
||||||
|
- The user is redirected to login instead of a 500 error
|
||||||
|
"""
|
||||||
|
doc = DocumentFactory.create(filename="missing-original.pdf")
|
||||||
|
share_link = ShareLink.objects.create(
|
||||||
|
slug="missingfilelink",
|
||||||
|
document=doc,
|
||||||
|
file_version=ShareLink.FileVersion.ORIGINAL,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.client.logout()
|
||||||
|
response = self.client.get(f"/share/{share_link.slug}/")
|
||||||
|
|
||||||
|
self.assertEqual(response.status_code, status.HTTP_302_FOUND)
|
||||||
|
self.assertIn("sharelink_notfound=1", response["Location"])
|
||||||
|
|
||||||
|
def test_download_ready_bundle_missing_file_returns_503(self) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A READY bundle whose zip file is missing from disk
|
||||||
|
WHEN:
|
||||||
|
- The public share link is requested anonymously
|
||||||
|
THEN:
|
||||||
|
- A 503 is returned instead of a 500 error
|
||||||
|
"""
|
||||||
|
bundle = ShareLinkBundle.objects.create(
|
||||||
|
slug="missingbundlefile",
|
||||||
|
file_version=ShareLink.FileVersion.ARCHIVE,
|
||||||
|
status=ShareLinkBundle.Status.READY,
|
||||||
|
file_path="bundles/gone.zip",
|
||||||
|
)
|
||||||
|
bundle.documents.set([self.document])
|
||||||
|
|
||||||
|
self.client.logout()
|
||||||
|
response = self.client.get(f"/share/{bundle.slug}/")
|
||||||
|
|
||||||
|
self.assertEqual(response.status_code, status.HTTP_503_SERVICE_UNAVAILABLE)
|
||||||
|
|
||||||
|
|
||||||
class ShareLinkBundleTaskTests(DirectoriesMixin, APITestCase):
|
class ShareLinkBundleTaskTests(DirectoriesMixin, APITestCase):
|
||||||
def setUp(self) -> None:
|
def setUp(self) -> None:
|
||||||
|
|||||||
@@ -44,6 +44,7 @@ from documents import tasks
|
|||||||
from documents.data_models import ConsumableDocument
|
from documents.data_models import ConsumableDocument
|
||||||
from documents.data_models import DocumentMetadataOverrides
|
from documents.data_models import DocumentMetadataOverrides
|
||||||
from documents.data_models import DocumentSource
|
from documents.data_models import DocumentSource
|
||||||
|
from documents.matching import UnsupportedWorkflowTriggerTypeError
|
||||||
from documents.matching import document_matches_workflow
|
from documents.matching import document_matches_workflow
|
||||||
from documents.matching import existing_document_matches_workflow
|
from documents.matching import existing_document_matches_workflow
|
||||||
from documents.matching import prefilter_documents_by_workflowtrigger
|
from documents.matching import prefilter_documents_by_workflowtrigger
|
||||||
@@ -2851,7 +2852,13 @@ class TestWorkflows(
|
|||||||
doc = Document.objects.create(
|
doc = Document.objects.create(
|
||||||
title="test",
|
title="test",
|
||||||
)
|
)
|
||||||
self.assertRaises(Exception, document_matches_workflow, doc, w, 99)
|
self.assertRaises(
|
||||||
|
UnsupportedWorkflowTriggerTypeError,
|
||||||
|
document_matches_workflow,
|
||||||
|
doc,
|
||||||
|
w,
|
||||||
|
99,
|
||||||
|
)
|
||||||
|
|
||||||
def test_removal_action_document_updated_workflow(self) -> None:
|
def test_removal_action_document_updated_workflow(self) -> None:
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -21,28 +21,32 @@ def uri_validator(value: str, allowed_schemes: set[str] | None = None) -> None:
|
|||||||
parts = urlparse(value)
|
parts = urlparse(value)
|
||||||
if not parts.scheme:
|
if not parts.scheme:
|
||||||
raise ValidationError(
|
raise ValidationError(
|
||||||
_(f"Unable to parse URI {value}, missing scheme"),
|
_("Unable to parse URI %(value)s, missing scheme"),
|
||||||
params={"value": value},
|
params={"value": value},
|
||||||
)
|
)
|
||||||
elif not parts.netloc and not parts.path:
|
elif not parts.netloc and not parts.path:
|
||||||
raise ValidationError(
|
raise ValidationError(
|
||||||
_(f"Unable to parse URI {value}, missing net location or path"),
|
_("Unable to parse URI %(value)s, missing net location or path"),
|
||||||
params={"value": value},
|
params={"value": value},
|
||||||
)
|
)
|
||||||
|
|
||||||
if allowed_schemes and parts.scheme not in allowed_schemes:
|
if allowed_schemes and parts.scheme not in allowed_schemes:
|
||||||
raise ValidationError(
|
raise ValidationError(
|
||||||
_(
|
_(
|
||||||
f"URI scheme '{parts.scheme}' is not allowed. Allowed schemes: {', '.join(allowed_schemes)}",
|
"URI scheme '%(scheme)s' is not allowed. Allowed schemes: %(allowed_schemes)s",
|
||||||
),
|
),
|
||||||
params={"value": value, "scheme": parts.scheme},
|
params={
|
||||||
|
"value": value,
|
||||||
|
"scheme": parts.scheme,
|
||||||
|
"allowed_schemes": ", ".join(allowed_schemes),
|
||||||
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
except ValidationError:
|
except ValidationError:
|
||||||
raise
|
raise
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
raise ValidationError(
|
raise ValidationError(
|
||||||
_(f"Unable to parse URI {value}"),
|
_("Unable to parse URI %(value)s"),
|
||||||
params={"value": value},
|
params={"value": value},
|
||||||
) from e
|
) from e
|
||||||
|
|
||||||
|
|||||||
+48
-35
@@ -577,13 +577,19 @@ class CorrespondentViewSet(
|
|||||||
def list(self, request, *args, **kwargs):
|
def list(self, request, *args, **kwargs):
|
||||||
if request.query_params.get("last_correspondence", None):
|
if request.query_params.get("last_correspondence", None):
|
||||||
self.queryset = self.queryset.annotate(
|
self.queryset = self.queryset.annotate(
|
||||||
last_correspondence=Max("documents__created"),
|
last_correspondence=Max(
|
||||||
|
"documents__created",
|
||||||
|
filter=self.get_document_count_filter(),
|
||||||
|
),
|
||||||
)
|
)
|
||||||
return super().list(request, *args, **kwargs)
|
return super().list(request, *args, **kwargs)
|
||||||
|
|
||||||
def retrieve(self, request, *args, **kwargs):
|
def retrieve(self, request, *args, **kwargs):
|
||||||
self.queryset = self.queryset.annotate(
|
self.queryset = self.queryset.annotate(
|
||||||
last_correspondence=Max("documents__created"),
|
last_correspondence=Max(
|
||||||
|
"documents__created",
|
||||||
|
filter=self.get_document_count_filter(),
|
||||||
|
),
|
||||||
)
|
)
|
||||||
return super().retrieve(request, *args, **kwargs)
|
return super().retrieve(request, *args, **kwargs)
|
||||||
|
|
||||||
@@ -1443,7 +1449,7 @@ class DocumentViewSet(
|
|||||||
try:
|
try:
|
||||||
lang = detect(doc.content)
|
lang = detect(doc.content)
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
logger.debug("Unable to detect language for document %s", doc.pk)
|
||||||
meta["lang"] = lang
|
meta["lang"] = lang
|
||||||
|
|
||||||
return Response(meta)
|
return Response(meta)
|
||||||
@@ -1481,13 +1487,12 @@ class DocumentViewSet(
|
|||||||
with get_date_parser() as date_parser:
|
with get_date_parser() as date_parser:
|
||||||
gen = date_parser.parse(doc.filename, doc.content)
|
gen = date_parser.parse(doc.filename, doc.content)
|
||||||
dates = sorted(
|
dates = sorted(
|
||||||
{
|
set(
|
||||||
i
|
itertools.islice(
|
||||||
for i in itertools.islice(
|
|
||||||
gen,
|
gen,
|
||||||
settings.NUMBER_OF_SUGGESTED_DATES,
|
settings.NUMBER_OF_SUGGESTED_DATES,
|
||||||
)
|
),
|
||||||
},
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
resp_data = {
|
resp_data = {
|
||||||
@@ -1575,21 +1580,16 @@ class DocumentViewSet(
|
|||||||
except ValueError as exc:
|
except ValueError as exc:
|
||||||
logger.exception(
|
logger.exception(
|
||||||
"Invalid AI configuration while generating suggestions for "
|
"Invalid AI configuration while generating suggestions for "
|
||||||
"document %s: %s",
|
"document %s",
|
||||||
doc.pk,
|
doc.pk,
|
||||||
exc,
|
|
||||||
exc_info=True,
|
|
||||||
)
|
)
|
||||||
raise ValidationError(
|
raise ValidationError(
|
||||||
{"ai": [_("Invalid AI configuration.")]},
|
{"ai": [_("Invalid AI configuration.")]},
|
||||||
) from exc
|
) from exc
|
||||||
except LLMTimeoutError as exc:
|
except LLMTimeoutError:
|
||||||
logger.exception(
|
logger.exception(
|
||||||
"AI backend timed out while generating suggestions for "
|
"AI backend timed out while generating suggestions for document %s",
|
||||||
"document %s: %s",
|
|
||||||
doc.pk,
|
doc.pk,
|
||||||
exc,
|
|
||||||
exc_info=True,
|
|
||||||
)
|
)
|
||||||
return Response(
|
return Response(
|
||||||
{"ai": [_("AI backend request timed out.")]},
|
{"ai": [_("AI backend request timed out.")]},
|
||||||
@@ -2062,7 +2062,7 @@ class DocumentViewSet(
|
|||||||
doc_name, doc_data = serializer.validated_data.get("document")
|
doc_name, doc_data = serializer.validated_data.get("document")
|
||||||
version_label = serializer.validated_data.get("version_label")
|
version_label = serializer.validated_data.get("version_label")
|
||||||
|
|
||||||
t = int(mktime(datetime.now().timetuple()))
|
t = int(mktime(datetime.now().timetuple())) # noqa: DTZ005 - mktime() requires a local time tuple
|
||||||
|
|
||||||
settings.SCRATCH_DIR.mkdir(parents=True, exist_ok=True)
|
settings.SCRATCH_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
@@ -3332,7 +3332,7 @@ class PostDocumentView(GenericAPIView[Any]):
|
|||||||
cf = serializer.validated_data.get("custom_fields")
|
cf = serializer.validated_data.get("custom_fields")
|
||||||
from_webui = serializer.validated_data.get("from_webui")
|
from_webui = serializer.validated_data.get("from_webui")
|
||||||
|
|
||||||
t = int(mktime(datetime.now().timetuple()))
|
t = int(mktime(datetime.now().timetuple())) # noqa: DTZ005 - mktime() requires a local time tuple
|
||||||
|
|
||||||
settings.SCRATCH_DIR.mkdir(parents=True, exist_ok=True)
|
settings.SCRATCH_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
@@ -4142,7 +4142,7 @@ class UiSettingsView(GenericAPIView[Any]):
|
|||||||
user_resp["last_name"] = user.last_name
|
user_resp["last_name"] = user.last_name
|
||||||
|
|
||||||
# strip <app_label>.
|
# strip <app_label>.
|
||||||
roles = map(lambda perm: re.sub(r"^\w+.", "", perm), user.get_all_permissions())
|
roles = (re.sub(r"^\w+.", "", perm) for perm in user.get_all_permissions())
|
||||||
return Response(
|
return Response(
|
||||||
{
|
{
|
||||||
"user": user_resp,
|
"user": user_resp,
|
||||||
@@ -4573,6 +4573,10 @@ class ShareLinkViewSet(
|
|||||||
class ShareLinkBundleViewSet(PassUserMixin, ModelViewSet[ShareLinkBundle]):
|
class ShareLinkBundleViewSet(PassUserMixin, ModelViewSet[ShareLinkBundle]):
|
||||||
model = ShareLinkBundle
|
model = ShareLinkBundle
|
||||||
|
|
||||||
|
# Bundles are immutable once created; rebuild via the dedicated action
|
||||||
|
# rather than PUT/PATCH.
|
||||||
|
http_method_names = ["get", "post", "delete", "head", "options"]
|
||||||
|
|
||||||
queryset = ShareLinkBundle.objects.all()
|
queryset = ShareLinkBundle.objects.all()
|
||||||
|
|
||||||
serializer_class = ShareLinkBundleSerializer
|
serializer_class = ShareLinkBundleSerializer
|
||||||
@@ -4707,12 +4711,15 @@ class SharedLinkView(View):
|
|||||||
and share_link.expiration < timezone.now()
|
and share_link.expiration < timezone.now()
|
||||||
):
|
):
|
||||||
return HttpResponseRedirect("/accounts/login/?sharelink_expired=1")
|
return HttpResponseRedirect("/accounts/login/?sharelink_expired=1")
|
||||||
return serve_file(
|
try:
|
||||||
doc=share_link.document,
|
return serve_file(
|
||||||
use_archive=share_link.file_version == ShareLink.FileVersion.ARCHIVE
|
doc=share_link.document,
|
||||||
and share_link.document.has_archive_version,
|
use_archive=share_link.file_version == ShareLink.FileVersion.ARCHIVE
|
||||||
disposition="inline",
|
and share_link.document.has_archive_version,
|
||||||
)
|
disposition="inline",
|
||||||
|
)
|
||||||
|
except FileNotFoundError:
|
||||||
|
return HttpResponseRedirect("/accounts/login/?sharelink_notfound=1")
|
||||||
|
|
||||||
bundle = ShareLinkBundle.objects.filter(slug=slug).first()
|
bundle = ShareLinkBundle.objects.filter(slug=slug).first()
|
||||||
if bundle is None:
|
if bundle is None:
|
||||||
@@ -4734,7 +4741,11 @@ class SharedLinkView(View):
|
|||||||
|
|
||||||
file_path = bundle.absolute_file_path
|
file_path = bundle.absolute_file_path
|
||||||
|
|
||||||
if bundle.status == ShareLinkBundle.Status.FAILED or file_path is None:
|
if (
|
||||||
|
bundle.status == ShareLinkBundle.Status.FAILED
|
||||||
|
or file_path is None
|
||||||
|
or not file_path.exists()
|
||||||
|
):
|
||||||
return HttpResponse(
|
return HttpResponse(
|
||||||
_(
|
_(
|
||||||
"The share link bundle is unavailable.",
|
"The share link bundle is unavailable.",
|
||||||
@@ -5169,11 +5180,11 @@ class SystemStatusView(PassUserMixin):
|
|||||||
f"{m.app}.{m.name}"
|
f"{m.app}.{m.name}"
|
||||||
for m in MigrationRecorder.Migration.objects.all().order_by("id")
|
for m in MigrationRecorder.Migration.objects.all().order_by("id")
|
||||||
]
|
]
|
||||||
except Exception as e: # pragma: no cover
|
except Exception: # pragma: no cover
|
||||||
applied_migrations = []
|
applied_migrations = []
|
||||||
db_status = "ERROR"
|
db_status = "ERROR"
|
||||||
logger.exception(
|
logger.exception(
|
||||||
f"System status detected a possible problem while connecting to the database: {e}",
|
"System status detected a possible problem while connecting to the database",
|
||||||
)
|
)
|
||||||
db_error = "Error connecting to database, check logs for more detail."
|
db_error = "Error connecting to database, check logs for more detail."
|
||||||
|
|
||||||
@@ -5189,10 +5200,10 @@ class SystemStatusView(PassUserMixin):
|
|||||||
try:
|
try:
|
||||||
client.ping()
|
client.ping()
|
||||||
redis_status = "OK"
|
redis_status = "OK"
|
||||||
except Exception as e:
|
except Exception:
|
||||||
redis_status = "ERROR"
|
redis_status = "ERROR"
|
||||||
logger.exception(
|
logger.exception(
|
||||||
f"System status detected a possible problem while connecting to redis: {e}",
|
"System status detected a possible problem while connecting to redis",
|
||||||
)
|
)
|
||||||
redis_error = "Error connecting to redis, check logs for more detail."
|
redis_error = "Error connecting to redis, check logs for more detail."
|
||||||
|
|
||||||
@@ -5222,10 +5233,10 @@ class SystemStatusView(PassUserMixin):
|
|||||||
else:
|
else:
|
||||||
celery_active = "WARNING"
|
celery_active = "WARNING"
|
||||||
celery_error = "Celery worker responded unexpectedly."
|
celery_error = "Celery worker responded unexpectedly."
|
||||||
except Exception as e:
|
except Exception:
|
||||||
celery_active = "ERROR"
|
celery_active = "ERROR"
|
||||||
logger.exception(
|
logger.exception(
|
||||||
f"System status detected a possible problem while connecting to celery: {e}",
|
"System status detected a possible problem while connecting to celery",
|
||||||
)
|
)
|
||||||
celery_error = "Error connecting to celery, check logs for more detail."
|
celery_error = "Error connecting to celery, check logs for more detail."
|
||||||
|
|
||||||
@@ -5240,13 +5251,15 @@ class SystemStatusView(PassUserMixin):
|
|||||||
index_dir = settings.INDEX_DIR
|
index_dir = settings.INDEX_DIR
|
||||||
mtimes = [p.stat().st_mtime for p in index_dir.iterdir() if p.is_file()]
|
mtimes = [p.stat().st_mtime for p in index_dir.iterdir() if p.is_file()]
|
||||||
index_last_modified = (
|
index_last_modified = (
|
||||||
make_aware(datetime.fromtimestamp(max(mtimes))) if mtimes else None
|
make_aware(datetime.fromtimestamp(max(mtimes))) # noqa: DTZ006 - make_aware() requires a naive datetime
|
||||||
|
if mtimes
|
||||||
|
else None
|
||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception:
|
||||||
index_status = "ERROR"
|
index_status = "ERROR"
|
||||||
index_error = "Error opening index, check logs for more detail."
|
index_error = "Error opening index, check logs for more detail."
|
||||||
logger.exception(
|
logger.exception(
|
||||||
f"System status detected a possible problem while opening the index: {e}",
|
"System status detected a possible problem while opening the index",
|
||||||
)
|
)
|
||||||
index_last_modified = None
|
index_last_modified = None
|
||||||
|
|
||||||
|
|||||||
@@ -66,7 +66,7 @@ def build_workflow_action_context(
|
|||||||
else None
|
else None
|
||||||
)
|
)
|
||||||
|
|
||||||
filename = document.original_file if document.original_file else ""
|
filename = document.original_file or ""
|
||||||
return {
|
return {
|
||||||
"title": overrides.title
|
"title": overrides.title
|
||||||
if overrides and overrides.title
|
if overrides and overrides.title
|
||||||
@@ -179,9 +179,9 @@ def execute_email_action(
|
|||||||
f"Sent {n_messages} notification email(s) to {action.email.to}",
|
f"Sent {n_messages} notification email(s) to {action.email.to}",
|
||||||
extra={"group": logging_group},
|
extra={"group": logging_group},
|
||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception(
|
logger.exception(
|
||||||
f"Error occurred sending notification email: {e}",
|
"Error occurred sending notification email",
|
||||||
extra={"group": logging_group},
|
extra={"group": logging_group},
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -265,9 +265,9 @@ def execute_webhook_action(
|
|||||||
f"Webhook to {action.webhook.url} queued",
|
f"Webhook to {action.webhook.url} queued",
|
||||||
extra={"group": logging_group},
|
extra={"group": logging_group},
|
||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception(
|
logger.exception(
|
||||||
f"Error occurred sending webhook: {e}",
|
"Error occurred sending webhook",
|
||||||
extra={"group": logging_group},
|
extra={"group": logging_group},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -47,7 +47,7 @@ def resolve_date(dates: list[str]) -> date | None:
|
|||||||
"""
|
"""
|
||||||
for value in dates:
|
for value in dates:
|
||||||
try:
|
try:
|
||||||
return datetime.strptime(value, "%Y-%m-%d").date()
|
return datetime.strptime(value, "%Y-%m-%d").date() # noqa: DTZ007 - only the calendar date is used, time/tz is discarded
|
||||||
except (TypeError, ValueError):
|
except (TypeError, ValueError):
|
||||||
logger.debug("Ignoring unparsable suggested date %s", value)
|
logger.debug("Ignoring unparsable suggested date %s", value)
|
||||||
return None
|
return None
|
||||||
|
|||||||
@@ -70,6 +70,6 @@ def send_webhook(
|
|||||||
logger.error(
|
logger.error(
|
||||||
f"Failed attempt sending webhook to {url}: {e}",
|
f"Failed attempt sending webhook to {url}: {e}",
|
||||||
)
|
)
|
||||||
raise e
|
raise
|
||||||
finally:
|
finally:
|
||||||
transport.close()
|
transport.close()
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
+3
-31
@@ -1,12 +1,12 @@
|
|||||||
import hmac
|
|
||||||
import os
|
import os
|
||||||
import pickle
|
|
||||||
from hashlib import sha256
|
|
||||||
|
|
||||||
from celery import Celery
|
from celery import Celery
|
||||||
from celery.signals import worker_process_init
|
from celery.signals import worker_process_init
|
||||||
from kombu.serialization import register
|
from kombu.serialization import register
|
||||||
|
|
||||||
|
from paperless.signed_pickle import signed_pickle_dumps
|
||||||
|
from paperless.signed_pickle import signed_pickle_loads
|
||||||
|
|
||||||
# Set the default Django settings module for the 'celery' program.
|
# Set the default Django settings module for the 'celery' program.
|
||||||
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "paperless.settings")
|
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "paperless.settings")
|
||||||
|
|
||||||
@@ -18,34 +18,6 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "paperless.settings")
|
|||||||
# on the worker side using Django's SECRET_KEY.
|
# on the worker side using Django's SECRET_KEY.
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
HMAC_SIZE = 32 # SHA-256 digest length
|
|
||||||
|
|
||||||
|
|
||||||
def _get_signing_key() -> bytes:
|
|
||||||
from django.conf import settings
|
|
||||||
|
|
||||||
return settings.SECRET_KEY.encode()
|
|
||||||
|
|
||||||
|
|
||||||
def signed_pickle_dumps(obj: object) -> bytes:
|
|
||||||
data = pickle.dumps(obj, protocol=pickle.HIGHEST_PROTOCOL)
|
|
||||||
signature = hmac.new(_get_signing_key(), data, sha256).digest()
|
|
||||||
return signature + data
|
|
||||||
|
|
||||||
|
|
||||||
def signed_pickle_loads(payload: bytes) -> object:
|
|
||||||
if len(payload) < HMAC_SIZE:
|
|
||||||
msg = "Signed-pickle payload too short"
|
|
||||||
raise ValueError(msg)
|
|
||||||
signature = payload[:HMAC_SIZE]
|
|
||||||
data = payload[HMAC_SIZE:]
|
|
||||||
expected = hmac.new(_get_signing_key(), data, sha256).digest()
|
|
||||||
if not hmac.compare_digest(signature, expected):
|
|
||||||
msg = "Signed-pickle HMAC verification failed — message may have been tampered with"
|
|
||||||
raise ValueError(msg)
|
|
||||||
return pickle.loads(data)
|
|
||||||
|
|
||||||
|
|
||||||
register(
|
register(
|
||||||
"signed-pickle",
|
"signed-pickle",
|
||||||
signed_pickle_dumps,
|
signed_pickle_dumps,
|
||||||
|
|||||||
@@ -241,7 +241,7 @@ def check_v3_minimum_upgrade_version(
|
|||||||
return []
|
return []
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
last_applied = sorted(applied)[-1] if applied else "(none)"
|
last_applied = max(applied) if applied else "(none)"
|
||||||
logger.error(
|
logger.error(
|
||||||
"V3 upgrade check failed: last applied documents migration is %r. "
|
"V3 upgrade check failed: last applied documents migration is %r. "
|
||||||
"Expected '1075_workflowaction_order' (v2.20.15). "
|
"Expected '1075_workflowaction_order' (v2.20.15). "
|
||||||
@@ -341,6 +341,7 @@ def get_tesseract_langs():
|
|||||||
proc = subprocess.run(
|
proc = subprocess.run(
|
||||||
[shutil.which("tesseract"), "--list-langs"],
|
[shutil.which("tesseract"), "--list-langs"],
|
||||||
capture_output=True,
|
capture_output=True,
|
||||||
|
check=False,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Decode bytes to string, split on newlines, trim out the header
|
# Decode bytes to string, split on newlines, trim out the header
|
||||||
|
|||||||
@@ -84,7 +84,7 @@ def get_parser_registry() -> ParserRegistry:
|
|||||||
ParserRegistry
|
ParserRegistry
|
||||||
The shared registry singleton.
|
The shared registry singleton.
|
||||||
"""
|
"""
|
||||||
global _registry, _discovery_complete
|
global _registry, _discovery_complete # noqa: PLW0603 - module-level singleton, no class to hold this state
|
||||||
|
|
||||||
with _lock:
|
with _lock:
|
||||||
if _registry is None:
|
if _registry is None:
|
||||||
@@ -113,7 +113,7 @@ def init_builtin_parsers() -> None:
|
|||||||
-------
|
-------
|
||||||
None
|
None
|
||||||
"""
|
"""
|
||||||
global _registry
|
global _registry # noqa: PLW0603 - module-level singleton, no class to hold this state
|
||||||
|
|
||||||
with _lock:
|
with _lock:
|
||||||
if _registry is None:
|
if _registry is None:
|
||||||
@@ -137,7 +137,7 @@ def reset_parser_registry() -> None:
|
|||||||
-------
|
-------
|
||||||
None
|
None
|
||||||
"""
|
"""
|
||||||
global _registry, _discovery_complete
|
global _registry, _discovery_complete # noqa: PLW0603 - module-level singleton, no class to hold this state
|
||||||
|
|
||||||
_registry = None
|
_registry = None
|
||||||
_discovery_complete = False
|
_discovery_complete = False
|
||||||
|
|||||||
@@ -32,6 +32,8 @@ if TYPE_CHECKING:
|
|||||||
import datetime
|
import datetime
|
||||||
from types import TracebackType
|
from types import TracebackType
|
||||||
|
|
||||||
|
from azure.core.pipeline import PipelineRequest
|
||||||
|
|
||||||
from paperless.parsers import MetadataEntry
|
from paperless.parsers import MetadataEntry
|
||||||
from paperless.parsers import ParserContext
|
from paperless.parsers import ParserContext
|
||||||
|
|
||||||
@@ -76,7 +78,7 @@ class RemoteEngineConfig:
|
|||||||
def engine_is_valid(self) -> bool:
|
def engine_is_valid(self) -> bool:
|
||||||
"""Return True when the engine is known and fully configured."""
|
"""Return True when the engine is known and fully configured."""
|
||||||
return (
|
return (
|
||||||
self.engine in ("azureai",)
|
self.engine == "azureai"
|
||||||
and self.api_key is not None
|
and self.api_key is not None
|
||||||
and not (self.engine == "azureai" and self.endpoint is None)
|
and not (self.engine == "azureai" and self.endpoint is None)
|
||||||
)
|
)
|
||||||
@@ -436,9 +438,45 @@ class RemoteDocumentParser:
|
|||||||
from azure.ai.documentintelligence.models import DocumentContentFormat
|
from azure.ai.documentintelligence.models import DocumentContentFormat
|
||||||
from azure.core.credentials import AzureKeyCredential
|
from azure.core.credentials import AzureKeyCredential
|
||||||
|
|
||||||
|
from paperless.network import validate_outbound_http_url
|
||||||
|
|
||||||
|
allow_internal = settings.REMOTE_OCR_ALLOW_INTERNAL_ENDPOINTS
|
||||||
|
|
||||||
|
try:
|
||||||
|
validate_outbound_http_url(config.endpoint, allow_internal=allow_internal)
|
||||||
|
except ValueError as e:
|
||||||
|
raise ParseError(f"Invalid remote OCR endpoint: {e}") from e
|
||||||
|
|
||||||
|
def _revalidate_request_host(request: PipelineRequest) -> None:
|
||||||
|
"""Re-validates the destination host of every request sent.
|
||||||
|
|
||||||
|
The check above only covers the moment the client is built. A
|
||||||
|
single analysis involves several requests spread over the
|
||||||
|
polling loop below, and any one of them can be redirected.
|
||||||
|
Wiring this through ``raw_request_hook`` (Azure's built-in
|
||||||
|
CustomHookPolicy) rather than a custom policy means it runs
|
||||||
|
*after* RedirectPolicy in the pipeline, so it sees - and
|
||||||
|
re-checks - every actual outbound URL, including redirect
|
||||||
|
targets, not just the original request.
|
||||||
|
"""
|
||||||
|
validate_outbound_http_url(
|
||||||
|
request.http_request.url,
|
||||||
|
allow_internal=allow_internal,
|
||||||
|
)
|
||||||
|
|
||||||
client = DocumentIntelligenceClient(
|
client = DocumentIntelligenceClient(
|
||||||
endpoint=config.endpoint,
|
endpoint=config.endpoint,
|
||||||
credential=AzureKeyCredential(config.api_key),
|
credential=AzureKeyCredential(config.api_key),
|
||||||
|
raw_request_hook=_revalidate_request_host,
|
||||||
|
# AzureKeyCredential is sent as Ocp-Apim-Subscription-Key, which
|
||||||
|
# Azure's default SensitiveHeaderCleanupPolicy does not strip on
|
||||||
|
# a cross-domain redirect (only Authorization and
|
||||||
|
# x-ms-authorization-auxiliary are, by default).
|
||||||
|
blocked_redirect_headers=[
|
||||||
|
"Authorization",
|
||||||
|
"x-ms-authorization-auxiliary",
|
||||||
|
"Ocp-Apim-Subscription-Key",
|
||||||
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@@ -467,7 +505,7 @@ class RemoteDocumentParser:
|
|||||||
return result.content
|
return result.content
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.exception("Azure AI Vision parsing failed: %s", e)
|
logger.exception("Azure AI Vision parsing failed")
|
||||||
raise ParseError(f"Azure AI Vision parsing failed: {e}") from e
|
raise ParseError(f"Azure AI Vision parsing failed: {e}") from e
|
||||||
|
|
||||||
finally:
|
finally:
|
||||||
|
|||||||
@@ -306,8 +306,9 @@ def extract_pdf_metadata(
|
|||||||
|
|
||||||
for key, value in meta.items():
|
for key, value in meta.items():
|
||||||
if isinstance(value, list):
|
if isinstance(value, list):
|
||||||
value = " ".join(str(e) for e in value)
|
str_value = " ".join(str(e) for e in value)
|
||||||
value = str(value)
|
else:
|
||||||
|
str_value = str(value)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
m = namespace_pattern.match(key)
|
m = namespace_pattern.match(key)
|
||||||
@@ -329,7 +330,7 @@ def extract_pdf_metadata(
|
|||||||
namespace=namespace,
|
namespace=namespace,
|
||||||
prefix=meta.REVERSE_NS[namespace],
|
prefix=meta.REVERSE_NS[namespace],
|
||||||
key=key_value,
|
key=key_value,
|
||||||
value=value,
|
value=str_value,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
|||||||
@@ -305,6 +305,22 @@ class ApplicationConfigurationSerializer(
|
|||||||
|
|
||||||
validate_llm_embedding_endpoint = validate_llm_endpoint
|
validate_llm_embedding_endpoint = validate_llm_endpoint
|
||||||
|
|
||||||
|
def validate_remote_ocr_endpoint(self, value: str | None) -> str | None:
|
||||||
|
if not value:
|
||||||
|
return value
|
||||||
|
|
||||||
|
try:
|
||||||
|
validate_outbound_http_url(
|
||||||
|
value,
|
||||||
|
allow_internal=settings.REMOTE_OCR_ALLOW_INTERNAL_ENDPOINTS,
|
||||||
|
)
|
||||||
|
except ValueError as e:
|
||||||
|
raise serializers.ValidationError(
|
||||||
|
f"Invalid remote OCR endpoint: {e.args[0]}, see logs for details",
|
||||||
|
) from e
|
||||||
|
|
||||||
|
return value
|
||||||
|
|
||||||
class Meta:
|
class Meta:
|
||||||
model = ApplicationConfiguration
|
model = ApplicationConfiguration
|
||||||
fields = "__all__"
|
fields = "__all__"
|
||||||
|
|||||||
@@ -294,7 +294,7 @@ if _CHANNELS_BACKEND.startswith("channels_redis."):
|
|||||||
###############################################################################
|
###############################################################################
|
||||||
|
|
||||||
EMAIL_HOST: Final[str] = os.getenv("PAPERLESS_EMAIL_HOST", "localhost")
|
EMAIL_HOST: Final[str] = os.getenv("PAPERLESS_EMAIL_HOST", "localhost")
|
||||||
EMAIL_PORT: Final[int] = int(os.getenv("PAPERLESS_EMAIL_PORT", 25))
|
EMAIL_PORT: Final[int] = get_int_from_env("PAPERLESS_EMAIL_PORT", 25)
|
||||||
EMAIL_HOST_USER: Final[str] = os.getenv("PAPERLESS_EMAIL_HOST_USER", "")
|
EMAIL_HOST_USER: Final[str] = os.getenv("PAPERLESS_EMAIL_HOST_USER", "")
|
||||||
EMAIL_HOST_PASSWORD: Final[str] = os.getenv("PAPERLESS_EMAIL_HOST_PASSWORD", "")
|
EMAIL_HOST_PASSWORD: Final[str] = os.getenv("PAPERLESS_EMAIL_HOST_PASSWORD", "")
|
||||||
DEFAULT_FROM_EMAIL: Final[str] = os.getenv("PAPERLESS_EMAIL_FROM", EMAIL_HOST_USER)
|
DEFAULT_FROM_EMAIL: Final[str] = os.getenv("PAPERLESS_EMAIL_FROM", EMAIL_HOST_USER)
|
||||||
@@ -381,8 +381,9 @@ ACCOUNT_SESSION_REMEMBER = get_bool_from_env(
|
|||||||
"True",
|
"True",
|
||||||
)
|
)
|
||||||
SESSION_EXPIRE_AT_BROWSER_CLOSE = not ACCOUNT_SESSION_REMEMBER
|
SESSION_EXPIRE_AT_BROWSER_CLOSE = not ACCOUNT_SESSION_REMEMBER
|
||||||
SESSION_COOKIE_AGE = int(
|
SESSION_COOKIE_AGE = get_int_from_env(
|
||||||
os.getenv("PAPERLESS_SESSION_COOKIE_AGE", 60 * 60 * 24 * 7 * 3),
|
"PAPERLESS_SESSION_COOKIE_AGE",
|
||||||
|
60 * 60 * 24 * 7 * 3,
|
||||||
)
|
)
|
||||||
# https://docs.djangoproject.com/en/5.1/ref/settings/#std-setting-SESSION_ENGINE
|
# https://docs.djangoproject.com/en/5.1/ref/settings/#std-setting-SESSION_ENGINE
|
||||||
SESSION_ENGINE = "django.contrib.sessions.backends.cached_db"
|
SESSION_ENGINE = "django.contrib.sessions.backends.cached_db"
|
||||||
@@ -395,7 +396,6 @@ if AUTO_LOGIN_USERNAME:
|
|||||||
|
|
||||||
|
|
||||||
def _parse_remote_user_settings() -> str:
|
def _parse_remote_user_settings() -> str:
|
||||||
global MIDDLEWARE, AUTHENTICATION_BACKENDS, REST_FRAMEWORK
|
|
||||||
enable = get_bool_from_env("PAPERLESS_ENABLE_HTTP_REMOTE_USER")
|
enable = get_bool_from_env("PAPERLESS_ENABLE_HTTP_REMOTE_USER")
|
||||||
enable_api = get_bool_from_env("PAPERLESS_ENABLE_HTTP_REMOTE_USER_API")
|
enable_api = get_bool_from_env("PAPERLESS_ENABLE_HTTP_REMOTE_USER_API")
|
||||||
if enable or enable_api:
|
if enable or enable_api:
|
||||||
@@ -454,7 +454,6 @@ if ALLOWED_HOSTS != ["*"]:
|
|||||||
|
|
||||||
|
|
||||||
def _parse_paperless_url():
|
def _parse_paperless_url():
|
||||||
global CSRF_TRUSTED_ORIGINS, CORS_ALLOWED_ORIGINS, ALLOWED_HOSTS
|
|
||||||
url = os.getenv("PAPERLESS_URL")
|
url = os.getenv("PAPERLESS_URL")
|
||||||
if url:
|
if url:
|
||||||
CSRF_TRUSTED_ORIGINS.append(url)
|
CSRF_TRUSTED_ORIGINS.append(url)
|
||||||
@@ -614,8 +613,8 @@ USE_TZ = True
|
|||||||
|
|
||||||
LOGGING_DIR.mkdir(parents=True, exist_ok=True)
|
LOGGING_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
LOGROTATE_MAX_SIZE = os.getenv("PAPERLESS_LOGROTATE_MAX_SIZE", 1024 * 1024)
|
LOGROTATE_MAX_SIZE = get_int_from_env("PAPERLESS_LOGROTATE_MAX_SIZE", 1024 * 1024)
|
||||||
LOGROTATE_MAX_BACKUPS = os.getenv("PAPERLESS_LOGROTATE_MAX_BACKUPS", 20)
|
LOGROTATE_MAX_BACKUPS = get_int_from_env("PAPERLESS_LOGROTATE_MAX_BACKUPS", 20)
|
||||||
|
|
||||||
LOGGING = {
|
LOGGING = {
|
||||||
"version": 1,
|
"version": 1,
|
||||||
@@ -811,9 +810,15 @@ IGNORABLE_FILES: Final[list[str]] = [
|
|||||||
"Thumbs.db",
|
"Thumbs.db",
|
||||||
]
|
]
|
||||||
|
|
||||||
CONSUMER_POLLING_INTERVAL = float(os.getenv("PAPERLESS_CONSUMER_POLLING_INTERVAL", 0))
|
CONSUMER_POLLING_INTERVAL = get_float_from_env(
|
||||||
|
"PAPERLESS_CONSUMER_POLLING_INTERVAL",
|
||||||
|
0.0,
|
||||||
|
)
|
||||||
|
|
||||||
CONSUMER_STABILITY_DELAY = float(os.getenv("PAPERLESS_CONSUMER_STABILITY_DELAY", 5))
|
CONSUMER_STABILITY_DELAY = get_float_from_env(
|
||||||
|
"PAPERLESS_CONSUMER_STABILITY_DELAY",
|
||||||
|
5.0,
|
||||||
|
)
|
||||||
|
|
||||||
CONSUMER_DELETE_DUPLICATES = get_bool_from_env("PAPERLESS_CONSUMER_DELETE_DUPLICATES")
|
CONSUMER_DELETE_DUPLICATES = get_bool_from_env("PAPERLESS_CONSUMER_DELETE_DUPLICATES")
|
||||||
|
|
||||||
@@ -1208,6 +1213,10 @@ REMOTE_OCR_MODE = get_choice_from_env(
|
|||||||
{"always", "workflow_only"},
|
{"always", "workflow_only"},
|
||||||
default="always",
|
default="always",
|
||||||
)
|
)
|
||||||
|
REMOTE_OCR_ALLOW_INTERNAL_ENDPOINTS = get_bool_from_env(
|
||||||
|
"PAPERLESS_REMOTE_OCR_ALLOW_INTERNAL_ENDPOINTS",
|
||||||
|
"true",
|
||||||
|
)
|
||||||
|
|
||||||
################################################################################
|
################################################################################
|
||||||
# AI Settings #
|
# AI Settings #
|
||||||
|
|||||||
@@ -111,7 +111,7 @@ def parse_dict_from_str(
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
settings: dict[str, Any] = copy.deepcopy(defaults) if defaults else {}
|
settings: dict[str, Any] = copy.deepcopy(defaults) if defaults else {}
|
||||||
_type_map = type_map if type_map else {}
|
_type_map = type_map or {}
|
||||||
|
|
||||||
if not env_str:
|
if not env_str:
|
||||||
return settings
|
return settings
|
||||||
|
|||||||
@@ -0,0 +1,39 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hmac
|
||||||
|
import pickle
|
||||||
|
from hashlib import sha256
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from django.conf import settings
|
||||||
|
|
||||||
|
HMAC_SIZE = sha256().digest_size
|
||||||
|
|
||||||
|
|
||||||
|
class SignedPickleError(ValueError):
|
||||||
|
"""Raised when a signed pickle payload cannot be authenticated."""
|
||||||
|
|
||||||
|
|
||||||
|
def _get_signing_key() -> bytes:
|
||||||
|
return settings.SECRET_KEY.encode()
|
||||||
|
|
||||||
|
|
||||||
|
def signed_pickle_dumps(obj: object) -> bytes:
|
||||||
|
data = pickle.dumps(obj, protocol=pickle.HIGHEST_PROTOCOL)
|
||||||
|
signature = hmac.new(_get_signing_key(), data, sha256).digest()
|
||||||
|
return signature + data
|
||||||
|
|
||||||
|
|
||||||
|
def signed_pickle_loads(payload: bytes) -> Any:
|
||||||
|
if len(payload) <= HMAC_SIZE:
|
||||||
|
msg = "Signed-pickle payload too short"
|
||||||
|
raise SignedPickleError(msg)
|
||||||
|
|
||||||
|
signature = payload[:HMAC_SIZE]
|
||||||
|
data = payload[HMAC_SIZE:]
|
||||||
|
expected = hmac.new(_get_signing_key(), data, sha256).digest()
|
||||||
|
if not hmac.compare_digest(signature, expected):
|
||||||
|
msg = "Signed-pickle HMAC verification failed; payload may have been tampered with"
|
||||||
|
raise SignedPickleError(msg)
|
||||||
|
|
||||||
|
return pickle.loads(data)
|
||||||
@@ -114,17 +114,17 @@ def test_cache_hit_when_enabled() -> None:
|
|||||||
assert settings.CACHALOT_TIMEOUT == 1
|
assert settings.CACHALOT_TIMEOUT == 1
|
||||||
|
|
||||||
# Read a table to populate the cache
|
# Read a table to populate the cache
|
||||||
list(list(Tag.objects.values_list("id", flat=True)))
|
list(Tag.objects.values_list("id", flat=True))
|
||||||
|
|
||||||
# Invalidate the cache then read the database, there should be DB hit
|
# Invalidate the cache then read the database, there should be DB hit
|
||||||
invalidate_db_cache()
|
invalidate_db_cache()
|
||||||
with CaptureQueriesContext(connection) as ctx:
|
with CaptureQueriesContext(connection) as ctx:
|
||||||
list(list(Tag.objects.values_list("id", flat=True)))
|
list(Tag.objects.values_list("id", flat=True))
|
||||||
assert len(ctx)
|
assert len(ctx)
|
||||||
|
|
||||||
# Doing the same request again should hit the cache, not the DB
|
# Doing the same request again should hit the cache, not the DB
|
||||||
with CaptureQueriesContext(connection) as ctx:
|
with CaptureQueriesContext(connection) as ctx:
|
||||||
list(list(Tag.objects.values_list("id", flat=True)))
|
list(Tag.objects.values_list("id", flat=True))
|
||||||
assert not len(ctx)
|
assert not len(ctx)
|
||||||
|
|
||||||
# Wait the end of TTL
|
# Wait the end of TTL
|
||||||
@@ -133,7 +133,7 @@ def test_cache_hit_when_enabled() -> None:
|
|||||||
|
|
||||||
# Read the DB again. The DB should be hit because the cache has expired
|
# Read the DB again. The DB should be hit because the cache has expired
|
||||||
with CaptureQueriesContext(connection) as ctx:
|
with CaptureQueriesContext(connection) as ctx:
|
||||||
list(list(Tag.objects.values_list("id", flat=True)))
|
list(Tag.objects.values_list("id", flat=True))
|
||||||
assert len(ctx)
|
assert len(ctx)
|
||||||
|
|
||||||
# Invalidate the cache at the end of test
|
# Invalidate the cache at the end of test
|
||||||
@@ -149,7 +149,7 @@ def test_cache_is_disabled_by_default() -> None:
|
|||||||
# Read the table multiple times: the DB should always be hit without cache
|
# Read the table multiple times: the DB should always be hit without cache
|
||||||
for _ in range(3):
|
for _ in range(3):
|
||||||
with CaptureQueriesContext(connection) as ctx:
|
with CaptureQueriesContext(connection) as ctx:
|
||||||
list(list(Tag.objects.values_list("id", flat=True)))
|
list(Tag.objects.values_list("id", flat=True))
|
||||||
assert len(ctx)
|
assert len(ctx)
|
||||||
|
|
||||||
# Invalidate the cache at the end of test
|
# Invalidate the cache at the end of test
|
||||||
|
|||||||
@@ -6,9 +6,9 @@ from pathlib import Path
|
|||||||
import pytest
|
import pytest
|
||||||
from django.test import override_settings
|
from django.test import override_settings
|
||||||
|
|
||||||
from paperless.celery import HMAC_SIZE
|
|
||||||
from paperless.celery import signed_pickle_dumps
|
from paperless.celery import signed_pickle_dumps
|
||||||
from paperless.celery import signed_pickle_loads
|
from paperless.celery import signed_pickle_loads
|
||||||
|
from paperless.signed_pickle import HMAC_SIZE
|
||||||
|
|
||||||
|
|
||||||
class TestSignedPickleSerializer:
|
class TestSignedPickleSerializer:
|
||||||
|
|||||||
@@ -59,7 +59,7 @@ def test_ocr_to_dateparser_languages_exception(
|
|||||||
raise RuntimeError("Simulated error")
|
raise RuntimeError("Simulated error")
|
||||||
|
|
||||||
with caplog.at_level(logging.WARNING):
|
with caplog.at_level(logging.WARNING):
|
||||||
monkeypatch.setattr(utils, "LocaleDataLoader", lambda: DummyLoader())
|
monkeypatch.setattr(utils, "LocaleDataLoader", DummyLoader)
|
||||||
result = utils.ocr_to_dateparser_languages("eng+fra")
|
result = utils.ocr_to_dateparser_languages("eng+fra")
|
||||||
assert result == []
|
assert result == []
|
||||||
assert (
|
assert (
|
||||||
|
|||||||
@@ -295,7 +295,7 @@ urlpatterns = [
|
|||||||
],
|
],
|
||||||
),
|
),
|
||||||
),
|
),
|
||||||
re_path(r"share/(?P<slug>\w+)/?$", SharedLinkView.as_view()),
|
re_path(r"^share/(?P<slug>\w+)/?$", SharedLinkView.as_view()),
|
||||||
re_path(r"^favicon.ico$", FaviconView.as_view(), name="favicon"),
|
re_path(r"^favicon.ico$", FaviconView.as_view(), name="favicon"),
|
||||||
re_path(r"admin/", admin.site.urls),
|
re_path(r"admin/", admin.site.urls),
|
||||||
re_path(
|
re_path(
|
||||||
|
|||||||
@@ -103,8 +103,8 @@ def stream_chat_with_documents(
|
|||||||
documents,
|
documents,
|
||||||
output_language=output_language,
|
output_language=output_language,
|
||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception("Failed to stream document chat response: %s", e)
|
logger.exception("Failed to stream document chat response")
|
||||||
yield CHAT_ERROR_MESSAGE
|
yield CHAT_ERROR_MESSAGE
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -164,7 +164,7 @@ def test_get_ai_document_classification_failure(mock_run_llm_query, mock_documen
|
|||||||
"""
|
"""
|
||||||
mock_run_llm_query.side_effect = Exception("LLM query failed")
|
mock_run_llm_query.side_effect = Exception("LLM query failed")
|
||||||
|
|
||||||
with pytest.raises(Exception):
|
with pytest.raises(Exception): # noqa: B017 - mock injects a bare Exception
|
||||||
get_ai_document_classification(mock_document)
|
get_ai_document_classification(mock_document)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -21,5 +21,6 @@ class TestLazyAiImports:
|
|||||||
capture_output=True,
|
capture_output=True,
|
||||||
text=True,
|
text=True,
|
||||||
cwd=_SRC_DIR,
|
cwd=_SRC_DIR,
|
||||||
|
check=False,
|
||||||
)
|
)
|
||||||
assert result.returncode == 0, result.stdout + result.stderr
|
assert result.returncode == 0, result.stdout + result.stderr
|
||||||
|
|||||||
@@ -7,7 +7,6 @@ import ssl
|
|||||||
import tempfile
|
import tempfile
|
||||||
import traceback
|
import traceback
|
||||||
import unicodedata
|
import unicodedata
|
||||||
from datetime import date
|
|
||||||
from datetime import timedelta
|
from datetime import timedelta
|
||||||
from fnmatch import fnmatch
|
from fnmatch import fnmatch
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -406,7 +405,7 @@ def make_criterias(rule: MailRule, *, supports_gmail_labels: bool):
|
|||||||
Returns criteria to be applied to MailBox.fetch for the given rule.
|
Returns criteria to be applied to MailBox.fetch for the given rule.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
maximum_age = date.today() - timedelta(days=rule.maximum_age)
|
maximum_age = timezone.localdate() - timedelta(days=rule.maximum_age)
|
||||||
criterias = {}
|
criterias = {}
|
||||||
if rule.maximum_age > 0:
|
if rule.maximum_age > 0:
|
||||||
criterias["date_gte"] = maximum_age
|
criterias["date_gte"] = maximum_age
|
||||||
@@ -723,9 +722,9 @@ class MailAccountHandler(LoggingMixin):
|
|||||||
f"Rule {rule}: Stopping processing rules due to stop_processing flag",
|
f"Rule {rule}: Stopping processing rules due to stop_processing flag",
|
||||||
)
|
)
|
||||||
break
|
break
|
||||||
except Exception as e:
|
except Exception:
|
||||||
self.log.exception(
|
self.log.exception(
|
||||||
f"Rule {rule}: Error while processing rule: {e}",
|
f"Rule {rule}: Error while processing rule",
|
||||||
)
|
)
|
||||||
except MailError:
|
except MailError:
|
||||||
raise
|
raise
|
||||||
@@ -767,8 +766,8 @@ class MailAccountHandler(LoggingMixin):
|
|||||||
self.log.info(f"Located folder: {folder_info.name}")
|
self.log.info(f"Located folder: {folder_info.name}")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
self.log.error(
|
self.log.error(
|
||||||
"Exception during folder listing, unable to provide list folders: "
|
"Exception during folder listing, unable to provide list folders: %s",
|
||||||
+ str(e),
|
str(e),
|
||||||
)
|
)
|
||||||
|
|
||||||
raise MailError(
|
raise MailError(
|
||||||
@@ -874,9 +873,9 @@ class MailAccountHandler(LoggingMixin):
|
|||||||
|
|
||||||
total_processed_files += processed_files
|
total_processed_files += processed_files
|
||||||
mails_processed += 1
|
mails_processed += 1
|
||||||
except Exception as e:
|
except Exception:
|
||||||
self.log.exception(
|
self.log.exception(
|
||||||
f"Rule {rule}: Error while processing mail {message.uid}: {e}",
|
f"Rule {rule}: Error while processing mail {message.uid}",
|
||||||
)
|
)
|
||||||
|
|
||||||
self.log.debug(f"Rule {rule}: Processed {mails_processed} matching mail(s)")
|
self.log.debug(f"Rule {rule}: Processed {mails_processed} matching mail(s)")
|
||||||
|
|||||||
@@ -11,6 +11,10 @@ from imap_tools import MailMessage
|
|||||||
from documents.loggers import LoggingMixin
|
from documents.loggers import LoggingMixin
|
||||||
|
|
||||||
|
|
||||||
|
class MailDecryptionError(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
class MailMessagePreprocessor(abc.ABC):
|
class MailMessagePreprocessor(abc.ABC):
|
||||||
"""
|
"""
|
||||||
Defines the interface for preprocessors that alter messages before they are handled in MailAccountHandler
|
Defines the interface for preprocessors that alter messages before they are handled in MailAccountHandler
|
||||||
@@ -69,7 +73,7 @@ class MailMessageDecryptor(MailMessagePreprocessor, LoggingMixin):
|
|||||||
f"Message decryption failed with status message "
|
f"Message decryption failed with status message "
|
||||||
f"{decrypted_raw_message.status}",
|
f"{decrypted_raw_message.status}",
|
||||||
)
|
)
|
||||||
raise Exception(
|
raise MailDecryptionError(
|
||||||
f"Decryption failed: {decrypted_raw_message.status}, {decrypted_raw_message.stderr}",
|
f"Decryption failed: {decrypted_raw_message.status}, {decrypted_raw_message.stderr}",
|
||||||
)
|
)
|
||||||
self.log.debug("Message decrypted successfully.")
|
self.log.debug("Message decrypted successfully.")
|
||||||
|
|||||||
@@ -50,7 +50,7 @@ class ProcessedMailFactory(DjangoModelFactory[ProcessedMail]):
|
|||||||
|
|
||||||
rule = factory.SubFactory(MailRuleFactory)
|
rule = factory.SubFactory(MailRuleFactory)
|
||||||
folder = "INBOX"
|
folder = "INBOX"
|
||||||
uid = factory.Sequence(lambda n: str(n))
|
uid = factory.Sequence(str)
|
||||||
subject = factory.Faker("sentence", nb_words=4)
|
subject = factory.Faker("sentence", nb_words=4)
|
||||||
received = factory.LazyFunction(timezone.now)
|
received = factory.LazyFunction(timezone.now)
|
||||||
processed = factory.LazyFunction(timezone.now)
|
processed = factory.LazyFunction(timezone.now)
|
||||||
|
|||||||
@@ -214,7 +214,7 @@ class BogusMailBox(AbstractContextManager):
|
|||||||
)
|
)
|
||||||
self.messages = list(filter(lambda m: m.uid not in uid_list, self.messages))
|
self.messages = list(filter(lambda m: m.uid not in uid_list, self.messages))
|
||||||
else:
|
else:
|
||||||
raise Exception
|
raise Exception # noqa: TRY002 - test double simulating a generic mailbox failure
|
||||||
|
|
||||||
|
|
||||||
def fake_magic_from_buffer(buffer, *, mime=False):
|
def fake_magic_from_buffer(buffer, *, mime=False):
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ from imap_tools import MailMessage
|
|||||||
|
|
||||||
from paperless_mail.mail import MailAccountHandler
|
from paperless_mail.mail import MailAccountHandler
|
||||||
from paperless_mail.models import MailRule
|
from paperless_mail.models import MailRule
|
||||||
|
from paperless_mail.preprocessor import MailDecryptionError
|
||||||
from paperless_mail.preprocessor import MailMessageDecryptor
|
from paperless_mail.preprocessor import MailMessageDecryptor
|
||||||
from paperless_mail.tests.factories import MailAccountFactory
|
from paperless_mail.tests.factories import MailAccountFactory
|
||||||
from paperless_mail.tests.test_mail import TestMail
|
from paperless_mail.tests.test_mail import TestMail
|
||||||
@@ -82,7 +83,9 @@ class MessageEncryptor:
|
|||||||
armor=True,
|
armor=True,
|
||||||
)
|
)
|
||||||
if not encrypted_data.ok:
|
if not encrypted_data.ok:
|
||||||
raise Exception(f"Encryption failed: {encrypted_data.stderr}")
|
raise Exception( # noqa: TRY002 - test fixture setup, not production code
|
||||||
|
f"Encryption failed: {encrypted_data.stderr}",
|
||||||
|
)
|
||||||
encrypted_email_content = encrypted_data.data
|
encrypted_email_content = encrypted_data.data
|
||||||
|
|
||||||
new_email = MIMEMultipart("encrypted", protocol="application/pgp-encrypted")
|
new_email = MIMEMultipart("encrypted", protocol="application/pgp-encrypted")
|
||||||
@@ -184,7 +187,11 @@ class TestMailMessageGpgDecryptor(TestMail):
|
|||||||
EMAIL_GNUPG_HOME=empty_gpg_home,
|
EMAIL_GNUPG_HOME=empty_gpg_home,
|
||||||
):
|
):
|
||||||
message_decryptor = MailMessageDecryptor()
|
message_decryptor = MailMessageDecryptor()
|
||||||
self.assertRaises(Exception, message_decryptor.run, encrypted_message)
|
self.assertRaises(
|
||||||
|
MailDecryptionError,
|
||||||
|
message_decryptor.run,
|
||||||
|
encrypted_message,
|
||||||
|
)
|
||||||
finally:
|
finally:
|
||||||
# Clean up the temporary GPG home used only by this test
|
# Clean up the temporary GPG home used only by this test
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -1,4 +1,3 @@
|
|||||||
import datetime
|
|
||||||
import logging
|
import logging
|
||||||
from datetime import timedelta
|
from datetime import timedelta
|
||||||
from http import HTTPStatus
|
from http import HTTPStatus
|
||||||
@@ -87,7 +86,7 @@ class MailAccountViewSet(PassUserMixin, ModelViewSet[MailAccount]):
|
|||||||
@action(methods=["post"], detail=False)
|
@action(methods=["post"], detail=False)
|
||||||
def test(self, request):
|
def test(self, request):
|
||||||
logger = logging.getLogger("paperless_mail")
|
logger = logging.getLogger("paperless_mail")
|
||||||
request.data["name"] = datetime.datetime.now().isoformat()
|
request.data["name"] = timezone.now().isoformat()
|
||||||
serializer = self.get_serializer(data=request.data)
|
serializer = self.get_serializer(data=request.data)
|
||||||
serializer.is_valid(raise_exception=True)
|
serializer.is_valid(raise_exception=True)
|
||||||
existing_account = None
|
existing_account = None
|
||||||
|
|||||||
@@ -3053,7 +3053,7 @@ requires-dist = [
|
|||||||
{ name = "httpx-oauth", specifier = "~=0.17" },
|
{ name = "httpx-oauth", specifier = "~=0.17" },
|
||||||
{ name = "ijson", specifier = ">=3.5.1" },
|
{ name = "ijson", specifier = ">=3.5.1" },
|
||||||
{ name = "imap-tools", specifier = "~=1.14.0" },
|
{ name = "imap-tools", specifier = "~=1.14.0" },
|
||||||
{ name = "jinja2", specifier = "~=3.1.5" },
|
{ name = "jinja2", specifier = "~=3.1.6" },
|
||||||
{ name = "langdetect", specifier = "~=1.0.9" },
|
{ name = "langdetect", specifier = "~=1.0.9" },
|
||||||
{ name = "llama-index-core", specifier = ">=0.14.23" },
|
{ name = "llama-index-core", specifier = ">=0.14.23" },
|
||||||
{ name = "llama-index-embeddings-huggingface", specifier = ">=0.6.1" },
|
{ name = "llama-index-embeddings-huggingface", specifier = ">=0.6.1" },
|
||||||
|
|||||||
Reference in New Issue
Block a user