Compare commits

...
22 changed files with 1792 additions and 1451 deletions
+14 -19
View File
@@ -129,8 +129,8 @@ jobs:
~/.pnpm-store ~/.pnpm-store
~/.cache ~/.cache
key: ${{ runner.os }}-frontend-${{ hashFiles('src-ui/pnpm-lock.yaml') }} key: ${{ runner.os }}-frontend-${{ hashFiles('src-ui/pnpm-lock.yaml') }}
- name: Re-link Angular CLI - name: Install dependencies
run: cd src-ui && pnpm link @angular/cli run: cd src-ui && pnpm install --frozen-lockfile
- name: Run lint - name: Run lint
run: cd src-ui && pnpm run lint run: cd src-ui && pnpm run lint
unit-tests: unit-tests:
@@ -168,8 +168,8 @@ jobs:
~/.pnpm-store ~/.pnpm-store
~/.cache ~/.cache
key: ${{ runner.os }}-frontend-${{ hashFiles('src-ui/pnpm-lock.yaml') }} key: ${{ runner.os }}-frontend-${{ hashFiles('src-ui/pnpm-lock.yaml') }}
- name: Re-link Angular CLI - name: Install dependencies
run: cd src-ui && pnpm link @angular/cli run: cd src-ui && pnpm install --frozen-lockfile
- name: Run Jest unit tests - name: Run Jest unit tests
run: cd src-ui && pnpm run test --max-workers=2 --shard=${{ matrix.shard-index }}/${{ matrix.shard-count }} run: cd src-ui && pnpm run test --max-workers=2 --shard=${{ matrix.shard-index }}/${{ matrix.shard-count }}
- name: Upload test results to Codecov - name: Upload test results to Codecov
@@ -223,18 +223,15 @@ jobs:
~/.pnpm-store ~/.pnpm-store
~/.cache ~/.cache
key: ${{ runner.os }}-frontend-${{ hashFiles('src-ui/pnpm-lock.yaml') }} key: ${{ runner.os }}-frontend-${{ hashFiles('src-ui/pnpm-lock.yaml') }}
- name: Re-link Angular CLI
run: cd src-ui && pnpm link @angular/cli
- name: Install dependencies - name: Install dependencies
run: cd src-ui && pnpm install --no-frozen-lockfile run: cd src-ui && pnpm install --frozen-lockfile
- name: Run Playwright E2E tests - name: Run Playwright E2E tests
run: cd src-ui && pnpm exec playwright test --shard ${{ matrix.shard-index }}/${{ matrix.shard-count }} run: cd src-ui && pnpm exec playwright test --shard ${{ matrix.shard-index }}/${{ matrix.shard-count }}
bundle-analysis: frontend-build:
name: Bundle Analysis name: Frontend Build
needs: [changes, unit-tests, e2e-tests] needs: [changes, unit-tests, e2e-tests]
if: needs.changes.outputs.frontend_changed == 'true' if: needs.changes.outputs.frontend_changed == 'true'
runs-on: ubuntu-24.04 runs-on: ubuntu-24.04
environment: bundle-analysis
permissions: permissions:
contents: read contents: read
steps: steps:
@@ -260,21 +257,19 @@ jobs:
~/.pnpm-store ~/.pnpm-store
~/.cache ~/.cache
key: ${{ runner.os }}-frontend-${{ hashFiles('src-ui/pnpm-lock.yaml') }} key: ${{ runner.os }}-frontend-${{ hashFiles('src-ui/pnpm-lock.yaml') }}
- name: Re-link Angular CLI - name: Install dependencies
run: cd src-ui && pnpm link @angular/cli run: cd src-ui && pnpm install --frozen-lockfile
- name: Build and analyze - name: Build
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
run: cd src-ui && pnpm run build --configuration=production run: cd src-ui && pnpm run build --configuration=production
gate: gate:
name: Frontend CI Gate name: Frontend CI Gate
needs: [changes, install-dependencies, lint, unit-tests, e2e-tests, bundle-analysis] needs: [changes, install-dependencies, lint, unit-tests, e2e-tests, frontend-build]
if: always() if: always()
runs-on: ubuntu-slim runs-on: ubuntu-slim
steps: steps:
- name: Check gate - name: Check gate
env: env:
BUNDLE_ANALYSIS_RESULT: ${{ needs['bundle-analysis'].result }} BUILD_RESULT: ${{ needs['frontend-build'].result }}
E2E_RESULT: ${{ needs['e2e-tests'].result }} E2E_RESULT: ${{ needs['e2e-tests'].result }}
FRONTEND_CHANGED: ${{ needs.changes.outputs.frontend_changed }} FRONTEND_CHANGED: ${{ needs.changes.outputs.frontend_changed }}
INSTALL_RESULT: ${{ needs['install-dependencies'].result }} INSTALL_RESULT: ${{ needs['install-dependencies'].result }}
@@ -306,8 +301,8 @@ jobs:
exit 1 exit 1
fi fi
if [[ "${BUNDLE_ANALYSIS_RESULT}" != "success" ]]; then if [[ "${BUILD_RESULT}" != "success" ]]; then
echo "::error::Frontend bundle-analysis job result: ${BUNDLE_ANALYSIS_RESULT}" echo "::error::Frontend build job result: ${BUILD_RESULT}"
exit 1 exit 1
fi fi
+1 -4
View File
@@ -61,10 +61,7 @@ jobs:
~/.cache ~/.cache
key: ${{ runner.os }}-frontenddeps-${{ hashFiles('src-ui/pnpm-lock.yaml') }} key: ${{ runner.os }}-frontenddeps-${{ hashFiles('src-ui/pnpm-lock.yaml') }}
- name: Install frontend dependencies - name: Install frontend dependencies
if: steps.cache-frontend-deps.outputs.cache-hit != 'true' run: cd src-ui && pnpm install --frozen-lockfile
run: cd src-ui && pnpm install
- name: Re-link Angular cli
run: cd src-ui && pnpm link @angular/cli
- name: Generate frontend translation strings - name: Generate frontend translation strings
run: | run: |
cd src-ui cd src-ui
+12 -9
View File
@@ -56,13 +56,13 @@
}, },
"architect": { "architect": {
"build": { "build": {
"builder": "@angular-builders/custom-webpack:browser", "builder": "@angular/build:application",
"options": { "options": {
"customWebpackConfig": { "outputPath": {
"path": "./extra-webpack.config.ts" "base": "dist/paperless-ui",
"browser": ""
}, },
"outputPath": "dist/paperless-ui", "browser": "src/main.ts",
"main": "src/main.ts",
"outputHashing": "none", "outputHashing": "none",
"index": "src/index.html", "index": "src/index.html",
"polyfills": [ "polyfills": [
@@ -97,6 +97,7 @@
"scripts": [], "scripts": [],
"allowedCommonJsDependencies": [ "allowedCommonJsDependencies": [
"file-saver", "file-saver",
"mime-names",
"utif" "utif"
], ],
"extractLicenses": false, "extractLicenses": false,
@@ -117,11 +118,13 @@
"with": "src/environments/environment.prod.ts" "with": "src/environments/environment.prod.ts"
} }
], ],
"outputPath": "../src/documents/static/frontend/", "outputPath": {
"base": "../src/documents/static/frontend/",
"browser": ""
},
"optimization": true, "optimization": true,
"outputHashing": "none", "outputHashing": "none",
"sourceMap": false, "sourceMap": false,
"namedChunks": false,
"extractLicenses": true, "extractLicenses": true,
"budgets": [ "budgets": [
{ {
@@ -145,7 +148,7 @@
"defaultConfiguration": "" "defaultConfiguration": ""
}, },
"serve": { "serve": {
"builder": "@angular-builders/custom-webpack:dev-server", "builder": "@angular/build:dev-server",
"options": { "options": {
"buildTarget": "paperless-ui:build:en-US" "buildTarget": "paperless-ui:build:en-US"
}, },
@@ -156,7 +159,7 @@
} }
}, },
"extract-i18n": { "extract-i18n": {
"builder": "@angular-builders/custom-webpack:extract-i18n", "builder": "@angular/build:extract-i18n",
"options": { "options": {
"buildTarget": "paperless-ui:build" "buildTarget": "paperless-ui:build"
} }
@@ -1,4 +1,4 @@
import { expect, test } from '@playwright/test' import { expect, test, type WebSocketRoute } from '@playwright/test'
import path from 'node:path' import path from 'node:path'
const REQUESTS_HAR = path.join(__dirname, 'requests/api-document-detail.har') const REQUESTS_HAR = path.join(__dirname, 'requests/api-document-detail.har')
@@ -95,3 +95,60 @@ test('should support quick filters', async ({ page }) => {
.click() .click()
await expect(page).toHaveURL(/tags__id__all=4&sort=created&reverse=1&page=1/) await expect(page).toHaveURL(/tags__id__all=4&sort=created&reverse=1&page=1/)
}) })
test('should finish reloading the preview after a remote document update', async ({
page,
}) => {
let resolveStatusSocket: (socket: WebSocketRoute) => void
const statusSocketReady = new Promise<WebSocketRoute>((resolve) => {
resolveStatusSocket = resolve
})
await page.routeWebSocket(/\/ws\/status\/$/, (socket) => {
resolveStatusSocket(socket)
})
await page.routeFromHAR(REQUESTS_HAR, { notFound: 'fallback' })
let previewRequestCount = 0
page.on('request', (request) => {
if (request.url().includes('/api/documents/175/preview/')) {
previewRequestCount++
}
})
await page.goto('/documents/175/details')
await page.locator('pngx-document-detail').waitFor()
await expect(page.getByTitle('Storage path', { exact: true })).toHaveText(
/\w+/
)
const previewWasLoaded = await page.evaluate(() => {
const detail = document.querySelector('pngx-document-detail')
const component = (window as any).ng.getComponent(detail)
component.pdfPreviewLoaded({ numPages: 1 })
return component.previewLoaded()
})
expect(previewWasLoaded).toBe(true)
const previewRequestsBeforeReload = previewRequestCount
const statusSocket = await statusSocketReady
const documentReloaded = page.waitForResponse(
(response) =>
response.url().includes('/api/documents/175/?full_perms=true') &&
response.request().method() === 'GET'
)
statusSocket.send(
JSON.stringify({
type: 'document_updated',
data: {
document_id: 175,
modified: '2026-07-26T20:00:00Z',
},
})
)
await documentReloaded
await expect(
page.getByText('Document reloaded with latest changes.').first()
).toBeVisible()
await expect
.poll(() => previewRequestCount)
.toBeGreaterThan(previewRequestsBeforeReload + 1)
})
-24
View File
@@ -1,24 +0,0 @@
import {
CustomWebpackBrowserSchema,
TargetOptions,
} from '@angular-builders/custom-webpack'
import * as webpack from 'webpack'
const { codecovWebpackPlugin } = require('@codecov/webpack-plugin')
export default (
config: webpack.Configuration,
options: CustomWebpackBrowserSchema,
targetOptions: TargetOptions
) => {
if (config.plugins) {
config.plugins.push(
codecovWebpackPlugin({
enableBundleAnalysis: process.env.CODECOV_TOKEN !== undefined,
bundleName: 'paperless-ngx',
uploadToken: process.env.CODECOV_TOKEN,
})
)
}
return config
}
+3 -6
View File
@@ -39,7 +39,6 @@
"uuid": "^14.0.1" "uuid": "^14.0.1"
}, },
"devDependencies": { "devDependencies": {
"@angular-builders/custom-webpack": "^22.0.1",
"@angular-builders/jest": "^22.0.1", "@angular-builders/jest": "^22.0.1",
"@angular-devkit/core": "^22.0.5", "@angular-devkit/core": "^22.0.5",
"@angular-devkit/schematics": "^22.0.5", "@angular-devkit/schematics": "^22.0.5",
@@ -48,10 +47,9 @@
"@angular-eslint/eslint-plugin-template": "22.0.0", "@angular-eslint/eslint-plugin-template": "22.0.0",
"@angular-eslint/schematics": "22.0.0", "@angular-eslint/schematics": "22.0.0",
"@angular-eslint/template-parser": "22.0.0", "@angular-eslint/template-parser": "22.0.0",
"@angular/build": "^22.0.5", "@angular/build": "22.1.0-rc.0",
"@angular/cli": "~22.0.5", "@angular/cli": "22.1.0-rc.0",
"@angular/compiler-cli": "~22.0.5", "@angular/compiler-cli": "~22.0.5",
"@codecov/webpack-plugin": "^2.0.1",
"@playwright/test": "^1.61.1", "@playwright/test": "^1.61.1",
"@types/jest": "^30.0.0", "@types/jest": "^30.0.0",
"@types/node": "^26.0.0", "@types/node": "^26.0.0",
@@ -66,8 +64,7 @@
"jest-websocket-mock": "^2.5.0", "jest-websocket-mock": "^2.5.0",
"prettier-plugin-organize-imports": "^4.3.0", "prettier-plugin-organize-imports": "^4.3.0",
"ts-node": "~10.9.1", "ts-node": "~10.9.1",
"typescript": "^6.0.3", "typescript": "^6.0.3"
"webpack": "^5.107.2"
}, },
"packageManager": "pnpm@10.26.0" "packageManager": "pnpm@10.26.0"
} }
+1563 -1365
View File
File diff suppressed because it is too large Load Diff
@@ -283,6 +283,22 @@ describe('PngxPdfViewerComponent', () => {
expect(mockViewer.currentPageNumber).toBe(1) expect(mockViewer.currentPageNumber).toBe(1)
}) })
it('reloads when the source revision changes', () => {
const resetSpy = jest.spyOn(component as any, 'resetViewerState')
const loadSpy = jest
.spyOn(component as any, 'loadDocument')
.mockImplementation(() => {})
component.src = 'test.pdf'
component.sourceRevision = 1
component.ngOnChanges({
sourceRevision: new SimpleChange(0, 1, false),
})
expect(resetSpy).toHaveBeenCalled()
expect(loadSpy).toHaveBeenCalled()
})
it('applies viewer state after view init when already loaded', () => { it('applies viewer state after view init when already loaded', () => {
const applySpy = jest.spyOn(component as any, 'applyViewerState') const applySpy = jest.spyOn(component as any, 'applyViewerState')
;(component as any).hasLoaded = true ;(component as any).hasLoaded = true
@@ -43,6 +43,7 @@ export class PngxPdfViewerComponent
private readonly document = inject<Document>(DOCUMENT) private readonly document = inject<Document>(DOCUMENT)
@Input() src!: string @Input() src!: string
@Input() sourceRevision = 0
@Input() password?: string @Input() password?: string
@Input() page?: number @Input() page?: number
@Output() pageChange = new EventEmitter<number>() @Output() pageChange = new EventEmitter<number>()
@@ -93,7 +94,7 @@ export class PngxPdfViewerComponent
} }
ngOnChanges(changes: SimpleChanges): void { ngOnChanges(changes: SimpleChanges): void {
if (changes['src'] || changes['password']) { if (changes['src'] || changes['sourceRevision'] || changes['password']) {
this.resetViewerState() this.resetViewerState()
if (this.src) { if (this.src) {
this.loadDocument() this.loadDocument()
@@ -472,6 +472,7 @@
<div class="preview-sticky pdf-viewer-container"> <div class="preview-sticky pdf-viewer-container">
<pngx-pdf-viewer <pngx-pdf-viewer
[src]="pdfSource()" [src]="pdfSource()"
[sourceRevision]="previewRevision()"
[password]="pdfPassword()" [password]="pdfPassword()"
[renderMode]="PdfRenderMode.All" [renderMode]="PdfRenderMode.All"
[page]="previewCurrentPage()" (pageChange)="previewCurrentPage.set($event)" [page]="previewCurrentPage()" (pageChange)="previewCurrentPage.set($event)"
@@ -1602,6 +1602,7 @@ describe('DocumentDetailComponent', () => {
expect(openDoc.__changedFields).toEqual([]) expect(openDoc.__changedFields).toEqual([])
expect(setDirtySpy).toHaveBeenCalledWith(openDoc, false) expect(setDirtySpy).toHaveBeenCalledWith(openDoc, false)
expect(saveSpy).toHaveBeenCalled() expect(saveSpy).toHaveBeenCalled()
expect(component.previewRevision()).toBe(1)
}) })
it('should ignore incoming update for a different document id', () => { it('should ignore incoming update for a different document id', () => {
@@ -249,6 +249,7 @@ export class DocumentDetailComponent
titleSubject: Subject<string> = new Subject() titleSubject: Subject<string> = new Subject()
readonly previewUrl = signal<string>(undefined) readonly previewUrl = signal<string>(undefined)
readonly pdfSource = signal<string>(undefined) readonly pdfSource = signal<string>(undefined)
readonly previewRevision = signal(0)
readonly pdfPassword = signal<string>(undefined) readonly pdfPassword = signal<string>(undefined)
readonly thumbUrl = signal<string>(undefined) readonly thumbUrl = signal<string>(undefined)
readonly previewText = signal<string>(undefined) readonly previewText = signal<string>(undefined)
@@ -609,6 +610,9 @@ export class DocumentDetailComponent
.subscribe() .subscribe()
} }
this.updateComponent(useDoc) this.updateComponent(useDoc)
if (forceRemote) {
this.previewRevision.update((revision) => revision + 1)
}
this.titleSubject this.titleSubject
.pipe( .pipe(
debounceTime(1000), debounceTime(1000),
@@ -25,7 +25,7 @@
<input #textFilterInput class="form-control form-control-sm" type="text" <input #textFilterInput class="form-control form-control-sm" type="text"
[disabled]="textFilterModifierIsNull" [disabled]="textFilterModifierIsNull"
[(ngModel)]="textFilter" [(ngModel)]="textFilter"
(keyup)="textFilterKeyup($event)" (keydown)="textFilterKeydown($event)"
[ngbTypeahead]="searchAutoComplete" [ngbTypeahead]="searchAutoComplete"
(selectItem)="itemSelected($event)" (selectItem)="itemSelected($event)"
[readonly]="textFilterTarget === 'fulltext-morelike'"> [readonly]="textFilterTarget === 'fulltext-morelike'">
@@ -2180,17 +2180,17 @@ describe('FilterEditorComponent', () => {
it('should support Enter / Esc key on text field', () => { it('should support Enter / Esc key on text field', () => {
component.textFilterInput.nativeElement.value = 'foo' component.textFilterInput.nativeElement.value = 'foo'
component.textFilterInput.nativeElement.dispatchEvent( component.textFilterInput.nativeElement.dispatchEvent(
new KeyboardEvent('keyup', { key: 'Enter' }) new KeyboardEvent('keydown', { key: 'Enter' })
) )
expect(component.textFilter).toEqual('foo') expect(component.textFilter).toEqual('foo')
component.textFilterInput.nativeElement.value = 'foo bar' component.textFilterInput.nativeElement.value = 'foo bar'
component.textFilterInput.nativeElement.dispatchEvent( component.textFilterInput.nativeElement.dispatchEvent(
new KeyboardEvent('keyup', { key: 'Escape' }) new KeyboardEvent('keydown', { key: 'Escape' })
) )
expect(component.textFilter).toEqual('') expect(component.textFilter).toEqual('')
const blurSpy = jest.spyOn(component.textFilterInput.nativeElement, 'blur') const blurSpy = jest.spyOn(component.textFilterInput.nativeElement, 'blur')
component.textFilterInput.nativeElement.dispatchEvent( component.textFilterInput.nativeElement.dispatchEvent(
new KeyboardEvent('keyup', { key: 'Escape' }) new KeyboardEvent('keydown', { key: 'Escape' })
) )
expect(blurSpy).toHaveBeenCalled() expect(blurSpy).toHaveBeenCalled()
}) })
@@ -1312,7 +1312,7 @@ export class FilterEditorComponent
} }
} }
textFilterKeyup(event: KeyboardEvent) { textFilterKeydown(event: KeyboardEvent) {
if (event.key == 'Enter') { if (event.key == 'Enter') {
const filterString = ( const filterString = (
this.textFilterInput.nativeElement as HTMLInputElement this.textFilterInput.nativeElement as HTMLInputElement
@@ -108,6 +108,18 @@ describe('PermissionsService', () => {
actionKey: 'View', // PermissionAction.View actionKey: 'View', // PermissionAction.View
typeKey: 'Document', // PermissionType.Document typeKey: 'Document', // PermissionType.Document
}) })
expect(
permissionsService.getPermissionKeys('view_global_statistics')
).toEqual({
actionKey: 'View', // PermissionAction.View
typeKey: 'GlobalStatistics', // PermissionType.GlobalStatistics
})
expect(
permissionsService.getPermissionKeys('view_system_monitoring')
).toEqual({
actionKey: 'View', // PermissionAction.View
typeKey: 'SystemMonitoring', // PermissionType.SystemMonitoring
})
}) })
it('correctly checks explicit global permissions', () => { it('correctly checks explicit global permissions', () => {
+7 -10
View File
@@ -110,19 +110,16 @@ export class PermissionsService {
actionKey: string actionKey: string
typeKey: string typeKey: string
} { } {
const matches = permissionStr.match(/(.+)_/)
let typeKey let typeKey
let actionKey let actionKey
if (matches?.length > 0) { const actionIndex = Object.values(PermissionAction).findIndex((action) =>
const action = matches[1] permissionStr.startsWith(`${action}_`)
const actionIndex = Object.values(PermissionAction).indexOf( )
action as PermissionAction if (actionIndex > -1) {
) const action = Object.values(PermissionAction)[actionIndex]
if (actionIndex > -1) { actionKey = Object.keys(PermissionAction)[actionIndex]
actionKey = Object.keys(PermissionAction)[actionIndex]
}
const typeIndex = Object.values(PermissionType).indexOf( const typeIndex = Object.values(PermissionType).indexOf(
permissionStr.replace(action, '%s') as PermissionType permissionStr.replace(`${action}_`, '%s_') as PermissionType
) )
if (typeIndex > -1) { if (typeIndex > -1) {
typeKey = Object.keys(PermissionType)[typeIndex] typeKey = Object.keys(PermissionType)[typeIndex]
@@ -5,12 +5,30 @@ from django.db import migrations
from django.db import models from django.db import models
def clamp_applicationconfiguration_integer_fields(apps, schema_editor):
# Clamp barcode_dpi, barcode_max_pages, image_dpi and pages because of
# PositiveIntegerField --> PositiveSmallIntegerField
ApplicationConfiguration = apps.get_model("paperless", "ApplicationConfiguration")
ApplicationConfiguration.objects.filter(barcode_dpi__gt=32767).update(
barcode_dpi=32767,
)
ApplicationConfiguration.objects.filter(barcode_max_pages__gt=32767).update(
barcode_max_pages=32767,
)
ApplicationConfiguration.objects.filter(image_dpi__gt=32767).update(image_dpi=32767)
ApplicationConfiguration.objects.filter(pages__gt=32767).update(pages=32767)
class Migration(migrations.Migration): class Migration(migrations.Migration):
dependencies = [ dependencies = [
("paperless", "0006_applicationconfiguration_barcode_tag_split"), ("paperless", "0006_applicationconfiguration_barcode_tag_split"),
] ]
operations = [ operations = [
migrations.RunPython(
clamp_applicationconfiguration_integer_fields,
migrations.RunPython.noop,
),
migrations.AlterField( migrations.AlterField(
model_name="applicationconfiguration", model_name="applicationconfiguration",
name="barcode_dpi", name="barcode_dpi",
+2
View File
@@ -2,6 +2,8 @@ from pydantic import BaseModel
class DocumentClassifierSchema(BaseModel): class DocumentClassifierSchema(BaseModel):
"""Schema for document classification suggestions."""
title: str title: str
tags: list[str] tags: list[str]
correspondents: list[str] correspondents: list[str]
+17 -7
View File
@@ -5,6 +5,7 @@ from datetime import timedelta
from typing import TYPE_CHECKING from typing import TYPE_CHECKING
from django.conf import settings from django.conf import settings
from django.core.exceptions import ObjectDoesNotExist
from django.utils import timezone from django.utils import timezone
from filelock import FileLock from filelock import FileLock
from filelock import ReadWriteLock from filelock import ReadWriteLock
@@ -167,6 +168,19 @@ def write_store(embed_model_name: str | None = None):
yield store yield store
def _safe_related_name(document: Document, field: str) -> str | None:
"""
Returns the ``name`` of a related object (correspondent, document_type,
storage_path), or None if the FK is unset or points at a row that has
since been deleted (e.g. concurrently with this call).
"""
try:
related = getattr(document, field)
except ObjectDoesNotExist:
return None
return related.name if related else None
def build_document_node( def build_document_node(
document: Document, document: Document,
*, *,
@@ -180,14 +194,10 @@ def build_document_node(
"document_id": str(document.id), "document_id": str(document.id),
"title": document.title, "title": document.title,
"tags": [t.name for t in document.tags.all()], "tags": [t.name for t in document.tags.all()],
"correspondent": document.correspondent.name "correspondent": _safe_related_name(document, "correspondent"),
if document.correspondent "document_type": _safe_related_name(document, "document_type"),
else None,
"document_type": document.document_type.name
if document.document_type
else None,
"filename": document.filename, "filename": document.filename,
"storage_path": document.storage_path.name if document.storage_path else None, "storage_path": _safe_related_name(document, "storage_path"),
"archive_serial_number": document.archive_serial_number, "archive_serial_number": document.archive_serial_number,
"created": document.created.isoformat() if document.created else None, "created": document.created.isoformat() if document.created else None,
"added": document.added.isoformat() if document.added else None, "added": document.added.isoformat() if document.added else None,
@@ -8,7 +8,9 @@ from django.test import override_settings
from django.utils import timezone from django.utils import timezone
from llama_index.core.schema import MetadataMode from llama_index.core.schema import MetadataMode
from documents.models import Correspondent
from documents.models import Document from documents.models import Document
from documents.models import DocumentType
from documents.models import PaperlessTask from documents.models import PaperlessTask
from documents.signals import document_consumption_finished from documents.signals import document_consumption_finished
from documents.signals import document_updated from documents.signals import document_updated
@@ -95,6 +97,38 @@ def test_build_document_node_structured_fields_in_metadata(
assert "modified" in node.metadata assert "modified" in node.metadata
@pytest.mark.django_db
def test_build_document_node_survives_concurrently_deleted_correspondent(
real_document: Document,
) -> None:
"""Regression test for #13314.
If a document's correspondent (or document type) is deleted after the
in-memory Document instance was loaded but before build_document_node
resolves the relation, accessing the FK must not raise -- it should
behave like an unset FK and produce None in the metadata instead of
aborting the whole indexing pass.
"""
correspondent = Correspondent.objects.create(name="Stale Correspondent")
document_type = DocumentType.objects.create(name="Stale Type")
real_document.correspondent = correspondent
real_document.document_type = document_type
real_document.save()
# Re-fetch to get an instance whose correspondent/document_type relations
# are unresolved (not yet cached), mirroring a task that loaded the
# document before the concurrent deletion below.
stale_document = Document.objects.get(pk=real_document.pk)
correspondent.delete()
document_type.delete()
nodes = indexing.build_document_node(stale_document)
assert len(nodes) > 0
assert nodes[0].metadata["correspondent"] is None
assert nodes[0].metadata["document_type"] is None
@pytest.mark.django_db @pytest.mark.django_db
def test_build_document_node_excludes_document_id_from_llm_context( def test_build_document_node_excludes_document_id_from_llm_context(
real_document: Document, real_document: Document,
@@ -10,6 +10,24 @@ def clamp_mailrule_maximum_age(apps, schema_editor):
MailRule.objects.filter(maximum_age__gt=32767).update(maximum_age=32767) MailRule.objects.filter(maximum_age__gt=32767).update(maximum_age=32767)
def clamp_mailrule_order(apps, schema_editor):
# The order field of MailRule is only used for relative sorting
# (account.rules.order_by("order")), so out-of-range values must be
# renumbered by rank rather than clamped to a single value, which
# would collapse distinct rules to the same order.
MailRule = apps.get_model("paperless_mail", "MailRule")
if (
MailRule.objects.filter(order__gt=32767).exists()
or MailRule.objects.filter(
order__lt=-32768,
).exists()
): # pragma: no cover
for index, rule_id in enumerate(
MailRule.objects.order_by("order", "pk").values_list("pk", flat=True),
):
MailRule.objects.filter(pk=rule_id).update(order=index)
class Migration(migrations.Migration): class Migration(migrations.Migration):
# The data update must commit before PostgreSQL rewrites the table, otherwise # The data update must commit before PostgreSQL rewrites the table, otherwise
# pending foreign-key trigger events can block the subsequent ALTER TABLE. # pending foreign-key trigger events can block the subsequent ALTER TABLE.
@@ -135,6 +153,10 @@ class Migration(migrations.Migration):
verbose_name="maximum age", verbose_name="maximum age",
), ),
), ),
migrations.RunPython(
clamp_mailrule_order,
migrations.RunPython.noop,
),
migrations.AlterField( migrations.AlterField(
model_name="mailrule", model_name="mailrule",
name="order", name="order",