mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-07-27 06:14:54 +00:00
Compare commits
9
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ed27ca9222 | ||
|
|
b57eb949cc | ||
|
|
317e534aa0 | ||
|
|
3ba6d0325a | ||
|
|
318520a0f1 | ||
|
|
ea1f3653a9 | ||
|
|
423d4b9f2d | ||
|
|
e4bc8d721c | ||
|
|
f549e1818f |
@@ -129,8 +129,8 @@ jobs:
|
||||
~/.pnpm-store
|
||||
~/.cache
|
||||
key: ${{ runner.os }}-frontend-${{ hashFiles('src-ui/pnpm-lock.yaml') }}
|
||||
- name: Re-link Angular CLI
|
||||
run: cd src-ui && pnpm link @angular/cli
|
||||
- name: Install dependencies
|
||||
run: cd src-ui && pnpm install --frozen-lockfile
|
||||
- name: Run lint
|
||||
run: cd src-ui && pnpm run lint
|
||||
unit-tests:
|
||||
@@ -168,8 +168,8 @@ jobs:
|
||||
~/.pnpm-store
|
||||
~/.cache
|
||||
key: ${{ runner.os }}-frontend-${{ hashFiles('src-ui/pnpm-lock.yaml') }}
|
||||
- name: Re-link Angular CLI
|
||||
run: cd src-ui && pnpm link @angular/cli
|
||||
- name: Install dependencies
|
||||
run: cd src-ui && pnpm install --frozen-lockfile
|
||||
- name: Run Jest unit tests
|
||||
run: cd src-ui && pnpm run test --max-workers=2 --shard=${{ matrix.shard-index }}/${{ matrix.shard-count }}
|
||||
- name: Upload test results to Codecov
|
||||
@@ -223,18 +223,15 @@ jobs:
|
||||
~/.pnpm-store
|
||||
~/.cache
|
||||
key: ${{ runner.os }}-frontend-${{ hashFiles('src-ui/pnpm-lock.yaml') }}
|
||||
- name: Re-link Angular CLI
|
||||
run: cd src-ui && pnpm link @angular/cli
|
||||
- name: Install dependencies
|
||||
run: cd src-ui && pnpm install --no-frozen-lockfile
|
||||
run: cd src-ui && pnpm install --frozen-lockfile
|
||||
- name: Run Playwright E2E tests
|
||||
run: cd src-ui && pnpm exec playwright test --shard ${{ matrix.shard-index }}/${{ matrix.shard-count }}
|
||||
bundle-analysis:
|
||||
name: Bundle Analysis
|
||||
frontend-build:
|
||||
name: Frontend Build
|
||||
needs: [changes, unit-tests, e2e-tests]
|
||||
if: needs.changes.outputs.frontend_changed == 'true'
|
||||
runs-on: ubuntu-24.04
|
||||
environment: bundle-analysis
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
@@ -260,21 +257,19 @@ jobs:
|
||||
~/.pnpm-store
|
||||
~/.cache
|
||||
key: ${{ runner.os }}-frontend-${{ hashFiles('src-ui/pnpm-lock.yaml') }}
|
||||
- name: Re-link Angular CLI
|
||||
run: cd src-ui && pnpm link @angular/cli
|
||||
- name: Build and analyze
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
- name: Install dependencies
|
||||
run: cd src-ui && pnpm install --frozen-lockfile
|
||||
- name: Build
|
||||
run: cd src-ui && pnpm run build --configuration=production
|
||||
gate:
|
||||
name: Frontend CI Gate
|
||||
needs: [changes, install-dependencies, lint, unit-tests, e2e-tests, bundle-analysis]
|
||||
needs: [changes, install-dependencies, lint, unit-tests, e2e-tests, frontend-build]
|
||||
if: always()
|
||||
runs-on: ubuntu-slim
|
||||
steps:
|
||||
- name: Check gate
|
||||
env:
|
||||
BUNDLE_ANALYSIS_RESULT: ${{ needs['bundle-analysis'].result }}
|
||||
BUILD_RESULT: ${{ needs['frontend-build'].result }}
|
||||
E2E_RESULT: ${{ needs['e2e-tests'].result }}
|
||||
FRONTEND_CHANGED: ${{ needs.changes.outputs.frontend_changed }}
|
||||
INSTALL_RESULT: ${{ needs['install-dependencies'].result }}
|
||||
@@ -306,8 +301,8 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "${BUNDLE_ANALYSIS_RESULT}" != "success" ]]; then
|
||||
echo "::error::Frontend bundle-analysis job result: ${BUNDLE_ANALYSIS_RESULT}"
|
||||
if [[ "${BUILD_RESULT}" != "success" ]]; then
|
||||
echo "::error::Frontend build job result: ${BUILD_RESULT}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
@@ -61,10 +61,7 @@ jobs:
|
||||
~/.cache
|
||||
key: ${{ runner.os }}-frontenddeps-${{ hashFiles('src-ui/pnpm-lock.yaml') }}
|
||||
- name: Install frontend dependencies
|
||||
if: steps.cache-frontend-deps.outputs.cache-hit != 'true'
|
||||
run: cd src-ui && pnpm install
|
||||
- name: Re-link Angular cli
|
||||
run: cd src-ui && pnpm link @angular/cli
|
||||
run: cd src-ui && pnpm install --frozen-lockfile
|
||||
- name: Generate frontend translation strings
|
||||
run: |
|
||||
cd src-ui
|
||||
|
||||
+12
-9
@@ -56,13 +56,13 @@
|
||||
},
|
||||
"architect": {
|
||||
"build": {
|
||||
"builder": "@angular-builders/custom-webpack:browser",
|
||||
"builder": "@angular/build:application",
|
||||
"options": {
|
||||
"customWebpackConfig": {
|
||||
"path": "./extra-webpack.config.ts"
|
||||
"outputPath": {
|
||||
"base": "dist/paperless-ui",
|
||||
"browser": ""
|
||||
},
|
||||
"outputPath": "dist/paperless-ui",
|
||||
"main": "src/main.ts",
|
||||
"browser": "src/main.ts",
|
||||
"outputHashing": "none",
|
||||
"index": "src/index.html",
|
||||
"polyfills": [
|
||||
@@ -97,6 +97,7 @@
|
||||
"scripts": [],
|
||||
"allowedCommonJsDependencies": [
|
||||
"file-saver",
|
||||
"mime-names",
|
||||
"utif"
|
||||
],
|
||||
"extractLicenses": false,
|
||||
@@ -117,11 +118,13 @@
|
||||
"with": "src/environments/environment.prod.ts"
|
||||
}
|
||||
],
|
||||
"outputPath": "../src/documents/static/frontend/",
|
||||
"outputPath": {
|
||||
"base": "../src/documents/static/frontend/",
|
||||
"browser": ""
|
||||
},
|
||||
"optimization": true,
|
||||
"outputHashing": "none",
|
||||
"sourceMap": false,
|
||||
"namedChunks": false,
|
||||
"extractLicenses": true,
|
||||
"budgets": [
|
||||
{
|
||||
@@ -145,7 +148,7 @@
|
||||
"defaultConfiguration": ""
|
||||
},
|
||||
"serve": {
|
||||
"builder": "@angular-builders/custom-webpack:dev-server",
|
||||
"builder": "@angular/build:dev-server",
|
||||
"options": {
|
||||
"buildTarget": "paperless-ui:build:en-US"
|
||||
},
|
||||
@@ -156,7 +159,7 @@
|
||||
}
|
||||
},
|
||||
"extract-i18n": {
|
||||
"builder": "@angular-builders/custom-webpack:extract-i18n",
|
||||
"builder": "@angular/build:extract-i18n",
|
||||
"options": {
|
||||
"buildTarget": "paperless-ui:build"
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { expect, test } from '@playwright/test'
|
||||
import { expect, test, type WebSocketRoute } from '@playwright/test'
|
||||
import path from 'node:path'
|
||||
|
||||
const REQUESTS_HAR = path.join(__dirname, 'requests/api-document-detail.har')
|
||||
@@ -95,3 +95,60 @@ test('should support quick filters', async ({ page }) => {
|
||||
.click()
|
||||
await expect(page).toHaveURL(/tags__id__all=4&sort=created&reverse=1&page=1/)
|
||||
})
|
||||
|
||||
test('should finish reloading the preview after a remote document update', async ({
|
||||
page,
|
||||
}) => {
|
||||
let resolveStatusSocket: (socket: WebSocketRoute) => void
|
||||
const statusSocketReady = new Promise<WebSocketRoute>((resolve) => {
|
||||
resolveStatusSocket = resolve
|
||||
})
|
||||
await page.routeWebSocket(/\/ws\/status\/$/, (socket) => {
|
||||
resolveStatusSocket(socket)
|
||||
})
|
||||
await page.routeFromHAR(REQUESTS_HAR, { notFound: 'fallback' })
|
||||
let previewRequestCount = 0
|
||||
page.on('request', (request) => {
|
||||
if (request.url().includes('/api/documents/175/preview/')) {
|
||||
previewRequestCount++
|
||||
}
|
||||
})
|
||||
await page.goto('/documents/175/details')
|
||||
|
||||
await page.locator('pngx-document-detail').waitFor()
|
||||
await expect(page.getByTitle('Storage path', { exact: true })).toHaveText(
|
||||
/\w+/
|
||||
)
|
||||
const previewWasLoaded = await page.evaluate(() => {
|
||||
const detail = document.querySelector('pngx-document-detail')
|
||||
const component = (window as any).ng.getComponent(detail)
|
||||
component.pdfPreviewLoaded({ numPages: 1 })
|
||||
return component.previewLoaded()
|
||||
})
|
||||
expect(previewWasLoaded).toBe(true)
|
||||
const previewRequestsBeforeReload = previewRequestCount
|
||||
|
||||
const statusSocket = await statusSocketReady
|
||||
const documentReloaded = page.waitForResponse(
|
||||
(response) =>
|
||||
response.url().includes('/api/documents/175/?full_perms=true') &&
|
||||
response.request().method() === 'GET'
|
||||
)
|
||||
statusSocket.send(
|
||||
JSON.stringify({
|
||||
type: 'document_updated',
|
||||
data: {
|
||||
document_id: 175,
|
||||
modified: '2026-07-26T20:00:00Z',
|
||||
},
|
||||
})
|
||||
)
|
||||
await documentReloaded
|
||||
|
||||
await expect(
|
||||
page.getByText('Document reloaded with latest changes.').first()
|
||||
).toBeVisible()
|
||||
await expect
|
||||
.poll(() => previewRequestCount)
|
||||
.toBeGreaterThan(previewRequestsBeforeReload + 1)
|
||||
})
|
||||
|
||||
@@ -1,24 +0,0 @@
|
||||
import {
|
||||
CustomWebpackBrowserSchema,
|
||||
TargetOptions,
|
||||
} from '@angular-builders/custom-webpack'
|
||||
import * as webpack from 'webpack'
|
||||
const { codecovWebpackPlugin } = require('@codecov/webpack-plugin')
|
||||
|
||||
export default (
|
||||
config: webpack.Configuration,
|
||||
options: CustomWebpackBrowserSchema,
|
||||
targetOptions: TargetOptions
|
||||
) => {
|
||||
if (config.plugins) {
|
||||
config.plugins.push(
|
||||
codecovWebpackPlugin({
|
||||
enableBundleAnalysis: process.env.CODECOV_TOKEN !== undefined,
|
||||
bundleName: 'paperless-ngx',
|
||||
uploadToken: process.env.CODECOV_TOKEN,
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
return config
|
||||
}
|
||||
+3
-6
@@ -39,7 +39,6 @@
|
||||
"uuid": "^14.0.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@angular-builders/custom-webpack": "^22.0.1",
|
||||
"@angular-builders/jest": "^22.0.1",
|
||||
"@angular-devkit/core": "^22.0.5",
|
||||
"@angular-devkit/schematics": "^22.0.5",
|
||||
@@ -48,10 +47,9 @@
|
||||
"@angular-eslint/eslint-plugin-template": "22.0.0",
|
||||
"@angular-eslint/schematics": "22.0.0",
|
||||
"@angular-eslint/template-parser": "22.0.0",
|
||||
"@angular/build": "^22.0.5",
|
||||
"@angular/cli": "~22.0.5",
|
||||
"@angular/build": "22.1.0-rc.0",
|
||||
"@angular/cli": "22.1.0-rc.0",
|
||||
"@angular/compiler-cli": "~22.0.5",
|
||||
"@codecov/webpack-plugin": "^2.0.1",
|
||||
"@playwright/test": "^1.61.1",
|
||||
"@types/jest": "^30.0.0",
|
||||
"@types/node": "^26.0.0",
|
||||
@@ -66,8 +64,7 @@
|
||||
"jest-websocket-mock": "^2.5.0",
|
||||
"prettier-plugin-organize-imports": "^4.3.0",
|
||||
"ts-node": "~10.9.1",
|
||||
"typescript": "^6.0.3",
|
||||
"webpack": "^5.107.2"
|
||||
"typescript": "^6.0.3"
|
||||
},
|
||||
"packageManager": "pnpm@10.26.0"
|
||||
}
|
||||
|
||||
Generated
+1563
-1365
File diff suppressed because it is too large
Load Diff
@@ -283,6 +283,22 @@ describe('PngxPdfViewerComponent', () => {
|
||||
expect(mockViewer.currentPageNumber).toBe(1)
|
||||
})
|
||||
|
||||
it('reloads when the source revision changes', () => {
|
||||
const resetSpy = jest.spyOn(component as any, 'resetViewerState')
|
||||
const loadSpy = jest
|
||||
.spyOn(component as any, 'loadDocument')
|
||||
.mockImplementation(() => {})
|
||||
component.src = 'test.pdf'
|
||||
component.sourceRevision = 1
|
||||
|
||||
component.ngOnChanges({
|
||||
sourceRevision: new SimpleChange(0, 1, false),
|
||||
})
|
||||
|
||||
expect(resetSpy).toHaveBeenCalled()
|
||||
expect(loadSpy).toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('applies viewer state after view init when already loaded', () => {
|
||||
const applySpy = jest.spyOn(component as any, 'applyViewerState')
|
||||
;(component as any).hasLoaded = true
|
||||
|
||||
@@ -43,6 +43,7 @@ export class PngxPdfViewerComponent
|
||||
private readonly document = inject<Document>(DOCUMENT)
|
||||
|
||||
@Input() src!: string
|
||||
@Input() sourceRevision = 0
|
||||
@Input() password?: string
|
||||
@Input() page?: number
|
||||
@Output() pageChange = new EventEmitter<number>()
|
||||
@@ -93,7 +94,7 @@ export class PngxPdfViewerComponent
|
||||
}
|
||||
|
||||
ngOnChanges(changes: SimpleChanges): void {
|
||||
if (changes['src'] || changes['password']) {
|
||||
if (changes['src'] || changes['sourceRevision'] || changes['password']) {
|
||||
this.resetViewerState()
|
||||
if (this.src) {
|
||||
this.loadDocument()
|
||||
|
||||
@@ -472,6 +472,7 @@
|
||||
<div class="preview-sticky pdf-viewer-container">
|
||||
<pngx-pdf-viewer
|
||||
[src]="pdfSource()"
|
||||
[sourceRevision]="previewRevision()"
|
||||
[password]="pdfPassword()"
|
||||
[renderMode]="PdfRenderMode.All"
|
||||
[page]="previewCurrentPage()" (pageChange)="previewCurrentPage.set($event)"
|
||||
|
||||
@@ -1602,6 +1602,7 @@ describe('DocumentDetailComponent', () => {
|
||||
expect(openDoc.__changedFields).toEqual([])
|
||||
expect(setDirtySpy).toHaveBeenCalledWith(openDoc, false)
|
||||
expect(saveSpy).toHaveBeenCalled()
|
||||
expect(component.previewRevision()).toBe(1)
|
||||
})
|
||||
|
||||
it('should ignore incoming update for a different document id', () => {
|
||||
|
||||
@@ -249,6 +249,7 @@ export class DocumentDetailComponent
|
||||
titleSubject: Subject<string> = new Subject()
|
||||
readonly previewUrl = signal<string>(undefined)
|
||||
readonly pdfSource = signal<string>(undefined)
|
||||
readonly previewRevision = signal(0)
|
||||
readonly pdfPassword = signal<string>(undefined)
|
||||
readonly thumbUrl = signal<string>(undefined)
|
||||
readonly previewText = signal<string>(undefined)
|
||||
@@ -609,6 +610,9 @@ export class DocumentDetailComponent
|
||||
.subscribe()
|
||||
}
|
||||
this.updateComponent(useDoc)
|
||||
if (forceRemote) {
|
||||
this.previewRevision.update((revision) => revision + 1)
|
||||
}
|
||||
this.titleSubject
|
||||
.pipe(
|
||||
debounceTime(1000),
|
||||
|
||||
@@ -25,7 +25,7 @@
|
||||
<input #textFilterInput class="form-control form-control-sm" type="text"
|
||||
[disabled]="textFilterModifierIsNull"
|
||||
[(ngModel)]="textFilter"
|
||||
(keyup)="textFilterKeyup($event)"
|
||||
(keydown)="textFilterKeydown($event)"
|
||||
[ngbTypeahead]="searchAutoComplete"
|
||||
(selectItem)="itemSelected($event)"
|
||||
[readonly]="textFilterTarget === 'fulltext-morelike'">
|
||||
|
||||
+3
-3
@@ -2180,17 +2180,17 @@ describe('FilterEditorComponent', () => {
|
||||
it('should support Enter / Esc key on text field', () => {
|
||||
component.textFilterInput.nativeElement.value = 'foo'
|
||||
component.textFilterInput.nativeElement.dispatchEvent(
|
||||
new KeyboardEvent('keyup', { key: 'Enter' })
|
||||
new KeyboardEvent('keydown', { key: 'Enter' })
|
||||
)
|
||||
expect(component.textFilter).toEqual('foo')
|
||||
component.textFilterInput.nativeElement.value = 'foo bar'
|
||||
component.textFilterInput.nativeElement.dispatchEvent(
|
||||
new KeyboardEvent('keyup', { key: 'Escape' })
|
||||
new KeyboardEvent('keydown', { key: 'Escape' })
|
||||
)
|
||||
expect(component.textFilter).toEqual('')
|
||||
const blurSpy = jest.spyOn(component.textFilterInput.nativeElement, 'blur')
|
||||
component.textFilterInput.nativeElement.dispatchEvent(
|
||||
new KeyboardEvent('keyup', { key: 'Escape' })
|
||||
new KeyboardEvent('keydown', { key: 'Escape' })
|
||||
)
|
||||
expect(blurSpy).toHaveBeenCalled()
|
||||
})
|
||||
|
||||
@@ -1312,7 +1312,7 @@ export class FilterEditorComponent
|
||||
}
|
||||
}
|
||||
|
||||
textFilterKeyup(event: KeyboardEvent) {
|
||||
textFilterKeydown(event: KeyboardEvent) {
|
||||
if (event.key == 'Enter') {
|
||||
const filterString = (
|
||||
this.textFilterInput.nativeElement as HTMLInputElement
|
||||
|
||||
@@ -172,39 +172,3 @@ class TestApiUiSettings(DirectoriesMixin, APITestCase):
|
||||
self.assertIsNotNone(
|
||||
response.data["settings"]["outlook_oauth_url"],
|
||||
)
|
||||
|
||||
@override_settings(
|
||||
OAUTH_CALLBACK_BASE_URL="http://localhost:8000",
|
||||
GMAIL_OAUTH_CLIENT_ID="abc123",
|
||||
GMAIL_OAUTH_CLIENT_SECRET="def456",
|
||||
GMAIL_OAUTH_ENABLED=True,
|
||||
OUTLOOK_OAUTH_CLIENT_ID="ghi789",
|
||||
OUTLOOK_OAUTH_CLIENT_SECRET="jkl012",
|
||||
OUTLOOK_OAUTH_ENABLED=True,
|
||||
)
|
||||
def test_settings_oauth_state_reused_across_calls(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A mail account oauth flow has been started, storing a state in the session
|
||||
WHEN:
|
||||
- The ui_settings endpoint is called again before the flow completes
|
||||
THEN:
|
||||
- The same oauth state is reused, not replaced with a new one
|
||||
"""
|
||||
response = self.client.get(self.ENDPOINT, format="json")
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
first_state = self.client.session["oauth_state"]
|
||||
first_gmail_url = response.data["settings"]["gmail_oauth_url"]
|
||||
first_outlook_url = response.data["settings"]["outlook_oauth_url"]
|
||||
|
||||
response = self.client.get(self.ENDPOINT, format="json")
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
self.assertEqual(self.client.session["oauth_state"], first_state)
|
||||
self.assertEqual(
|
||||
response.data["settings"]["gmail_oauth_url"],
|
||||
first_gmail_url,
|
||||
)
|
||||
self.assertEqual(
|
||||
response.data["settings"]["outlook_oauth_url"],
|
||||
first_outlook_url,
|
||||
)
|
||||
|
||||
@@ -4002,19 +4002,15 @@ class UiSettingsView(GenericAPIView[Any]):
|
||||
ui_settings["auditlog_enabled"] = settings.AUDIT_LOG_ENABLED
|
||||
|
||||
if settings.GMAIL_OAUTH_ENABLED or settings.OUTLOOK_OAUTH_ENABLED:
|
||||
# Reuse an in-flight state instead of minting a new one on every
|
||||
# settings fetch, or a concurrent request can invalidate a login
|
||||
# that's still in progress.
|
||||
manager = PaperlessMailOAuth2Manager(
|
||||
state=request.session.get("oauth_state"),
|
||||
)
|
||||
request.session["oauth_state"] = manager.state
|
||||
manager = PaperlessMailOAuth2Manager()
|
||||
if settings.GMAIL_OAUTH_ENABLED:
|
||||
ui_settings["gmail_oauth_url"] = manager.get_gmail_authorization_url()
|
||||
request.session["oauth_state"] = manager.state
|
||||
if settings.OUTLOOK_OAUTH_ENABLED:
|
||||
ui_settings["outlook_oauth_url"] = (
|
||||
manager.get_outlook_authorization_url()
|
||||
)
|
||||
request.session["oauth_state"] = manager.state
|
||||
|
||||
ui_settings["email_enabled"] = settings.EMAIL_ENABLED
|
||||
|
||||
|
||||
@@ -5,12 +5,30 @@ from django.db import migrations
|
||||
from django.db import models
|
||||
|
||||
|
||||
def clamp_applicationconfiguration_integer_fields(apps, schema_editor):
|
||||
# Clamp barcode_dpi, barcode_max_pages, image_dpi and pages because of
|
||||
# PositiveIntegerField --> PositiveSmallIntegerField
|
||||
ApplicationConfiguration = apps.get_model("paperless", "ApplicationConfiguration")
|
||||
ApplicationConfiguration.objects.filter(barcode_dpi__gt=32767).update(
|
||||
barcode_dpi=32767,
|
||||
)
|
||||
ApplicationConfiguration.objects.filter(barcode_max_pages__gt=32767).update(
|
||||
barcode_max_pages=32767,
|
||||
)
|
||||
ApplicationConfiguration.objects.filter(image_dpi__gt=32767).update(image_dpi=32767)
|
||||
ApplicationConfiguration.objects.filter(pages__gt=32767).update(pages=32767)
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("paperless", "0006_applicationconfiguration_barcode_tag_split"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RunPython(
|
||||
clamp_applicationconfiguration_integer_fields,
|
||||
migrations.RunPython.noop,
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name="applicationconfiguration",
|
||||
name="barcode_dpi",
|
||||
|
||||
@@ -2,6 +2,8 @@ from pydantic import BaseModel
|
||||
|
||||
|
||||
class DocumentClassifierSchema(BaseModel):
|
||||
"""Schema for document classification suggestions."""
|
||||
|
||||
title: str
|
||||
tags: list[str]
|
||||
correspondents: list[str]
|
||||
|
||||
@@ -5,6 +5,7 @@ from datetime import timedelta
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.exceptions import ObjectDoesNotExist
|
||||
from django.utils import timezone
|
||||
from filelock import FileLock
|
||||
from filelock import ReadWriteLock
|
||||
@@ -167,6 +168,19 @@ def write_store(embed_model_name: str | None = None):
|
||||
yield store
|
||||
|
||||
|
||||
def _safe_related_name(document: Document, field: str) -> str | None:
|
||||
"""
|
||||
Returns the ``name`` of a related object (correspondent, document_type,
|
||||
storage_path), or None if the FK is unset or points at a row that has
|
||||
since been deleted (e.g. concurrently with this call).
|
||||
"""
|
||||
try:
|
||||
related = getattr(document, field)
|
||||
except ObjectDoesNotExist:
|
||||
return None
|
||||
return related.name if related else None
|
||||
|
||||
|
||||
def build_document_node(
|
||||
document: Document,
|
||||
*,
|
||||
@@ -180,14 +194,10 @@ def build_document_node(
|
||||
"document_id": str(document.id),
|
||||
"title": document.title,
|
||||
"tags": [t.name for t in document.tags.all()],
|
||||
"correspondent": document.correspondent.name
|
||||
if document.correspondent
|
||||
else None,
|
||||
"document_type": document.document_type.name
|
||||
if document.document_type
|
||||
else None,
|
||||
"correspondent": _safe_related_name(document, "correspondent"),
|
||||
"document_type": _safe_related_name(document, "document_type"),
|
||||
"filename": document.filename,
|
||||
"storage_path": document.storage_path.name if document.storage_path else None,
|
||||
"storage_path": _safe_related_name(document, "storage_path"),
|
||||
"archive_serial_number": document.archive_serial_number,
|
||||
"created": document.created.isoformat() if document.created else None,
|
||||
"added": document.added.isoformat() if document.added else None,
|
||||
|
||||
@@ -8,7 +8,9 @@ from django.test import override_settings
|
||||
from django.utils import timezone
|
||||
from llama_index.core.schema import MetadataMode
|
||||
|
||||
from documents.models import Correspondent
|
||||
from documents.models import Document
|
||||
from documents.models import DocumentType
|
||||
from documents.models import PaperlessTask
|
||||
from documents.signals import document_consumption_finished
|
||||
from documents.signals import document_updated
|
||||
@@ -95,6 +97,38 @@ def test_build_document_node_structured_fields_in_metadata(
|
||||
assert "modified" in node.metadata
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_build_document_node_survives_concurrently_deleted_correspondent(
|
||||
real_document: Document,
|
||||
) -> None:
|
||||
"""Regression test for #13314.
|
||||
|
||||
If a document's correspondent (or document type) is deleted after the
|
||||
in-memory Document instance was loaded but before build_document_node
|
||||
resolves the relation, accessing the FK must not raise -- it should
|
||||
behave like an unset FK and produce None in the metadata instead of
|
||||
aborting the whole indexing pass.
|
||||
"""
|
||||
correspondent = Correspondent.objects.create(name="Stale Correspondent")
|
||||
document_type = DocumentType.objects.create(name="Stale Type")
|
||||
real_document.correspondent = correspondent
|
||||
real_document.document_type = document_type
|
||||
real_document.save()
|
||||
|
||||
# Re-fetch to get an instance whose correspondent/document_type relations
|
||||
# are unresolved (not yet cached), mirroring a task that loaded the
|
||||
# document before the concurrent deletion below.
|
||||
stale_document = Document.objects.get(pk=real_document.pk)
|
||||
|
||||
correspondent.delete()
|
||||
document_type.delete()
|
||||
|
||||
nodes = indexing.build_document_node(stale_document)
|
||||
assert len(nodes) > 0
|
||||
assert nodes[0].metadata["correspondent"] is None
|
||||
assert nodes[0].metadata["document_type"] is None
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_build_document_node_excludes_document_id_from_llm_context(
|
||||
real_document: Document,
|
||||
|
||||
@@ -10,6 +10,24 @@ def clamp_mailrule_maximum_age(apps, schema_editor):
|
||||
MailRule.objects.filter(maximum_age__gt=32767).update(maximum_age=32767)
|
||||
|
||||
|
||||
def clamp_mailrule_order(apps, schema_editor):
|
||||
# The order field of MailRule is only used for relative sorting
|
||||
# (account.rules.order_by("order")), so out-of-range values must be
|
||||
# renumbered by rank rather than clamped to a single value, which
|
||||
# would collapse distinct rules to the same order.
|
||||
MailRule = apps.get_model("paperless_mail", "MailRule")
|
||||
if (
|
||||
MailRule.objects.filter(order__gt=32767).exists()
|
||||
or MailRule.objects.filter(
|
||||
order__lt=-32768,
|
||||
).exists()
|
||||
): # pragma: no cover
|
||||
for index, rule_id in enumerate(
|
||||
MailRule.objects.order_by("order", "pk").values_list("pk", flat=True),
|
||||
):
|
||||
MailRule.objects.filter(pk=rule_id).update(order=index)
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
# The data update must commit before PostgreSQL rewrites the table, otherwise
|
||||
# pending foreign-key trigger events can block the subsequent ALTER TABLE.
|
||||
@@ -135,6 +153,10 @@ class Migration(migrations.Migration):
|
||||
verbose_name="maximum age",
|
||||
),
|
||||
),
|
||||
migrations.RunPython(
|
||||
clamp_mailrule_order,
|
||||
migrations.RunPython.noop,
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name="mailrule",
|
||||
name="order",
|
||||
|
||||
@@ -138,16 +138,6 @@ class TestMailOAuth(
|
||||
MailAccount.objects.filter(imap_server="imap.gmail.com").exists(),
|
||||
)
|
||||
|
||||
# State is single-use and was cleared by the callback above, so a new
|
||||
# flow needs a new state stored in the session.
|
||||
session = self.client.session
|
||||
session.update(
|
||||
{
|
||||
"oauth_state": "test_state",
|
||||
},
|
||||
)
|
||||
session.save()
|
||||
|
||||
# Test Outlook OAuth callback
|
||||
response = self.client.get(
|
||||
"/api/oauth/callback/?code=test_code&state=test_state",
|
||||
@@ -190,16 +180,6 @@ class TestMailOAuth(
|
||||
MailAccount.objects.filter(imap_server="imap.gmail.com").exists(),
|
||||
)
|
||||
|
||||
# State is single-use and was cleared by the callback above, so a
|
||||
# new flow needs a new state stored in the session.
|
||||
session = self.client.session
|
||||
session.update(
|
||||
{
|
||||
"oauth_state": "test_state",
|
||||
},
|
||||
)
|
||||
session.save()
|
||||
|
||||
# Test Outlook OAuth callback
|
||||
response = self.client.get(
|
||||
"/api/oauth/callback/?code=test_code&state=test_state",
|
||||
|
||||
@@ -4,7 +4,6 @@ from datetime import timedelta
|
||||
from http import HTTPStatus
|
||||
from typing import Any
|
||||
|
||||
from django.conf import settings
|
||||
from django.http import HttpResponseBadRequest
|
||||
from django.http import HttpResponseForbidden
|
||||
from django.http import HttpResponseRedirect
|
||||
@@ -257,14 +256,9 @@ class OauthCallbackView(GenericAPIView[Any]):
|
||||
)
|
||||
return HttpResponseBadRequest("Invalid request, see logs for more detail")
|
||||
|
||||
session_state = request.session.get("oauth_state")
|
||||
if not session_state and not settings.DEBUG:
|
||||
logger.error(
|
||||
"Invalid oauth callback request: no state in session",
|
||||
)
|
||||
return HttpResponseBadRequest("Invalid request, see logs for more detail")
|
||||
|
||||
oauth_manager = PaperlessMailOAuth2Manager(state=session_state)
|
||||
oauth_manager = PaperlessMailOAuth2Manager(
|
||||
state=request.session.get("oauth_state"),
|
||||
)
|
||||
|
||||
state = request.query_params.get("state", "")
|
||||
if not oauth_manager.validate_state(state):
|
||||
@@ -321,5 +315,3 @@ class OauthCallbackView(GenericAPIView[Any]):
|
||||
return HttpResponseRedirect(
|
||||
f"{oauth_manager.oauth_redirect_url}?oauth_success=0",
|
||||
)
|
||||
finally:
|
||||
request.session.pop("oauth_state", None)
|
||||
|
||||
Reference in New Issue
Block a user