Compare commits

..
24 changed files with 1779 additions and 1515 deletions
+14 -19
View File
@@ -129,8 +129,8 @@ jobs:
~/.pnpm-store
~/.cache
key: ${{ runner.os }}-frontend-${{ hashFiles('src-ui/pnpm-lock.yaml') }}
- name: Re-link Angular CLI
run: cd src-ui && pnpm link @angular/cli
- name: Install dependencies
run: cd src-ui && pnpm install --frozen-lockfile
- name: Run lint
run: cd src-ui && pnpm run lint
unit-tests:
@@ -168,8 +168,8 @@ jobs:
~/.pnpm-store
~/.cache
key: ${{ runner.os }}-frontend-${{ hashFiles('src-ui/pnpm-lock.yaml') }}
- name: Re-link Angular CLI
run: cd src-ui && pnpm link @angular/cli
- name: Install dependencies
run: cd src-ui && pnpm install --frozen-lockfile
- name: Run Jest unit tests
run: cd src-ui && pnpm run test --max-workers=2 --shard=${{ matrix.shard-index }}/${{ matrix.shard-count }}
- name: Upload test results to Codecov
@@ -223,18 +223,15 @@ jobs:
~/.pnpm-store
~/.cache
key: ${{ runner.os }}-frontend-${{ hashFiles('src-ui/pnpm-lock.yaml') }}
- name: Re-link Angular CLI
run: cd src-ui && pnpm link @angular/cli
- name: Install dependencies
run: cd src-ui && pnpm install --no-frozen-lockfile
run: cd src-ui && pnpm install --frozen-lockfile
- name: Run Playwright E2E tests
run: cd src-ui && pnpm exec playwright test --shard ${{ matrix.shard-index }}/${{ matrix.shard-count }}
bundle-analysis:
name: Bundle Analysis
frontend-build:
name: Frontend Build
needs: [changes, unit-tests, e2e-tests]
if: needs.changes.outputs.frontend_changed == 'true'
runs-on: ubuntu-24.04
environment: bundle-analysis
permissions:
contents: read
steps:
@@ -260,21 +257,19 @@ jobs:
~/.pnpm-store
~/.cache
key: ${{ runner.os }}-frontend-${{ hashFiles('src-ui/pnpm-lock.yaml') }}
- name: Re-link Angular CLI
run: cd src-ui && pnpm link @angular/cli
- name: Build and analyze
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
- name: Install dependencies
run: cd src-ui && pnpm install --frozen-lockfile
- name: Build
run: cd src-ui && pnpm run build --configuration=production
gate:
name: Frontend CI Gate
needs: [changes, install-dependencies, lint, unit-tests, e2e-tests, bundle-analysis]
needs: [changes, install-dependencies, lint, unit-tests, e2e-tests, frontend-build]
if: always()
runs-on: ubuntu-slim
steps:
- name: Check gate
env:
BUNDLE_ANALYSIS_RESULT: ${{ needs['bundle-analysis'].result }}
BUILD_RESULT: ${{ needs['frontend-build'].result }}
E2E_RESULT: ${{ needs['e2e-tests'].result }}
FRONTEND_CHANGED: ${{ needs.changes.outputs.frontend_changed }}
INSTALL_RESULT: ${{ needs['install-dependencies'].result }}
@@ -306,8 +301,8 @@ jobs:
exit 1
fi
if [[ "${BUNDLE_ANALYSIS_RESULT}" != "success" ]]; then
echo "::error::Frontend bundle-analysis job result: ${BUNDLE_ANALYSIS_RESULT}"
if [[ "${BUILD_RESULT}" != "success" ]]; then
echo "::error::Frontend build job result: ${BUILD_RESULT}"
exit 1
fi
+1 -4
View File
@@ -61,10 +61,7 @@ jobs:
~/.cache
key: ${{ runner.os }}-frontenddeps-${{ hashFiles('src-ui/pnpm-lock.yaml') }}
- name: Install frontend dependencies
if: steps.cache-frontend-deps.outputs.cache-hit != 'true'
run: cd src-ui && pnpm install
- name: Re-link Angular cli
run: cd src-ui && pnpm link @angular/cli
run: cd src-ui && pnpm install --frozen-lockfile
- name: Generate frontend translation strings
run: |
cd src-ui
+12 -9
View File
@@ -56,13 +56,13 @@
},
"architect": {
"build": {
"builder": "@angular-builders/custom-webpack:browser",
"builder": "@angular/build:application",
"options": {
"customWebpackConfig": {
"path": "./extra-webpack.config.ts"
"outputPath": {
"base": "dist/paperless-ui",
"browser": ""
},
"outputPath": "dist/paperless-ui",
"main": "src/main.ts",
"browser": "src/main.ts",
"outputHashing": "none",
"index": "src/index.html",
"polyfills": [
@@ -97,6 +97,7 @@
"scripts": [],
"allowedCommonJsDependencies": [
"file-saver",
"mime-names",
"utif"
],
"extractLicenses": false,
@@ -117,11 +118,13 @@
"with": "src/environments/environment.prod.ts"
}
],
"outputPath": "../src/documents/static/frontend/",
"outputPath": {
"base": "../src/documents/static/frontend/",
"browser": ""
},
"optimization": true,
"outputHashing": "none",
"sourceMap": false,
"namedChunks": false,
"extractLicenses": true,
"budgets": [
{
@@ -145,7 +148,7 @@
"defaultConfiguration": ""
},
"serve": {
"builder": "@angular-builders/custom-webpack:dev-server",
"builder": "@angular/build:dev-server",
"options": {
"buildTarget": "paperless-ui:build:en-US"
},
@@ -156,7 +159,7 @@
}
},
"extract-i18n": {
"builder": "@angular-builders/custom-webpack:extract-i18n",
"builder": "@angular/build:extract-i18n",
"options": {
"buildTarget": "paperless-ui:build"
}
@@ -1,4 +1,4 @@
import { expect, test } from '@playwright/test'
import { expect, test, type WebSocketRoute } from '@playwright/test'
import path from 'node:path'
const REQUESTS_HAR = path.join(__dirname, 'requests/api-document-detail.har')
@@ -95,3 +95,60 @@ test('should support quick filters', async ({ page }) => {
.click()
await expect(page).toHaveURL(/tags__id__all=4&sort=created&reverse=1&page=1/)
})
test('should finish reloading the preview after a remote document update', async ({
page,
}) => {
let resolveStatusSocket: (socket: WebSocketRoute) => void
const statusSocketReady = new Promise<WebSocketRoute>((resolve) => {
resolveStatusSocket = resolve
})
await page.routeWebSocket(/\/ws\/status\/$/, (socket) => {
resolveStatusSocket(socket)
})
await page.routeFromHAR(REQUESTS_HAR, { notFound: 'fallback' })
let previewRequestCount = 0
page.on('request', (request) => {
if (request.url().includes('/api/documents/175/preview/')) {
previewRequestCount++
}
})
await page.goto('/documents/175/details')
await page.locator('pngx-document-detail').waitFor()
await expect(page.getByTitle('Storage path', { exact: true })).toHaveText(
/\w+/
)
const previewWasLoaded = await page.evaluate(() => {
const detail = document.querySelector('pngx-document-detail')
const component = (window as any).ng.getComponent(detail)
component.pdfPreviewLoaded({ numPages: 1 })
return component.previewLoaded()
})
expect(previewWasLoaded).toBe(true)
const previewRequestsBeforeReload = previewRequestCount
const statusSocket = await statusSocketReady
const documentReloaded = page.waitForResponse(
(response) =>
response.url().includes('/api/documents/175/?full_perms=true') &&
response.request().method() === 'GET'
)
statusSocket.send(
JSON.stringify({
type: 'document_updated',
data: {
document_id: 175,
modified: '2026-07-26T20:00:00Z',
},
})
)
await documentReloaded
await expect(
page.getByText('Document reloaded with latest changes.').first()
).toBeVisible()
await expect
.poll(() => previewRequestCount)
.toBeGreaterThan(previewRequestsBeforeReload + 1)
})
-24
View File
@@ -1,24 +0,0 @@
import {
CustomWebpackBrowserSchema,
TargetOptions,
} from '@angular-builders/custom-webpack'
import * as webpack from 'webpack'
const { codecovWebpackPlugin } = require('@codecov/webpack-plugin')
export default (
config: webpack.Configuration,
options: CustomWebpackBrowserSchema,
targetOptions: TargetOptions
) => {
if (config.plugins) {
config.plugins.push(
codecovWebpackPlugin({
enableBundleAnalysis: process.env.CODECOV_TOKEN !== undefined,
bundleName: 'paperless-ngx',
uploadToken: process.env.CODECOV_TOKEN,
})
)
}
return config
}
+3 -6
View File
@@ -39,7 +39,6 @@
"uuid": "^14.0.1"
},
"devDependencies": {
"@angular-builders/custom-webpack": "^22.0.1",
"@angular-builders/jest": "^22.0.1",
"@angular-devkit/core": "^22.0.5",
"@angular-devkit/schematics": "^22.0.5",
@@ -48,10 +47,9 @@
"@angular-eslint/eslint-plugin-template": "22.0.0",
"@angular-eslint/schematics": "22.0.0",
"@angular-eslint/template-parser": "22.0.0",
"@angular/build": "^22.0.5",
"@angular/cli": "~22.0.5",
"@angular/build": "22.1.0-rc.0",
"@angular/cli": "22.1.0-rc.0",
"@angular/compiler-cli": "~22.0.5",
"@codecov/webpack-plugin": "^2.0.1",
"@playwright/test": "^1.61.1",
"@types/jest": "^30.0.0",
"@types/node": "^26.0.0",
@@ -66,8 +64,7 @@
"jest-websocket-mock": "^2.5.0",
"prettier-plugin-organize-imports": "^4.3.0",
"ts-node": "~10.9.1",
"typescript": "^6.0.3",
"webpack": "^5.107.2"
"typescript": "^6.0.3"
},
"packageManager": "pnpm@10.26.0"
}
+1563 -1365
View File
File diff suppressed because it is too large Load Diff
@@ -283,6 +283,22 @@ describe('PngxPdfViewerComponent', () => {
expect(mockViewer.currentPageNumber).toBe(1)
})
it('reloads when the source revision changes', () => {
const resetSpy = jest.spyOn(component as any, 'resetViewerState')
const loadSpy = jest
.spyOn(component as any, 'loadDocument')
.mockImplementation(() => {})
component.src = 'test.pdf'
component.sourceRevision = 1
component.ngOnChanges({
sourceRevision: new SimpleChange(0, 1, false),
})
expect(resetSpy).toHaveBeenCalled()
expect(loadSpy).toHaveBeenCalled()
})
it('applies viewer state after view init when already loaded', () => {
const applySpy = jest.spyOn(component as any, 'applyViewerState')
;(component as any).hasLoaded = true
@@ -43,6 +43,7 @@ export class PngxPdfViewerComponent
private readonly document = inject<Document>(DOCUMENT)
@Input() src!: string
@Input() sourceRevision = 0
@Input() password?: string
@Input() page?: number
@Output() pageChange = new EventEmitter<number>()
@@ -93,7 +94,7 @@ export class PngxPdfViewerComponent
}
ngOnChanges(changes: SimpleChanges): void {
if (changes['src'] || changes['password']) {
if (changes['src'] || changes['sourceRevision'] || changes['password']) {
this.resetViewerState()
if (this.src) {
this.loadDocument()
@@ -472,6 +472,7 @@
<div class="preview-sticky pdf-viewer-container">
<pngx-pdf-viewer
[src]="pdfSource()"
[sourceRevision]="previewRevision()"
[password]="pdfPassword()"
[renderMode]="PdfRenderMode.All"
[page]="previewCurrentPage()" (pageChange)="previewCurrentPage.set($event)"
@@ -1602,6 +1602,7 @@ describe('DocumentDetailComponent', () => {
expect(openDoc.__changedFields).toEqual([])
expect(setDirtySpy).toHaveBeenCalledWith(openDoc, false)
expect(saveSpy).toHaveBeenCalled()
expect(component.previewRevision()).toBe(1)
})
it('should ignore incoming update for a different document id', () => {
@@ -249,6 +249,7 @@ export class DocumentDetailComponent
titleSubject: Subject<string> = new Subject()
readonly previewUrl = signal<string>(undefined)
readonly pdfSource = signal<string>(undefined)
readonly previewRevision = signal(0)
readonly pdfPassword = signal<string>(undefined)
readonly thumbUrl = signal<string>(undefined)
readonly previewText = signal<string>(undefined)
@@ -609,6 +610,9 @@ export class DocumentDetailComponent
.subscribe()
}
this.updateComponent(useDoc)
if (forceRemote) {
this.previewRevision.update((revision) => revision + 1)
}
this.titleSubject
.pipe(
debounceTime(1000),
@@ -25,7 +25,7 @@
<input #textFilterInput class="form-control form-control-sm" type="text"
[disabled]="textFilterModifierIsNull"
[(ngModel)]="textFilter"
(keyup)="textFilterKeyup($event)"
(keydown)="textFilterKeydown($event)"
[ngbTypeahead]="searchAutoComplete"
(selectItem)="itemSelected($event)"
[readonly]="textFilterTarget === 'fulltext-morelike'">
@@ -2180,17 +2180,17 @@ describe('FilterEditorComponent', () => {
it('should support Enter / Esc key on text field', () => {
component.textFilterInput.nativeElement.value = 'foo'
component.textFilterInput.nativeElement.dispatchEvent(
new KeyboardEvent('keyup', { key: 'Enter' })
new KeyboardEvent('keydown', { key: 'Enter' })
)
expect(component.textFilter).toEqual('foo')
component.textFilterInput.nativeElement.value = 'foo bar'
component.textFilterInput.nativeElement.dispatchEvent(
new KeyboardEvent('keyup', { key: 'Escape' })
new KeyboardEvent('keydown', { key: 'Escape' })
)
expect(component.textFilter).toEqual('')
const blurSpy = jest.spyOn(component.textFilterInput.nativeElement, 'blur')
component.textFilterInput.nativeElement.dispatchEvent(
new KeyboardEvent('keyup', { key: 'Escape' })
new KeyboardEvent('keydown', { key: 'Escape' })
)
expect(blurSpy).toHaveBeenCalled()
})
@@ -1312,7 +1312,7 @@ export class FilterEditorComponent
}
}
textFilterKeyup(event: KeyboardEvent) {
textFilterKeydown(event: KeyboardEvent) {
if (event.key == 'Enter') {
const filterString = (
this.textFilterInput.nativeElement as HTMLInputElement
@@ -172,39 +172,3 @@ class TestApiUiSettings(DirectoriesMixin, APITestCase):
self.assertIsNotNone(
response.data["settings"]["outlook_oauth_url"],
)
@override_settings(
OAUTH_CALLBACK_BASE_URL="http://localhost:8000",
GMAIL_OAUTH_CLIENT_ID="abc123",
GMAIL_OAUTH_CLIENT_SECRET="def456",
GMAIL_OAUTH_ENABLED=True,
OUTLOOK_OAUTH_CLIENT_ID="ghi789",
OUTLOOK_OAUTH_CLIENT_SECRET="jkl012",
OUTLOOK_OAUTH_ENABLED=True,
)
def test_settings_oauth_state_reused_across_calls(self) -> None:
"""
GIVEN:
- A mail account oauth flow has been started, storing a state in the session
WHEN:
- The ui_settings endpoint is called again before the flow completes
THEN:
- The same oauth state is reused, not replaced with a new one
"""
response = self.client.get(self.ENDPOINT, format="json")
self.assertEqual(response.status_code, status.HTTP_200_OK)
first_state = self.client.session["oauth_state"]
first_gmail_url = response.data["settings"]["gmail_oauth_url"]
first_outlook_url = response.data["settings"]["outlook_oauth_url"]
response = self.client.get(self.ENDPOINT, format="json")
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(self.client.session["oauth_state"], first_state)
self.assertEqual(
response.data["settings"]["gmail_oauth_url"],
first_gmail_url,
)
self.assertEqual(
response.data["settings"]["outlook_oauth_url"],
first_outlook_url,
)
+3 -7
View File
@@ -4002,19 +4002,15 @@ class UiSettingsView(GenericAPIView[Any]):
ui_settings["auditlog_enabled"] = settings.AUDIT_LOG_ENABLED
if settings.GMAIL_OAUTH_ENABLED or settings.OUTLOOK_OAUTH_ENABLED:
# Reuse an in-flight state instead of minting a new one on every
# settings fetch, or a concurrent request can invalidate a login
# that's still in progress.
manager = PaperlessMailOAuth2Manager(
state=request.session.get("oauth_state"),
)
request.session["oauth_state"] = manager.state
manager = PaperlessMailOAuth2Manager()
if settings.GMAIL_OAUTH_ENABLED:
ui_settings["gmail_oauth_url"] = manager.get_gmail_authorization_url()
request.session["oauth_state"] = manager.state
if settings.OUTLOOK_OAUTH_ENABLED:
ui_settings["outlook_oauth_url"] = (
manager.get_outlook_authorization_url()
)
request.session["oauth_state"] = manager.state
ui_settings["email_enabled"] = settings.EMAIL_ENABLED
@@ -5,12 +5,30 @@ from django.db import migrations
from django.db import models
def clamp_applicationconfiguration_integer_fields(apps, schema_editor):
# Clamp barcode_dpi, barcode_max_pages, image_dpi and pages because of
# PositiveIntegerField --> PositiveSmallIntegerField
ApplicationConfiguration = apps.get_model("paperless", "ApplicationConfiguration")
ApplicationConfiguration.objects.filter(barcode_dpi__gt=32767).update(
barcode_dpi=32767,
)
ApplicationConfiguration.objects.filter(barcode_max_pages__gt=32767).update(
barcode_max_pages=32767,
)
ApplicationConfiguration.objects.filter(image_dpi__gt=32767).update(image_dpi=32767)
ApplicationConfiguration.objects.filter(pages__gt=32767).update(pages=32767)
class Migration(migrations.Migration):
dependencies = [
("paperless", "0006_applicationconfiguration_barcode_tag_split"),
]
operations = [
migrations.RunPython(
clamp_applicationconfiguration_integer_fields,
migrations.RunPython.noop,
),
migrations.AlterField(
model_name="applicationconfiguration",
name="barcode_dpi",
+2
View File
@@ -2,6 +2,8 @@ from pydantic import BaseModel
class DocumentClassifierSchema(BaseModel):
"""Schema for document classification suggestions."""
title: str
tags: list[str]
correspondents: list[str]
+17 -7
View File
@@ -5,6 +5,7 @@ from datetime import timedelta
from typing import TYPE_CHECKING
from django.conf import settings
from django.core.exceptions import ObjectDoesNotExist
from django.utils import timezone
from filelock import FileLock
from filelock import ReadWriteLock
@@ -167,6 +168,19 @@ def write_store(embed_model_name: str | None = None):
yield store
def _safe_related_name(document: Document, field: str) -> str | None:
"""
Returns the ``name`` of a related object (correspondent, document_type,
storage_path), or None if the FK is unset or points at a row that has
since been deleted (e.g. concurrently with this call).
"""
try:
related = getattr(document, field)
except ObjectDoesNotExist:
return None
return related.name if related else None
def build_document_node(
document: Document,
*,
@@ -180,14 +194,10 @@ def build_document_node(
"document_id": str(document.id),
"title": document.title,
"tags": [t.name for t in document.tags.all()],
"correspondent": document.correspondent.name
if document.correspondent
else None,
"document_type": document.document_type.name
if document.document_type
else None,
"correspondent": _safe_related_name(document, "correspondent"),
"document_type": _safe_related_name(document, "document_type"),
"filename": document.filename,
"storage_path": document.storage_path.name if document.storage_path else None,
"storage_path": _safe_related_name(document, "storage_path"),
"archive_serial_number": document.archive_serial_number,
"created": document.created.isoformat() if document.created else None,
"added": document.added.isoformat() if document.added else None,
@@ -8,7 +8,9 @@ from django.test import override_settings
from django.utils import timezone
from llama_index.core.schema import MetadataMode
from documents.models import Correspondent
from documents.models import Document
from documents.models import DocumentType
from documents.models import PaperlessTask
from documents.signals import document_consumption_finished
from documents.signals import document_updated
@@ -95,6 +97,38 @@ def test_build_document_node_structured_fields_in_metadata(
assert "modified" in node.metadata
@pytest.mark.django_db
def test_build_document_node_survives_concurrently_deleted_correspondent(
real_document: Document,
) -> None:
"""Regression test for #13314.
If a document's correspondent (or document type) is deleted after the
in-memory Document instance was loaded but before build_document_node
resolves the relation, accessing the FK must not raise -- it should
behave like an unset FK and produce None in the metadata instead of
aborting the whole indexing pass.
"""
correspondent = Correspondent.objects.create(name="Stale Correspondent")
document_type = DocumentType.objects.create(name="Stale Type")
real_document.correspondent = correspondent
real_document.document_type = document_type
real_document.save()
# Re-fetch to get an instance whose correspondent/document_type relations
# are unresolved (not yet cached), mirroring a task that loaded the
# document before the concurrent deletion below.
stale_document = Document.objects.get(pk=real_document.pk)
correspondent.delete()
document_type.delete()
nodes = indexing.build_document_node(stale_document)
assert len(nodes) > 0
assert nodes[0].metadata["correspondent"] is None
assert nodes[0].metadata["document_type"] is None
@pytest.mark.django_db
def test_build_document_node_excludes_document_id_from_llm_context(
real_document: Document,
@@ -10,6 +10,24 @@ def clamp_mailrule_maximum_age(apps, schema_editor):
MailRule.objects.filter(maximum_age__gt=32767).update(maximum_age=32767)
def clamp_mailrule_order(apps, schema_editor):
# The order field of MailRule is only used for relative sorting
# (account.rules.order_by("order")), so out-of-range values must be
# renumbered by rank rather than clamped to a single value, which
# would collapse distinct rules to the same order.
MailRule = apps.get_model("paperless_mail", "MailRule")
if (
MailRule.objects.filter(order__gt=32767).exists()
or MailRule.objects.filter(
order__lt=-32768,
).exists()
): # pragma: no cover
for index, rule_id in enumerate(
MailRule.objects.order_by("order", "pk").values_list("pk", flat=True),
):
MailRule.objects.filter(pk=rule_id).update(order=index)
class Migration(migrations.Migration):
# The data update must commit before PostgreSQL rewrites the table, otherwise
# pending foreign-key trigger events can block the subsequent ALTER TABLE.
@@ -135,6 +153,10 @@ class Migration(migrations.Migration):
verbose_name="maximum age",
),
),
migrations.RunPython(
clamp_mailrule_order,
migrations.RunPython.noop,
),
migrations.AlterField(
model_name="mailrule",
name="order",
@@ -138,16 +138,6 @@ class TestMailOAuth(
MailAccount.objects.filter(imap_server="imap.gmail.com").exists(),
)
# State is single-use and was cleared by the callback above, so a new
# flow needs a new state stored in the session.
session = self.client.session
session.update(
{
"oauth_state": "test_state",
},
)
session.save()
# Test Outlook OAuth callback
response = self.client.get(
"/api/oauth/callback/?code=test_code&state=test_state",
@@ -190,16 +180,6 @@ class TestMailOAuth(
MailAccount.objects.filter(imap_server="imap.gmail.com").exists(),
)
# State is single-use and was cleared by the callback above, so a
# new flow needs a new state stored in the session.
session = self.client.session
session.update(
{
"oauth_state": "test_state",
},
)
session.save()
# Test Outlook OAuth callback
response = self.client.get(
"/api/oauth/callback/?code=test_code&state=test_state",
+3 -11
View File
@@ -4,7 +4,6 @@ from datetime import timedelta
from http import HTTPStatus
from typing import Any
from django.conf import settings
from django.http import HttpResponseBadRequest
from django.http import HttpResponseForbidden
from django.http import HttpResponseRedirect
@@ -257,14 +256,9 @@ class OauthCallbackView(GenericAPIView[Any]):
)
return HttpResponseBadRequest("Invalid request, see logs for more detail")
session_state = request.session.get("oauth_state")
if not session_state and not settings.DEBUG:
logger.error(
"Invalid oauth callback request: no state in session",
)
return HttpResponseBadRequest("Invalid request, see logs for more detail")
oauth_manager = PaperlessMailOAuth2Manager(state=session_state)
oauth_manager = PaperlessMailOAuth2Manager(
state=request.session.get("oauth_state"),
)
state = request.query_params.get("state", "")
if not oauth_manager.validate_state(state):
@@ -321,5 +315,3 @@ class OauthCallbackView(GenericAPIView[Any]):
return HttpResponseRedirect(
f"{oauth_manager.oauth_redirect_url}?oauth_success=0",
)
finally:
request.session.pop("oauth_state", None)