Compare commits

..
Author SHA1 Message Date
stumpylog beaa37125c Fix: don't skip OCR/archive for tagged PDFs with no actual text (#13349) 2026-07-27 15:15:06 -07:00
shamoonandGitHub 7eee8538ec Performance: more performant filtering diacritic normalization (#13347) 2026-07-27 22:13:16 +00:00
98b66fdf24 Perf: prefetch notes and custom fields for LLM index text building (#13350)
build_llm_index_text queried Note and CustomFieldInstance (plus its
field FK) per document, uncovered by the earlier correspondent/type/
storage_path/tags prefetch fix. Add notes and custom_fields__field to
the rebuild and scoped document querysets, and read from doc.notes.all()
instead of a fresh Note.objects.filter() so the prefetch is actually used.

Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>
2026-07-27 14:50:37 -07:00
GitHub Actions 693a111c5a Auto translate strings 2026-07-27 19:35:15 +00:00
Trenton HandGitHub b19edd0b74 Fix: dedupe permission-visible documents when combined with multi-tag ALL filtering (#13331) (#13345)
The permission filter OR'd three querysets together on top of a
queryset that could already carry two independent tags__id__all
joins, letting a document that matched more than one branch (e.g.
unowned + group-permissioned) come back twice. Replaced it with a
single id__in filter against the existing permitted_document_ids
helper, which is join-free and can't hit this.
2026-07-27 19:33:26 +00:00
Trenton HandGitHub 0e98a7f1ce Performance: Scope llm index updates to actually modified documents (#13322)
* Perf: scope bulk_update_documents' LLM index refresh to the edited document ids instead of the whole library

* Perf: select_related/prefetch_related for the scoped LLM index batch

* Perf: select_related/prefetch_related for the full LLM index rebuild path

* Fix: address Copilot review findings on LLM index scoping
2026-07-27 10:23:43 -07:00
GitHub Actions 14517062c4 Auto translate strings 2026-07-27 16:53:33 +00:00
shamoonandGitHub 0c0faf7f80 Fixhancement: pass LLM output language to chat if specified (#13340) 2026-07-27 16:51:31 +00:00
SandroandGitHub eda79603fe Documentation: fix PAPERLESS_AI_LLM_OUTPUT_LANGUAGE heading level (#13341) 2026-07-27 09:37:09 -07:00
shamoonandGitHub 8f017942d9 Chore: resolve npm provenance issues with chokidar and semver (#13323) 2026-07-26 16:25:49 -07:00
GitHub Actions f4e7a2e9e4 Auto translate strings 2026-07-26 22:38:16 +00:00
shamoonandGitHub 317e534aa0 Fix: ensure preview reload on live changes (#13321) 2026-07-26 22:36:03 +00:00
Trenton HandGitHub 3ba6d0325a Fix: clamp out-of-range MailRule.order and ApplicationConfiguration DPI/page fields before smallint migration (#13316) 2026-07-26 22:11:27 +00:00
Trenton HandGitHub 318520a0f1 Fix: add docstring to DocumentClassifierSchema for cleaner LLM tool description (#13315) 2026-07-26 22:00:06 +00:00
Trenton HandGitHub ea1f3653a9 Fix: guard build_document_node against stale FK on deleted correspondent/doc type (#13318) 2026-07-26 21:45:59 +00:00
shamoonandGitHub 423d4b9f2d Fix: prevent search filter loss when closing document with Escape key (#13317) 2026-07-26 14:17:26 -07:00
shamoonandGitHub 8bc4e5ee94 Fix: fix frontend permissions display for stats/system perms (#13305) 2026-07-26 02:56:06 -07:00
42 changed files with 677 additions and 160 deletions
+3 -3
View File
@@ -932,8 +932,8 @@ for display in the web interface.
| Document type | `never` | `auto` (default) | `always` |
| -------------------------- | ------- | -------------------------- | -------- |
| Scanned image (TIFF, JPEG) | No | **Yes** | Yes |
| Image-based PDF | No | **Yes** (short/no text, untagged) | Yes |
| Born-digital PDF | No | No (tagged or has embedded text) | Yes |
| Image-based PDF | No | **Yes** (no embedded text) | Yes |
| Born-digital PDF | No | No (has embedded text, optionally confirmed by tag) | Yes |
| Plain text, email, HTML | No | No | No |
| DOCX / ODT (via Tika) | Yes\* | Yes\* | Yes\* |
@@ -2135,7 +2135,7 @@ used with the OpenAI-compatible backend to target a custom provider or local gat
Defaults to None.
### [`PAPERLESS_AI_LLM_OUTPUT_LANGUAGE=<str>`](#PAPERLESS_AI_LLM_OUTPUT_LANGUAGE) {#PAPERLESS_AI_LLM_OUTPUT_LANGUAGE}
#### [`PAPERLESS_AI_LLM_OUTPUT_LANGUAGE=<str>`](#PAPERLESS_AI_LLM_OUTPUT_LANGUAGE) {#PAPERLESS_AI_LLM_OUTPUT_LANGUAGE}
: The language to use for AI suggestions (results may vary by LLM model). If not supplied, defaults to the user's UI language setting or None.
+1 -1
View File
@@ -1,6 +1,6 @@
[project]
name = "paperless-ngx"
version = "3.0.3"
version = "3.0.2"
description = "A community-supported supercharged document management system: scan, index and archive all your physical documents"
readme = "README.md"
requires-python = ">=3.11"
@@ -1,4 +1,4 @@
import { expect, test } from '@playwright/test'
import { expect, test, type WebSocketRoute } from '@playwright/test'
import path from 'node:path'
const REQUESTS_HAR = path.join(__dirname, 'requests/api-document-detail.har')
@@ -95,3 +95,60 @@ test('should support quick filters', async ({ page }) => {
.click()
await expect(page).toHaveURL(/tags__id__all=4&sort=created&reverse=1&page=1/)
})
test('should finish reloading the preview after a remote document update', async ({
page,
}) => {
let resolveStatusSocket: (socket: WebSocketRoute) => void
const statusSocketReady = new Promise<WebSocketRoute>((resolve) => {
resolveStatusSocket = resolve
})
await page.routeWebSocket(/\/ws\/status\/$/, (socket) => {
resolveStatusSocket(socket)
})
await page.routeFromHAR(REQUESTS_HAR, { notFound: 'fallback' })
let previewRequestCount = 0
page.on('request', (request) => {
if (request.url().includes('/api/documents/175/preview/')) {
previewRequestCount++
}
})
await page.goto('/documents/175/details')
await page.locator('pngx-document-detail').waitFor()
await expect(page.getByTitle('Storage path', { exact: true })).toHaveText(
/\w+/
)
const previewWasLoaded = await page.evaluate(() => {
const detail = document.querySelector('pngx-document-detail')
const component = (window as any).ng.getComponent(detail)
component.pdfPreviewLoaded({ numPages: 1 })
return component.previewLoaded()
})
expect(previewWasLoaded).toBe(true)
const previewRequestsBeforeReload = previewRequestCount
const statusSocket = await statusSocketReady
const documentReloaded = page.waitForResponse(
(response) =>
response.url().includes('/api/documents/175/?full_perms=true') &&
response.request().method() === 'GET'
)
statusSocket.send(
JSON.stringify({
type: 'document_updated',
data: {
document_id: 175,
modified: '2026-07-26T20:00:00Z',
},
})
)
await documentReloaded
await expect(
page.getByText('Document reloaded with latest changes.').first()
).toBeVisible()
await expect
.poll(() => previewRequestCount)
.toBeGreaterThan(previewRequestsBeforeReload + 1)
})
+49 -49
View File
@@ -1277,7 +1277,7 @@
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">1787</context>
<context context-type="linenumber">1791</context>
</context-group>
</trans-unit>
<trans-unit id="1577733187050997705" datatype="html">
@@ -2184,7 +2184,7 @@
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">653</context>
<context context-type="linenumber">657</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.html</context>
@@ -3087,11 +3087,11 @@
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">1405</context>
<context context-type="linenumber">1409</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">1788</context>
<context context-type="linenumber">1792</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-list/bulk-editor/bulk-editor.component.ts</context>
@@ -3688,7 +3688,7 @@
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">1358</context>
<context context-type="linenumber">1362</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-list/bulk-editor/bulk-editor.component.ts</context>
@@ -3800,7 +3800,7 @@
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">1841</context>
<context context-type="linenumber">1845</context>
</context-group>
</trans-unit>
<trans-unit id="6661109599266152398" datatype="html">
@@ -3811,7 +3811,7 @@
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">1842</context>
<context context-type="linenumber">1846</context>
</context-group>
</trans-unit>
<trans-unit id="5162686434580248853" datatype="html">
@@ -3822,7 +3822,7 @@
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">1843</context>
<context context-type="linenumber">1847</context>
</context-group>
</trans-unit>
<trans-unit id="8157388568390631653" datatype="html">
@@ -5724,7 +5724,7 @@
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">1362</context>
<context context-type="linenumber">1366</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-list/bulk-editor/bulk-editor.component.ts</context>
@@ -7966,88 +7966,88 @@
<source>Enter Password</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.html</context>
<context context-type="linenumber">512</context>
<context context-type="linenumber">513</context>
</context-group>
</trans-unit>
<trans-unit id="5758784066858623886" datatype="html">
<source>Error retrieving metadata</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">423</context>
<context context-type="linenumber">424</context>
</context-group>
</trans-unit>
<trans-unit id="2218903673684131427" datatype="html">
<source>An error occurred loading content: <x id="PH" equiv-text="err.message ?? err.toString()"/></source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">525,527</context>
<context context-type="linenumber">526,528</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">982,984</context>
<context context-type="linenumber">986,988</context>
</context-group>
</trans-unit>
<trans-unit id="6357361810318120957" datatype="html">
<source>Document was updated</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">648</context>
<context context-type="linenumber">652</context>
</context-group>
</trans-unit>
<trans-unit id="5154064822428631306" datatype="html">
<source>Document was updated at <x id="PH" equiv-text="formattedModified"/>.</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">649</context>
<context context-type="linenumber">653</context>
</context-group>
</trans-unit>
<trans-unit id="8462497568316256794" datatype="html">
<source>Reload to discard your local unsaved edits and load the latest remote version.</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">650</context>
<context context-type="linenumber">654</context>
</context-group>
</trans-unit>
<trans-unit id="7967484035994732534" datatype="html">
<source>Reload</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">652</context>
<context context-type="linenumber">656</context>
</context-group>
</trans-unit>
<trans-unit id="2907037627372942104" datatype="html">
<source>Document reloaded with latest changes.</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">708</context>
<context context-type="linenumber">712</context>
</context-group>
</trans-unit>
<trans-unit id="6435639868943916539" datatype="html">
<source>Document reloaded.</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">719</context>
<context context-type="linenumber">723</context>
</context-group>
</trans-unit>
<trans-unit id="6142395741265832184" datatype="html">
<source>Next document</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">821</context>
<context context-type="linenumber">825</context>
</context-group>
</trans-unit>
<trans-unit id="651985345816518480" datatype="html">
<source>Previous document</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">831</context>
<context context-type="linenumber">835</context>
</context-group>
</trans-unit>
<trans-unit id="2885986061416655600" datatype="html">
<source>Close document</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">839</context>
<context context-type="linenumber">843</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/services/open-documents.service.ts</context>
@@ -8058,67 +8058,67 @@
<source>Save document</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">846</context>
<context context-type="linenumber">850</context>
</context-group>
</trans-unit>
<trans-unit id="1784543155727940353" datatype="html">
<source>Save and close / next</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">855</context>
<context context-type="linenumber">859</context>
</context-group>
</trans-unit>
<trans-unit id="7427704425579737895" datatype="html">
<source>Error retrieving version content</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">965</context>
<context context-type="linenumber">969</context>
</context-group>
</trans-unit>
<trans-unit id="3456881259945295697" datatype="html">
<source>Error retrieving suggestions.</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">1026</context>
<context context-type="linenumber">1030</context>
</context-group>
</trans-unit>
<trans-unit id="2194092841814123758" datatype="html">
<source>Document &quot;<x id="PH" equiv-text="newValues.title"/>&quot; saved successfully.</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">1238</context>
<context context-type="linenumber">1242</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">1265</context>
<context context-type="linenumber">1269</context>
</context-group>
</trans-unit>
<trans-unit id="6626387786259219838" datatype="html">
<source>Error saving document &quot;<x id="PH" equiv-text="this.document().title"/>&quot;</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">1271</context>
<context context-type="linenumber">1275</context>
</context-group>
</trans-unit>
<trans-unit id="448882439049417053" datatype="html">
<source>Error saving document</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">1326</context>
<context context-type="linenumber">1330</context>
</context-group>
</trans-unit>
<trans-unit id="8410796510716511826" datatype="html">
<source>Do you really want to move the document &quot;<x id="PH" equiv-text="this.document().title"/>&quot; to the trash?</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">1359</context>
<context context-type="linenumber">1363</context>
</context-group>
</trans-unit>
<trans-unit id="282586936710748252" datatype="html">
<source>Documents can be restored prior to permanent deletion.</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">1360</context>
<context context-type="linenumber">1364</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-list/bulk-editor/bulk-editor.component.ts</context>
@@ -8129,14 +8129,14 @@
<source>Error deleting document</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">1381</context>
<context context-type="linenumber">1385</context>
</context-group>
</trans-unit>
<trans-unit id="619486176823357521" datatype="html">
<source>Reprocess confirm</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">1401</context>
<context context-type="linenumber">1405</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-list/bulk-editor/bulk-editor.component.ts</context>
@@ -8147,102 +8147,102 @@
<source>This operation will permanently recreate the archive file for this document.</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">1402</context>
<context context-type="linenumber">1406</context>
</context-group>
</trans-unit>
<trans-unit id="302054111564709516" datatype="html">
<source>The archive file will be re-generated with the current settings.</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">1403</context>
<context context-type="linenumber">1407</context>
</context-group>
</trans-unit>
<trans-unit id="4700389117298802932" datatype="html">
<source>Reprocess operation for &quot;<x id="PH" equiv-text="this.document().title"/>&quot; will begin in the background.</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">1413</context>
<context context-type="linenumber">1417</context>
</context-group>
</trans-unit>
<trans-unit id="4409560272830824468" datatype="html">
<source>Error executing operation</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">1424</context>
<context context-type="linenumber">1428</context>
</context-group>
</trans-unit>
<trans-unit id="6030453331794586802" datatype="html">
<source>Error downloading document</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">1487</context>
<context context-type="linenumber">1491</context>
</context-group>
</trans-unit>
<trans-unit id="4458954481601077369" datatype="html">
<source>Page Fit</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">1565</context>
<context context-type="linenumber">1569</context>
</context-group>
</trans-unit>
<trans-unit id="4663705961777238777" datatype="html">
<source>PDF edit operation for &quot;<x id="PH" equiv-text="this.document().title"/>&quot; will begin in the background.</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">1808</context>
<context context-type="linenumber">1812</context>
</context-group>
</trans-unit>
<trans-unit id="9043972994040261999" datatype="html">
<source>Error executing PDF edit operation</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">1820</context>
<context context-type="linenumber">1824</context>
</context-group>
</trans-unit>
<trans-unit id="6172690334763056188" datatype="html">
<source>Please enter the current password before attempting to remove it.</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">1831</context>
<context context-type="linenumber">1835</context>
</context-group>
</trans-unit>
<trans-unit id="968660764814228922" datatype="html">
<source>Password removal operation for &quot;<x id="PH" equiv-text="this.document().title"/>&quot; will begin in the background.</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">1865</context>
<context context-type="linenumber">1869</context>
</context-group>
</trans-unit>
<trans-unit id="2282118435712883014" datatype="html">
<source>Error executing password removal operation</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">1879</context>
<context context-type="linenumber">1883</context>
</context-group>
</trans-unit>
<trans-unit id="3740891324955700797" datatype="html">
<source>Print failed.</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">1925</context>
<context context-type="linenumber">1929</context>
</context-group>
</trans-unit>
<trans-unit id="6457245677384603573" datatype="html">
<source>Error loading document for printing.</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">1938</context>
<context context-type="linenumber">1942</context>
</context-group>
</trans-unit>
<trans-unit id="6085793215710522488" datatype="html">
<source>An error occurred loading tiff: <x id="PH" equiv-text="err.toString()"/></source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">2008</context>
<context context-type="linenumber">2012</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">2014</context>
<context context-type="linenumber">2018</context>
</context-group>
</trans-unit>
<trans-unit id="4958946940233632319" datatype="html">
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "paperless-ngx-ui",
"version": "3.0.3",
"version": "3.0.2",
"scripts": {
"preinstall": "npx only-allow pnpm",
"ng": "ng",
+3
View File
@@ -2,6 +2,9 @@ packages:
- "."
minimumReleaseAge: 10080
trustPolicy: no-downgrade
trustPolicyExclude:
- "chokidar@4.0.3"
- "semver@6.3.1 || 5.7.2"
allowBuilds:
"@parcel/watcher": true
canvas: true
@@ -283,6 +283,22 @@ describe('PngxPdfViewerComponent', () => {
expect(mockViewer.currentPageNumber).toBe(1)
})
it('reloads when the source revision changes', () => {
const resetSpy = jest.spyOn(component as any, 'resetViewerState')
const loadSpy = jest
.spyOn(component as any, 'loadDocument')
.mockImplementation(() => {})
component.src = 'test.pdf'
component.sourceRevision = 1
component.ngOnChanges({
sourceRevision: new SimpleChange(0, 1, false),
})
expect(resetSpy).toHaveBeenCalled()
expect(loadSpy).toHaveBeenCalled()
})
it('applies viewer state after view init when already loaded', () => {
const applySpy = jest.spyOn(component as any, 'applyViewerState')
;(component as any).hasLoaded = true
@@ -43,6 +43,7 @@ export class PngxPdfViewerComponent
private readonly document = inject<Document>(DOCUMENT)
@Input() src!: string
@Input() sourceRevision = 0
@Input() password?: string
@Input() page?: number
@Output() pageChange = new EventEmitter<number>()
@@ -93,7 +94,7 @@ export class PngxPdfViewerComponent
}
ngOnChanges(changes: SimpleChanges): void {
if (changes['src'] || changes['password']) {
if (changes['src'] || changes['sourceRevision'] || changes['password']) {
this.resetViewerState()
if (this.src) {
this.loadDocument()
@@ -472,6 +472,7 @@
<div class="preview-sticky pdf-viewer-container">
<pngx-pdf-viewer
[src]="pdfSource()"
[sourceRevision]="previewRevision()"
[password]="pdfPassword()"
[renderMode]="PdfRenderMode.All"
[page]="previewCurrentPage()" (pageChange)="previewCurrentPage.set($event)"
@@ -1602,6 +1602,7 @@ describe('DocumentDetailComponent', () => {
expect(openDoc.__changedFields).toEqual([])
expect(setDirtySpy).toHaveBeenCalledWith(openDoc, false)
expect(saveSpy).toHaveBeenCalled()
expect(component.previewRevision()).toBe(1)
})
it('should ignore incoming update for a different document id', () => {
@@ -249,6 +249,7 @@ export class DocumentDetailComponent
titleSubject: Subject<string> = new Subject()
readonly previewUrl = signal<string>(undefined)
readonly pdfSource = signal<string>(undefined)
readonly previewRevision = signal(0)
readonly pdfPassword = signal<string>(undefined)
readonly thumbUrl = signal<string>(undefined)
readonly previewText = signal<string>(undefined)
@@ -609,6 +610,9 @@ export class DocumentDetailComponent
.subscribe()
}
this.updateComponent(useDoc)
if (forceRemote) {
this.previewRevision.update((revision) => revision + 1)
}
this.titleSubject
.pipe(
debounceTime(1000),
@@ -25,7 +25,7 @@
<input #textFilterInput class="form-control form-control-sm" type="text"
[disabled]="textFilterModifierIsNull"
[(ngModel)]="textFilter"
(keyup)="textFilterKeyup($event)"
(keydown)="textFilterKeydown($event)"
[ngbTypeahead]="searchAutoComplete"
(selectItem)="itemSelected($event)"
[readonly]="textFilterTarget === 'fulltext-morelike'">
@@ -2180,17 +2180,17 @@ describe('FilterEditorComponent', () => {
it('should support Enter / Esc key on text field', () => {
component.textFilterInput.nativeElement.value = 'foo'
component.textFilterInput.nativeElement.dispatchEvent(
new KeyboardEvent('keyup', { key: 'Enter' })
new KeyboardEvent('keydown', { key: 'Enter' })
)
expect(component.textFilter).toEqual('foo')
component.textFilterInput.nativeElement.value = 'foo bar'
component.textFilterInput.nativeElement.dispatchEvent(
new KeyboardEvent('keyup', { key: 'Escape' })
new KeyboardEvent('keydown', { key: 'Escape' })
)
expect(component.textFilter).toEqual('')
const blurSpy = jest.spyOn(component.textFilterInput.nativeElement, 'blur')
component.textFilterInput.nativeElement.dispatchEvent(
new KeyboardEvent('keyup', { key: 'Escape' })
new KeyboardEvent('keydown', { key: 'Escape' })
)
expect(blurSpy).toHaveBeenCalled()
})
@@ -1312,7 +1312,7 @@ export class FilterEditorComponent
}
}
textFilterKeyup(event: KeyboardEvent) {
textFilterKeydown(event: KeyboardEvent) {
if (event.key == 'Enter') {
const filterString = (
this.textFilterInput.nativeElement as HTMLInputElement
@@ -108,6 +108,18 @@ describe('PermissionsService', () => {
actionKey: 'View', // PermissionAction.View
typeKey: 'Document', // PermissionType.Document
})
expect(
permissionsService.getPermissionKeys('view_global_statistics')
).toEqual({
actionKey: 'View', // PermissionAction.View
typeKey: 'GlobalStatistics', // PermissionType.GlobalStatistics
})
expect(
permissionsService.getPermissionKeys('view_system_monitoring')
).toEqual({
actionKey: 'View', // PermissionAction.View
typeKey: 'SystemMonitoring', // PermissionType.SystemMonitoring
})
})
it('correctly checks explicit global permissions', () => {
+7 -10
View File
@@ -110,19 +110,16 @@ export class PermissionsService {
actionKey: string
typeKey: string
} {
const matches = permissionStr.match(/(.+)_/)
let typeKey
let actionKey
if (matches?.length > 0) {
const action = matches[1]
const actionIndex = Object.values(PermissionAction).indexOf(
action as PermissionAction
)
if (actionIndex > -1) {
actionKey = Object.keys(PermissionAction)[actionIndex]
}
const actionIndex = Object.values(PermissionAction).findIndex((action) =>
permissionStr.startsWith(`${action}_`)
)
if (actionIndex > -1) {
const action = Object.values(PermissionAction)[actionIndex]
actionKey = Object.keys(PermissionAction)[actionIndex]
const typeIndex = Object.values(PermissionType).indexOf(
permissionStr.replace(action, '%s') as PermissionType
permissionStr.replace(`${action}_`, '%s_') as PermissionType
)
if (typeIndex > -1) {
typeKey = Object.keys(PermissionType)[typeIndex]
+16
View File
@@ -14,4 +14,20 @@ describe('text search utilities', () => {
expect(matchesSearchText('Tax\u00e9s 2026', 'taxe 26')).toBeTruthy()
expect(matchesSearchText('taxes 2026', 'tax receipt')).toBeFalsy()
})
it('matches a large set of tag names without blocking input', () => {
const tagNames = Array.from(
{ length: 1280 },
(_, index) =>
`Customer Party ${index} Jos\u00e9 M\u00fcller \u00c5ngstr\u00f6m`
)
const start = performance.now()
const matches = tagNames.filter((name) => matchesSearchText(name, 'party'))
const duration = performance.now() - start
expect(matches).toHaveLength(tagNames.length)
// The previous implementation took roughly 500 ms for this workload.
expect(duration).toBeLessThan(250)
})
})
+9 -2
View File
@@ -1,10 +1,17 @@
import { normalizeSync } from 'normalize-diacritics'
import { diacritics } from 'normalize-diacritics/diacritics'
export type SearchTextValue =
string | number | boolean | bigint | null | undefined
export function normalizeSearchText(value: SearchTextValue): string {
return normalizeSync(String(value ?? '')).toLocaleLowerCase()
const normalized = diacritics.reduce(
(text, replacement) => {
return text.replace(replacement.diacritics, replacement.letter)
},
String(value ?? '')
)
return normalized.toLocaleLowerCase()
}
export function matchesSearchText(
+1 -1
View File
@@ -8,7 +8,7 @@ export const environment = {
apiVersion: '10', // match src/paperless/settings.py
appTitle: DEFAULT_APP_TITLE,
tag: 'prod',
version: '3.0.3',
version: '3.0.2',
webSocketHost: window.location.host,
webSocketProtocol: window.location.protocol == 'https:' ? 'wss:' : 'ws:',
webSocketBaseUrl: base_url.pathname + 'ws/',
+3 -2
View File
@@ -160,13 +160,14 @@ def should_produce_archive(
_log.debug("Archive: yes — image document, ARCHIVE_FILE_GENERATION=auto")
return True
if mime_type == "application/pdf":
if is_tagged_pdf(document_path):
text = extract_pdf_text(document_path)
has_text = text is not None and len(text) > 0
if has_text and is_tagged_pdf(document_path):
_log.debug(
"Archive: no — born-digital PDF (structure tags detected),"
" ARCHIVE_FILE_GENERATION=auto",
)
return False
text = extract_pdf_text(document_path)
if text is None or len(text) <= PDF_TEXT_MIN_LENGTH:
_log.debug(
"Archive: yes — scanned PDF (text_length=%d%d),"
+27
View File
@@ -37,6 +37,7 @@ from drf_spectacular.utils import extend_schema_field
from guardian.utils import get_group_obj_perms_model
from guardian.utils import get_user_obj_perms_model
from rest_framework import serializers
from rest_framework.filters import BaseFilterBackend
from rest_framework.filters import OrderingFilter
from rest_framework_guardian.filters import ObjectPermissionsFilter
@@ -50,6 +51,7 @@ from documents.models import ShareLink
from documents.models import ShareLinkBundle
from documents.models import StoragePath
from documents.models import Tag
from documents.permissions import permitted_document_ids
if TYPE_CHECKING:
from collections.abc import Callable
@@ -1038,6 +1040,31 @@ class ObjectOwnedOrGrantedPermissionsFilter(ObjectPermissionsFilter):
return objects_with_perms | objects_owned | objects_unowned
class DocumentPermissionsFilter(BaseFilterBackend):
"""
A filter backend limiting Document results to those the requesting user
owns, are unowned, or has explicit (user- or group-level) view
permission on.
Unlike ``ObjectOwnedOrGrantedPermissionsFilter``, this does not build an
``objects_with_perms | objects_owned | objects_unowned`` union of
querysets derived from the same base queryset. When that base queryset
already carries independent joins on a multi-valued relation (e.g. two
separate joins from ``tags__id__all`` filtering on two tags), each
OR-ed branch can end up pairing those joins' aliases differently,
letting more than one row out of the join's cross product satisfy the
combined WHERE -- returning the same document more than once. Filtering
via a single ``id__in`` against ``permitted_document_ids`` (a plain
subquery, not a join) sidesteps that entirely and is also cheaper than
guardian's join-based permission check.
"""
def filter_queryset(self, request, queryset, view):
if request.user.is_superuser:
return queryset
return queryset.filter(id__in=permitted_document_ids(request.user))
class ObjectOwnedPermissionsFilter(ObjectPermissionsFilter):
"""
A filter backend that limits results to those where the requesting user
+3 -3
View File
@@ -163,7 +163,7 @@ def set_permissions_for_object(
)
def _permitted_document_ids(user):
def permitted_document_ids(user):
"""
Return a queryset of document IDs the user may view, limited to non-deleted
documents. This intentionally avoids ``get_objects_for_user`` to keep the
@@ -220,7 +220,7 @@ def get_document_count_filter_for_user(user, related_name: str = "documents"):
# Superuser: no permission filtering needed
return Q(**{f"{related_name}__deleted_at__isnull": True})
permitted_ids = _permitted_document_ids(user)
permitted_ids = permitted_document_ids(user)
return Q(**{f"{related_name}__id__in": permitted_ids})
@@ -311,7 +311,7 @@ def annotate_document_count_for_related_queryset(
queryset,
through_model=through_model,
related_object_field=related_object_field,
document_ids=_permitted_document_ids(user),
document_ids=permitted_document_ids(user),
target_field=target_field,
)
+2
View File
@@ -311,6 +311,7 @@ def sanity_check(*, raise_on_error: bool = True) -> str:
def bulk_update_documents(document_ids) -> None:
from documents.search import get_backend
document_ids = list(document_ids)
documents = Document.objects.filter(id__in=document_ids)
for doc in documents:
@@ -331,6 +332,7 @@ def bulk_update_documents(document_ids) -> None:
if ai_config.llm_index_enabled:
update_llm_index(
rebuild=False,
document_ids=document_ids,
)
+88
View File
@@ -14,6 +14,7 @@ from unittest import mock
import celery
from dateutil import parser
from django.conf import settings
from django.contrib.auth.models import Group
from django.contrib.auth.models import Permission
from django.contrib.auth.models import User
from django.core import mail
@@ -47,6 +48,8 @@ from documents.models import Workflow
from documents.models import WorkflowAction
from documents.models import WorkflowTrigger
from documents.signals.handlers import run_workflows
from documents.tests.factories import DocumentFactory
from documents.tests.factories import TagFactory
from documents.tests.utils import ConsumeTaskMixin
from documents.tests.utils import DirectoriesMixin
from documents.tests.utils import read_streaming_response
@@ -1212,6 +1215,91 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
[u1_doc1.id],
)
def test_document_owned_and_group_shared_not_duplicated_when_filtering_by_tags(
self,
) -> None:
"""
GIVEN:
- A document owned by a user and also shared with a group the user belongs to
WHEN:
- The user filters documents by more than one tag (tags__id__all)
THEN:
- The document is returned exactly once, not once per permission path
(regression test for https://github.com/paperless-ngx/paperless-ngx/issues/13331)
"""
user = User.objects.create_user("user1")
user.user_permissions.add(*Permission.objects.filter(codename="view_document"))
group = Group.objects.create(name="group1")
user.groups.add(group)
tag1 = TagFactory()
tag2 = TagFactory()
doc = DocumentFactory(title="shared", owner=user)
doc.tags.add(tag1, tag2)
assign_perm("view_document", group, doc)
self.client.force_authenticate(user=user)
response = self.client.get(
f"/api/documents/?tags__id__all={tag1.id},{tag2.id}",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data["count"], 1)
self.assertEqual(response.data["results"][0]["id"], doc.id)
def test_document_permission_filter_excludes_unrelated_documents(self) -> None:
"""
GIVEN:
- A document owned by one user, with no permission granted to another user
WHEN:
- The unrelated user requests the document list
THEN:
- The document does not appear in their results
"""
owner = User.objects.create_user("owner1")
stranger = User.objects.create_user("stranger1")
stranger.user_permissions.add(
*Permission.objects.filter(codename="view_document"),
)
DocumentFactory(title="private", owner=owner)
self.client.force_authenticate(user=stranger)
response = self.client.get("/api/documents/")
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data["count"], 0)
def test_document_permission_filter_only_visible_to_group_members(self) -> None:
"""
GIVEN:
- A document shared with a group via object permissions
WHEN:
- A group member and a non-member both request the document list
THEN:
- Only the group member sees the document
"""
owner = User.objects.create_user("owner2")
member = User.objects.create_user("member1")
non_member = User.objects.create_user("nonmember1")
for u in (member, non_member):
u.user_permissions.add(*Permission.objects.filter(codename="view_document"))
group = Group.objects.create(name="group2")
member.groups.add(group)
doc = DocumentFactory(title="shared2", owner=owner)
assign_perm("view_document", group, doc)
self.client.force_authenticate(user=member)
response = self.client.get("/api/documents/")
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data["count"], 1)
self.assertEqual(response.data["results"][0]["id"], doc.id)
self.client.force_authenticate(user=non_member)
response = self.client.get("/api/documents/")
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data["count"], 0)
def test_pagination_results(self) -> None:
"""
GIVEN:
+10 -6
View File
@@ -166,24 +166,28 @@ class TestShouldProduceArchive:
mocker: MockerFixture,
settings,
) -> None:
"""Tagged PDFs (e.g. Word exports) are treated as born-digital regardless of text length."""
"""Tagged PDFs (e.g. Word exports) with real text are treated as born-digital, even below PDF_TEXT_MIN_LENGTH."""
settings.ARCHIVE_FILE_GENERATION = "auto"
mocker.patch("documents.consumer.is_tagged_pdf", return_value=True)
mocker.patch("documents.consumer.extract_pdf_text", return_value="tiny")
parser = _parser_instance(can_produce=True, requires_rendition=False)
assert (
should_produce_archive(parser, "application/pdf", Path("/tmp/doc.pdf"))
is False
)
def test_tagged_pdf_does_not_call_pdftotext(
def test_tagged_pdf_without_text_produces_archive(
self,
mocker: MockerFixture,
settings,
) -> None:
"""When a PDF is tagged, pdftotext is not invoked (fast path)."""
"""A tagged PDF with no actual extractable text (e.g. some scanner firmware) is not
trusted as born-digital the tag alone must not bypass OCR."""
settings.ARCHIVE_FILE_GENERATION = "auto"
mocker.patch("documents.consumer.is_tagged_pdf", return_value=True)
mock_extract = mocker.patch("documents.consumer.extract_pdf_text")
mocker.patch("documents.consumer.extract_pdf_text", return_value=None)
parser = _parser_instance(can_produce=True, requires_rendition=False)
should_produce_archive(parser, "application/pdf", Path("/tmp/doc.pdf"))
mock_extract.assert_not_called()
assert (
should_produce_archive(parser, "application/pdf", Path("/tmp/doc.pdf"))
is True
)
+6 -2
View File
@@ -401,6 +401,10 @@ class TestAIIndex(DirectoriesMixin, TestCase):
"documents.tasks.update_llm_index",
) as update_llm_index,
):
tasks.bulk_update_documents([doc.pk for doc in docs])
doc_ids = [doc.pk for doc in docs]
tasks.bulk_update_documents(doc_ids)
self.assertEqual(update_document_in_llm_index.apply_async.call_count, 0)
update_llm_index.assert_called_once()
update_llm_index.assert_called_once_with(
rebuild=False,
document_ids=doc_ids,
)
+31
View File
@@ -625,6 +625,37 @@ class TestAIChatStreamingView(DirectoriesMixin, TestCase):
)
self.assertEqual(response.status_code, 200)
self.assertEqual(response["Content-Type"], "text/event-stream")
mock_stream_chat.assert_called_once_with(
query_str="question",
documents=[self.document],
output_language=None,
)
@patch("documents.views.stream_chat_with_documents")
@patch("documents.views.get_objects_for_user_owner_aware")
@override_settings(AI_ENABLED=True)
def test_post_uses_user_display_language(
self,
mock_get_objects,
mock_stream_chat,
) -> None:
UiSettings.objects.create(user=self.user, settings={"language": "de-de"})
self.grant_view_document_permission()
mock_get_objects.return_value = [self.document]
mock_stream_chat.return_value = iter([b"data"])
response = self.client.post(
self.ENDPOINT,
data='{"q": "question"}',
content_type="application/json",
)
self.assertEqual(response.status_code, 200)
mock_stream_chat.assert_called_once_with(
query_str="question",
documents=[self.document],
output_language="de-de",
)
@patch("documents.views.stream_chat_with_documents")
@override_settings(AI_ENABLED=True)
+24 -12
View File
@@ -133,6 +133,7 @@ from documents.file_handling import format_filename
from documents.filters import CorrespondentFilterSet
from documents.filters import CustomFieldFilterSet
from documents.filters import DocumentFilterSet
from documents.filters import DocumentPermissionsFilter
from documents.filters import DocumentsOrderingFilter
from documents.filters import DocumentTypeFilterSet
from documents.filters import ObjectOwnedOrGrantedPermissionsFilter
@@ -653,6 +654,20 @@ class TagViewSet(PermissionsAwareDocumentCountMixin, ModelViewSet[Tag]):
update_document_parent_tags(tag, new_parent)
def _get_llm_output_language(ai_config: AIConfig, request) -> str | None:
output_language = ai_config.llm_output_language
if (
not output_language
and hasattr(request.user, "ui_settings")
and isinstance(
request.user.ui_settings.settings,
dict,
)
):
output_language = request.user.ui_settings.settings.get("language")
return output_language
@extend_schema_view(**generate_object_with_permissions_schema(DocumentTypeSerializer))
class DocumentTypeViewSet(
PermissionsAwareDocumentCountMixin,
@@ -972,7 +987,7 @@ class DocumentViewSet(
DjangoFilterBackend,
SearchFilter,
DocumentsOrderingFilter,
ObjectOwnedOrGrantedPermissionsFilter,
DocumentPermissionsFilter,
)
filterset_class = DocumentFilterSet
search_fields = ("title", "correspondent__name", "effective_content")
@@ -1514,16 +1529,7 @@ class DocumentViewSet(
if not ai_config.ai_enabled:
return HttpResponseBadRequest("AI is required for this feature")
output_language = ai_config.llm_output_language
if (
not output_language
and hasattr(request.user, "ui_settings")
and isinstance(
request.user.ui_settings.settings,
dict,
)
):
output_language = request.user.ui_settings.settings.get("language") or None
output_language = _get_llm_output_language(ai_config=ai_config, request=request)
llm_cache_backend = (
f"{ai_config.llm_backend}:{output_language}"
if output_language
@@ -2265,8 +2271,14 @@ class ChatStreamingView(GenericAPIView[Any]):
Document,
)
output_language = _get_llm_output_language(ai_config=ai_config, request=request)
response = StreamingHttpResponse(
stream_chat_with_documents(query_str=question, documents=documents),
stream_chat_with_documents(
query_str=question,
documents=documents,
output_language=output_language,
),
content_type="text/event-stream",
)
return response
+20 -20
View File
@@ -2,7 +2,7 @@ msgid ""
msgstr ""
"Project-Id-Version: paperless-ngx\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2026-07-25 07:12+0000\n"
"POT-Creation-Date: 2026-07-27 19:34+0000\n"
"PO-Revision-Date: 2022-02-17 04:17\n"
"Last-Translator: \n"
"Language-Team: English\n"
@@ -21,39 +21,39 @@ msgstr ""
msgid "Documents"
msgstr ""
#: documents/filters.py:470
#: documents/filters.py:472
msgid "Value must be valid JSON."
msgstr ""
#: documents/filters.py:489
#: documents/filters.py:491
msgid "Invalid custom field query expression"
msgstr ""
#: documents/filters.py:499
#: documents/filters.py:501
msgid "Invalid expression list. Must be nonempty."
msgstr ""
#: documents/filters.py:520
#: documents/filters.py:522
msgid "Invalid logical operator {op!r}"
msgstr ""
#: documents/filters.py:534
#: documents/filters.py:536
msgid "Maximum number of query conditions exceeded."
msgstr ""
#: documents/filters.py:598
#: documents/filters.py:600
msgid "{name!r} is not a valid custom field."
msgstr ""
#: documents/filters.py:635
#: documents/filters.py:637
msgid "{data_type} does not support query expr {expr!r}."
msgstr ""
#: documents/filters.py:750 documents/models.py:136
#: documents/filters.py:752 documents/models.py:136
msgid "Maximum nesting depth exceeded."
msgstr ""
#: documents/filters.py:1067
#: documents/filters.py:1094
msgid "Custom field not found"
msgstr ""
@@ -1352,7 +1352,7 @@ msgid "workflow runs"
msgstr ""
#: documents/serialisers.py:522 documents/serialisers.py:874
#: documents/serialisers.py:2763 documents/views.py:298 documents/views.py:2541
#: documents/serialisers.py:2763 documents/views.py:299 documents/views.py:2553
#: paperless_mail/serialisers.py:155
msgid "Insufficient permissions."
msgstr ""
@@ -1393,7 +1393,7 @@ msgstr ""
msgid "Duplicate document identifiers are not allowed."
msgstr ""
#: documents/serialisers.py:2849 documents/views.py:4500
#: documents/serialisers.py:2849 documents/views.py:4512
#, python-format
msgid "Documents not found: %(ids)s"
msgstr ""
@@ -1661,36 +1661,36 @@ msgstr ""
msgid "Unable to parse URI {value}"
msgstr ""
#: documents/views.py:291 documents/views.py:2538
#: documents/views.py:292 documents/views.py:2550
msgid "Invalid more_like_id"
msgstr ""
#: documents/views.py:1556
#: documents/views.py:1562
msgid "Invalid AI configuration."
msgstr ""
#: documents/views.py:1565
#: documents/views.py:1571
msgid "AI backend request timed out."
msgstr ""
#: documents/views.py:2363 documents/views.py:2684
#: documents/views.py:2375 documents/views.py:2696
msgid "Specify only one of text, title_search, query, or more_like_id."
msgstr ""
#: documents/views.py:4512
#: documents/views.py:4524
#, python-format
msgid "Insufficient permissions to share document %(id)s."
msgstr ""
#: documents/views.py:4558
#: documents/views.py:4570
msgid "Bundle is already being processed."
msgstr ""
#: documents/views.py:4619
#: documents/views.py:4631
msgid "The share link bundle is still being prepared. Please try again later."
msgstr ""
#: documents/views.py:4629
#: documents/views.py:4641
msgid "The share link bundle is unavailable."
msgstr ""
@@ -5,12 +5,30 @@ from django.db import migrations
from django.db import models
def clamp_applicationconfiguration_integer_fields(apps, schema_editor):
# Clamp barcode_dpi, barcode_max_pages, image_dpi and pages because of
# PositiveIntegerField --> PositiveSmallIntegerField
ApplicationConfiguration = apps.get_model("paperless", "ApplicationConfiguration")
ApplicationConfiguration.objects.filter(barcode_dpi__gt=32767).update(
barcode_dpi=32767,
)
ApplicationConfiguration.objects.filter(barcode_max_pages__gt=32767).update(
barcode_max_pages=32767,
)
ApplicationConfiguration.objects.filter(image_dpi__gt=32767).update(image_dpi=32767)
ApplicationConfiguration.objects.filter(pages__gt=32767).update(pages=32767)
class Migration(migrations.Migration):
dependencies = [
("paperless", "0006_applicationconfiguration_barcode_tag_split"),
]
operations = [
migrations.RunPython(
clamp_applicationconfiguration_integer_fields,
migrations.RunPython.noop,
),
migrations.AlterField(
model_name="applicationconfiguration",
name="barcode_dpi",
+4 -2
View File
@@ -510,8 +510,10 @@ class RasterisedDocumentParser:
if mime_type == "application/pdf":
text_original = self.extract_text(None, document_path)
original_has_text = is_tagged_pdf(document_path, log=self.log) or (
text_original is not None and len(text_original) > PDF_TEXT_MIN_LENGTH
has_text = text_original is not None and len(text_original) > 0
original_has_text = has_text and (
is_tagged_pdf(document_path, log=self.log)
or len(text_original) > PDF_TEXT_MIN_LENGTH
)
else:
text_original = None
@@ -906,6 +906,34 @@ class TestSkipArchive:
assert tesseract_parser.archive_path is None
assert tesseract_parser.get_text()
def test_tagged_pdf_without_text_does_not_skip_ocr(
self,
mocker: MockerFixture,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
) -> None:
"""
GIVEN:
- A PDF that reports itself as tagged (/MarkInfo /Marked true) but
has no actual extractable text (some scanner firmware produces
this see GitHub issue #13349)
- Mode: auto, produce_archive=False
WHEN:
- Document is parsed
THEN:
- The tag alone is not trusted as "has text"; OCRmyPDF still runs
"""
tesseract_parser.settings.mode = ModeChoices.AUTO
mocker.patch("paperless.parsers.tesseract.is_tagged_pdf", return_value=True)
mocker.patch.object(tesseract_parser, "extract_text", return_value=None)
mock_ocr = mocker.patch("ocrmypdf.ocr")
tesseract_parser.parse(
tesseract_samples_dir / "multi-page-images.pdf",
"application/pdf",
produce_archive=False,
)
mock_ocr.assert_called()
def test_tagged_pdf_produces_pdfa_archive_without_ocr(
self,
tesseract_parser: RasterisedDocumentParser,
+1 -1
View File
@@ -1,6 +1,6 @@
from typing import Final
__version__: Final[tuple[int, int, int]] = (3, 0, 3)
__version__: Final[tuple[int, int, int]] = (3, 0, 2)
# Version string like X.Y.Z
__full_version_str__: Final[str] = ".".join(map(str, __version__))
# Version string like X.Y
+2
View File
@@ -2,6 +2,8 @@ from pydantic import BaseModel
class DocumentClassifierSchema(BaseModel):
"""Schema for document classification suggestions."""
title: str
tags: list[str]
correspondents: list[str]
+27 -4
View File
@@ -28,11 +28,22 @@ CHAT_PROMPT_TMPL = (
"---------------------\n"
"Using only the context above, answer the query. "
"Do not use prior knowledge.\n"
"{output_language_line}"
"Query: {query_str}\n"
"Answer:"
)
def _build_chat_prompt(output_language: str | None) -> str:
output_language_line = (
f"Respond in {output_language}.\n" if output_language is not None else ""
)
return CHAT_PROMPT_TMPL.replace(
"{output_language_line}",
output_language_line,
)
def _build_document_reference(
document: Document,
title: str | None = None,
@@ -79,15 +90,27 @@ def _format_chat_metadata_trailer(references: list[dict[str, int | str]]) -> str
)
def stream_chat_with_documents(query_str: str, documents: list[Document]):
def stream_chat_with_documents(
query_str: str,
documents: list[Document],
output_language: str | None = None,
):
try:
yield from _stream_chat_with_documents(query_str, documents)
yield from _stream_chat_with_documents(
query_str,
documents,
output_language=output_language,
)
except Exception as e:
logger.exception("Failed to stream document chat response: %s", e)
yield CHAT_ERROR_MESSAGE
def _stream_chat_with_documents(query_str: str, documents: list[Document]):
def _stream_chat_with_documents(
query_str: str,
documents: list[Document],
output_language: str | None = None,
):
if not documents:
yield CHAT_NO_CONTENT_MESSAGE
return
@@ -125,7 +148,7 @@ def _stream_chat_with_documents(query_str: str, documents: list[Document]):
references = _get_document_references(documents, top_nodes)
prompt_template = PromptTemplate(template=CHAT_PROMPT_TMPL)
prompt_template = PromptTemplate(template=_build_chat_prompt(output_language))
response_synthesizer = get_response_synthesizer(
llm=client.llm,
prompt_helper=get_rag_prompt_helper(
+1 -2
View File
@@ -7,7 +7,6 @@ if TYPE_CHECKING:
from llama_index.core.base.embeddings.base import BaseEmbedding
from documents.models import Document
from documents.models import Note
from paperless.config import AIConfig
from paperless.models import LLMEmbeddingBackend
from paperless.network import PinnedHostAsyncHTTPTransport
@@ -126,7 +125,7 @@ def build_llm_index_text(doc: Document) -> str:
# prepend. Notes and Custom Fields stay in the body: Notes can be long free
# text, Custom Fields are dynamic in count and best kept in the embedding.
lines = [
f"Notes: {','.join([str(c.note) for c in Note.objects.filter(document=doc)])}",
f"Notes: {','.join([str(c.note) for c in doc.notes.all()])}",
]
for instance in doc.custom_fields.all():
+37 -10
View File
@@ -5,6 +5,7 @@ from datetime import timedelta
from typing import TYPE_CHECKING
from django.conf import settings
from django.core.exceptions import ObjectDoesNotExist
from django.utils import timezone
from filelock import FileLock
from filelock import ReadWriteLock
@@ -167,6 +168,19 @@ def write_store(embed_model_name: str | None = None):
yield store
def _safe_related_name(document: Document, field: str) -> str | None:
"""
Returns the ``name`` of a related object (correspondent, document_type,
storage_path), or None if the FK is unset or points at a row that has
since been deleted (e.g. concurrently with this call).
"""
try:
related = getattr(document, field)
except ObjectDoesNotExist:
return None
return related.name if related else None
def build_document_node(
document: Document,
*,
@@ -180,14 +194,10 @@ def build_document_node(
"document_id": str(document.id),
"title": document.title,
"tags": [t.name for t in document.tags.all()],
"correspondent": document.correspondent.name
if document.correspondent
else None,
"document_type": document.document_type.name
if document.document_type
else None,
"correspondent": _safe_related_name(document, "correspondent"),
"document_type": _safe_related_name(document, "document_type"),
"filename": document.filename,
"storage_path": document.storage_path.name if document.storage_path else None,
"storage_path": _safe_related_name(document, "storage_path"),
"archive_serial_number": document.archive_serial_number,
"created": document.created.isoformat() if document.created else None,
"added": document.added.isoformat() if document.added else None,
@@ -318,8 +328,16 @@ def update_llm_index(
*,
iter_wrapper: IterWrapper[Document] = identity,
rebuild=False,
document_ids: Iterable[int] | None = None,
) -> str:
"""Rebuild or incrementally update the LLM index."""
"""Rebuild or incrementally update the LLM index.
``document_ids``, when given, scopes an incremental update to just those
documents instead of scanning the whole library -- callers that already
know which documents changed (e.g. a bulk edit) should pass this to avoid
an O(library size) scan per call. Ignored whenever a rebuild actually
happens, since a rebuild always covers the whole library regardless.
"""
with write_store() as store:
try:
with _exclude_readers():
@@ -335,7 +353,11 @@ def update_llm_index(
"LLM index migration requires re-embedding; forcing rebuild.",
)
rebuild = True
documents = Document.objects.all()
documents = Document.objects.select_related(
"correspondent",
"document_type",
"storage_path",
).prefetch_related("tags", "notes", "custom_fields__field")
no_documents = not documents.exists()
# Fast exit before touching config: nothing to index and no existing index.
@@ -369,9 +391,14 @@ def update_llm_index(
store.add(nodes)
msg = "LLM index rebuilt successfully."
else:
scoped_documents = (
documents.filter(id__in=document_ids)
if document_ids is not None
else documents
)
existing = store.get_modified_times()
changed = 0
for document in iter_wrapper(documents):
for document in iter_wrapper(scoped_documents):
doc_id = str(document.id)
if existing.get(doc_id) == document.modified.isoformat():
continue
@@ -4,11 +4,18 @@ from unittest.mock import patch
import pytest
import pytest_mock
from django.db import connection
from django.test import override_settings
from django.test.utils import CaptureQueriesContext
from django.utils import timezone
from llama_index.core.schema import MetadataMode
from documents.models import Correspondent
from documents.models import CustomField
from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import DocumentType
from documents.models import Note
from documents.models import PaperlessTask
from documents.signals import document_consumption_finished
from documents.signals import document_updated
@@ -95,6 +102,38 @@ def test_build_document_node_structured_fields_in_metadata(
assert "modified" in node.metadata
@pytest.mark.django_db
def test_build_document_node_survives_concurrently_deleted_correspondent(
real_document: Document,
) -> None:
"""Regression test for #13314.
If a document's correspondent (or document type) is deleted after the
in-memory Document instance was loaded but before build_document_node
resolves the relation, accessing the FK must not raise -- it should
behave like an unset FK and produce None in the metadata instead of
aborting the whole indexing pass.
"""
correspondent = Correspondent.objects.create(name="Stale Correspondent")
document_type = DocumentType.objects.create(name="Stale Type")
real_document.correspondent = correspondent
real_document.document_type = document_type
real_document.save()
# Re-fetch to get an instance whose correspondent/document_type relations
# are unresolved (not yet cached), mirroring a task that loaded the
# document before the concurrent deletion below.
stale_document = Document.objects.get(pk=real_document.pk)
correspondent.delete()
document_type.delete()
nodes = indexing.build_document_node(stale_document)
assert len(nodes) > 0
assert nodes[0].metadata["correspondent"] is None
assert nodes[0].metadata["document_type"] is None
@pytest.mark.django_db
def test_build_document_node_excludes_document_id_from_llm_context(
real_document: Document,
@@ -163,6 +202,8 @@ def test_update_llm_index(
mock_queryset = MagicMock()
mock_queryset.exists.return_value = True
mock_queryset.__iter__.return_value = iter([real_document])
mock_queryset.select_related.return_value = mock_queryset
mock_queryset.prefetch_related.return_value = mock_queryset
mock_all.return_value = mock_queryset
build_document_node.return_value = []
indexing.update_llm_index(rebuild=True)
@@ -182,6 +223,8 @@ def test_update_llm_index_rebuilds_on_model_name_change(
mock_queryset = MagicMock()
mock_queryset.exists.return_value = True
mock_queryset.__iter__.return_value = iter([real_document])
mock_queryset.select_related.return_value = mock_queryset
mock_queryset.prefetch_related.return_value = mock_queryset
mock_all.return_value = mock_queryset
with patch(
"paperless_ai.indexing.get_configured_model_name",
@@ -194,6 +237,8 @@ def test_update_llm_index_rebuilds_on_model_name_change(
mock_queryset = MagicMock()
mock_queryset.exists.return_value = True
mock_queryset.__iter__.return_value = iter([real_document])
mock_queryset.select_related.return_value = mock_queryset
mock_queryset.prefetch_related.return_value = mock_queryset
mock_all.return_value = mock_queryset
with patch(
"paperless_ai.indexing.get_configured_model_name",
@@ -224,6 +269,8 @@ def test_update_llm_index_partial_update(
mock_queryset = MagicMock()
mock_queryset.exists.return_value = True
mock_queryset.__iter__.return_value = iter([real_document, doc2])
mock_queryset.select_related.return_value = mock_queryset
mock_queryset.prefetch_related.return_value = mock_queryset
mock_all.return_value = mock_queryset
indexing.update_llm_index(rebuild=True)
@@ -252,6 +299,52 @@ def test_update_llm_index_partial_update(
assert store.table_exists(), (
"Expected the vector store table to exist after incremental update"
)
before = store.get_modified_times()
# new doc, also touched by the scoped update below
doc4 = DocumentFactory.create(title="Test Document 4", added=timezone.now())
# A further edit, scoped via document_ids to doc3 + doc4 -- doc2 must be
# left exactly as it was, proving document_ids restricts the scan
# instead of falling back to the whole library.
doc3.modified = timezone.now()
doc3.save()
doc4.modified = timezone.now()
doc4.save()
# Give both scoped documents a note and a custom field: build_llm_index_text
# reads both per document, so without the notes/custom_fields__field
# prefetch on scoped_documents, each additional document adds 3 more
# queries (N+1 regression) instead of the query count staying flat.
custom_field = CustomField.objects.create(
name="Priority",
data_type=CustomField.FieldDataType.STRING,
)
for doc in (doc3, doc4):
Note.objects.create(document=doc, note=f"a note on {doc.title}")
CustomFieldInstance.objects.create(
document=doc,
field=custom_field,
value_text="high",
)
with CaptureQueriesContext(connection) as ctx:
result = indexing.update_llm_index(
rebuild=False,
document_ids=[doc3.pk, doc4.pk],
)
assert result == "LLM index updated successfully."
# Notes/custom fields are prefetched in one batch query each (plus one
# more for custom_fields__field), not re-queried per document -- an N+1
# regression here would scale with document count instead of staying flat
# (7 with the prefetch vs. 10 without it, for these 2 documents).
assert len(ctx.captured_queries) <= 8
with indexing.get_vector_store() as store:
after = store.get_modified_times()
assert after[str(doc3.pk)] == doc3.modified.isoformat()
assert after[str(doc2.pk)] == before[str(doc2.pk)]
@pytest.mark.django_db
+21
View File
@@ -12,6 +12,7 @@ from paperless_ai import chat
from paperless_ai import indexing
from paperless_ai.chat import CHAT_ERROR_MESSAGE
from paperless_ai.chat import CHAT_METADATA_DELIMITER
from paperless_ai.chat import _build_chat_prompt
from paperless_ai.chat import stream_chat_with_documents
@@ -59,6 +60,26 @@ def assert_chat_output(
}
@pytest.mark.parametrize(
("output_language", "expected_language_line"),
[
(None, ""),
("de-de", "Respond in de-de.\n"),
],
)
def test_build_chat_prompt(
output_language,
expected_language_line,
) -> None:
prompt = _build_chat_prompt(output_language)
assert "{output_language_line}" not in prompt
assert (
prompt.split("Do not use prior knowledge.\n", maxsplit=1)[1]
== f"{expected_language_line}Query: {{query_str}}\nAnswer:"
)
@pytest.mark.django_db
def test_stream_chat_with_one_document_retrieval(
mock_document,
+19 -21
View File
@@ -54,6 +54,7 @@ def mock_document():
cf2.field.name = "Field2"
cf2.value = "Value2"
doc.custom_fields.all = MagicMock(return_value=[cf1, cf2])
doc.notes.all = MagicMock(return_value=[])
return doc
@@ -219,28 +220,26 @@ def test_get_configured_model_name_explicit_overrides_default(mock_ai_config):
def test_build_llm_index_text(mock_document):
with patch("documents.models.Note.objects.filter") as mock_notes_filter:
mock_notes_filter.return_value = [
MagicMock(note="Note1"),
MagicMock(note="Note2"),
]
mock_document.notes.all = MagicMock(
return_value=[MagicMock(note="Note1"), MagicMock(note="Note2")],
)
result = build_llm_index_text(mock_document)
result = build_llm_index_text(mock_document)
# Structured fields live in node.metadata for LLM context -- not body text
assert "Title: Test Title" not in result
assert "Created: 2023-01-01" not in result
assert "Tags: Tag1, Tag2" not in result
assert "Document Type: Invoice" not in result
assert "Correspondent: Test Correspondent" not in result
assert "Filename:" not in result
assert "Storage Path:" not in result
assert "Archive Serial Number:" not in result
# Structured fields live in node.metadata for LLM context -- not body text
assert "Title: Test Title" not in result
assert "Created: 2023-01-01" not in result
assert "Tags: Tag1, Tag2" not in result
assert "Document Type: Invoice" not in result
assert "Correspondent: Test Correspondent" not in result
assert "Filename:" not in result
assert "Storage Path:" not in result
assert "Archive Serial Number:" not in result
# Fields without a metadata equivalent stay in body text
assert "Notes: Note1,Note2" in result
assert "Content:\n\nThis is the document content." in result
assert "Custom Field - Field1: Value1\nCustom Field - Field2: Value2" in result
# Fields without a metadata equivalent stay in body text
assert "Notes: Note1,Note2" in result
assert "Content:\n\nThis is the document content." in result
assert "Custom Field - Field1: Value1\nCustom Field - Field2: Value2" in result
def test_build_llm_index_text_normalizes_ocr_punctuation_runs(mock_document):
@@ -250,8 +249,7 @@ def test_build_llm_index_text_normalizes_ocr_punctuation_runs(mock_document):
"Keep short punctuation like INV-100 and ellipses..."
)
with patch("documents.models.Note.objects.filter", return_value=[]):
result = build_llm_index_text(mock_document)
result = build_llm_index_text(mock_document)
assert "Introduction 7" in result
assert "Hardware Limitation 9" in result
@@ -10,6 +10,24 @@ def clamp_mailrule_maximum_age(apps, schema_editor):
MailRule.objects.filter(maximum_age__gt=32767).update(maximum_age=32767)
def clamp_mailrule_order(apps, schema_editor):
# The order field of MailRule is only used for relative sorting
# (account.rules.order_by("order")), so out-of-range values must be
# renumbered by rank rather than clamped to a single value, which
# would collapse distinct rules to the same order.
MailRule = apps.get_model("paperless_mail", "MailRule")
if (
MailRule.objects.filter(order__gt=32767).exists()
or MailRule.objects.filter(
order__lt=-32768,
).exists()
): # pragma: no cover
for index, rule_id in enumerate(
MailRule.objects.order_by("order", "pk").values_list("pk", flat=True),
):
MailRule.objects.filter(pk=rule_id).update(order=index)
class Migration(migrations.Migration):
# The data update must commit before PostgreSQL rewrites the table, otherwise
# pending foreign-key trigger events can block the subsequent ALTER TABLE.
@@ -135,6 +153,10 @@ class Migration(migrations.Migration):
verbose_name="maximum age",
),
),
migrations.RunPython(
clamp_mailrule_order,
migrations.RunPython.noop,
),
migrations.AlterField(
model_name="mailrule",
name="order",
Generated
+1 -1
View File
@@ -2880,7 +2880,7 @@ wheels = [
[[package]]
name = "paperless-ngx"
version = "3.0.3"
version = "3.0.2"
source = { virtual = "." }
dependencies = [
{ name = "azure-ai-documentintelligence", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },