mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-08-27 05:03:20 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a13e8a59dd | ||
|
|
cafed919a1 | ||
|
|
549afde1fd | ||
|
|
a8576d827e | ||
|
|
7330c4d9cb |
@@ -667,6 +667,35 @@ The action takes no options, its presence is what enables remote OCR for a match
|
||||
If the remote engine is not configured, or does not support the document's file type, the document is
|
||||
processed locally instead and a warning is written to the log.
|
||||
|
||||
##### Apply AI Suggestions {#workflow-action-apply-ai-suggestions}
|
||||
|
||||
"Apply AI Suggestions" actions ask the configured AI service for title and metadata suggestions,
|
||||
the same as the AI suggestions shown on the document detail page, except applied automatically and in bulk.
|
||||
It requires [AI features](configuration.md#ai) to be enabled. You can specify:
|
||||
|
||||
- Which suggestions to apply: title, tags, correspondent, document type, storage path and / or created
|
||||
date. Suggestions for fields you did not select are discarded.
|
||||
- Whether to create missing items. By default only tags, correspondents and document types that
|
||||
already exist are assigned and any other suggestion is dropped. With this enabled, suggested items
|
||||
that do not exist are created. Storage paths are never created.
|
||||
- Whether to overwrite existing values. By default a field is only filled in if it is currently empty.
|
||||
Note that documents almost always already have a title and created date, so if you select those you
|
||||
will usually want to enable this too. Tags are an exception: suggested tags are always added and
|
||||
never replace the document's existing tags.
|
||||
|
||||
The action works with every trigger **except Consumption Started**, because suggestions are made from
|
||||
the document's text, which does not exist until after the document has been processed.
|
||||
|
||||
Because the query to the AI service is slow, the action is queued and runs in the background rather
|
||||
than as part of the workflow run itself. The document is updated once the suggestions come back.
|
||||
|
||||
!!! warning
|
||||
|
||||
Every matching document results in a query to the AI service, which may incur costs and have privacy
|
||||
implications. Queries can be slow, so a workflow matching a large number of documents can occupy the
|
||||
task queue, and delay consumption of new documents, etc. Consider narrowing the trigger filters,
|
||||
running in small batches and / or increasing workers.
|
||||
|
||||
#### Workflow placeholders
|
||||
|
||||
Titles and webhook payloads can be generated by workflows using [Jinja templates](https://jinja.palletsprojects.com/en/3.1.x/templates/).
|
||||
|
||||
+140
-57
@@ -1620,6 +1620,10 @@
|
||||
<context context-type="sourcefile">src/app/components/admin/settings/settings.component.ts</context>
|
||||
<context context-type="linenumber">99</context>
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts</context>
|
||||
<context context-type="linenumber">173</context>
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.html</context>
|
||||
<context context-type="linenumber">174</context>
|
||||
@@ -1651,6 +1655,10 @@
|
||||
<context context-type="sourcefile">src/app/components/admin/settings/settings.component.ts</context>
|
||||
<context context-type="linenumber">100</context>
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts</context>
|
||||
<context context-type="linenumber">177</context>
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.html</context>
|
||||
<context context-type="linenumber">178</context>
|
||||
@@ -1682,6 +1690,10 @@
|
||||
<context context-type="sourcefile">src/app/components/admin/settings/settings.component.ts</context>
|
||||
<context context-type="linenumber">101</context>
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts</context>
|
||||
<context context-type="linenumber">181</context>
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.html</context>
|
||||
<context context-type="linenumber">182</context>
|
||||
@@ -1713,6 +1725,10 @@
|
||||
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
|
||||
<context context-type="linenumber">211</context>
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts</context>
|
||||
<context context-type="linenumber">169</context>
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/input/tags/tags.component.ts</context>
|
||||
<context context-type="linenumber">86</context>
|
||||
@@ -2087,7 +2103,7 @@
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts</context>
|
||||
<context context-type="linenumber">95</context>
|
||||
<context context-type="linenumber">96</context>
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/share-link-bundle-manage-dialog/share-link-bundle-manage-dialog.component.html</context>
|
||||
@@ -2402,7 +2418,7 @@
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts</context>
|
||||
<context context-type="linenumber">80</context>
|
||||
<context context-type="linenumber">81</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="228975554646076615" datatype="html">
|
||||
@@ -2462,7 +2478,7 @@
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts</context>
|
||||
<context context-type="linenumber">122</context>
|
||||
<context context-type="linenumber">123</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="235571817610183244" datatype="html">
|
||||
@@ -2473,7 +2489,7 @@
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts</context>
|
||||
<context context-type="linenumber">84</context>
|
||||
<context context-type="linenumber">85</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="526966086395145275" datatype="html">
|
||||
@@ -2484,7 +2500,7 @@
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts</context>
|
||||
<context context-type="linenumber">76</context>
|
||||
<context context-type="linenumber">77</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="6520092939021340973" datatype="html">
|
||||
@@ -4332,7 +4348,7 @@
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts</context>
|
||||
<context context-type="linenumber">91</context>
|
||||
<context context-type="linenumber">92</context>
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-list/document-list.component.html</context>
|
||||
@@ -5322,7 +5338,7 @@
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts</context>
|
||||
<context context-type="linenumber">137</context>
|
||||
<context context-type="linenumber">138</context>
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html</context>
|
||||
@@ -5954,18 +5970,67 @@
|
||||
<context context-type="linenumber">461,462</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="4231754882471903566" datatype="html">
|
||||
<source>The document will be sent to the configured AI service for suggestions. Consider costs and privacy.</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html</context>
|
||||
<context context-type="linenumber">468,469</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="3473589590476379802" datatype="html">
|
||||
<source>Apply suggestions for</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html</context>
|
||||
<context context-type="linenumber">471,472</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="6859826074028847234" datatype="html">
|
||||
<source>Suggestions for fields that are not selected are discarded.</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html</context>
|
||||
<context context-type="linenumber">476,477</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="1137167921211640884" datatype="html">
|
||||
<source>Create missing items</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html</context>
|
||||
<context context-type="linenumber">486,487</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="449779688394460071" datatype="html">
|
||||
<source>Create suggested tags, correspondents and document types that do not exist yet.</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html</context>
|
||||
<context context-type="linenumber">488,489</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="5192325026233872238" datatype="html">
|
||||
<source>Overwrite existing values</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html</context>
|
||||
<context context-type="linenumber">496,497</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="9010697250406492150" datatype="html">
|
||||
<source>Apply suggestions even if the document already has a value. Tags are always added, never replaced.</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html</context>
|
||||
<context context-type="linenumber">498,499</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="4626030417479279989" datatype="html">
|
||||
<source>Consume Folder</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts</context>
|
||||
<context context-type="linenumber">72</context>
|
||||
<context context-type="linenumber">73</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="3553216189604488439" datatype="html">
|
||||
<source>Modified</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts</context>
|
||||
<context context-type="linenumber">99</context>
|
||||
<context context-type="linenumber">100</context>
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/data/document.ts</context>
|
||||
@@ -5976,63 +6041,63 @@
|
||||
<source>Custom Field</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts</context>
|
||||
<context context-type="linenumber">103</context>
|
||||
<context context-type="linenumber">104</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="8696908693776094667" datatype="html">
|
||||
<source>Consumption Started</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts</context>
|
||||
<context context-type="linenumber">110</context>
|
||||
<context context-type="linenumber">111</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="7858311467093621703" datatype="html">
|
||||
<source>Document Added</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts</context>
|
||||
<context context-type="linenumber">114</context>
|
||||
<context context-type="linenumber">115</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="7955486237346046731" datatype="html">
|
||||
<source>Document Updated</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts</context>
|
||||
<context context-type="linenumber">118</context>
|
||||
<context context-type="linenumber">119</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="5502398334173581061" datatype="html">
|
||||
<source>Assignment</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts</context>
|
||||
<context context-type="linenumber">129</context>
|
||||
<context context-type="linenumber">130</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="6234812824772766804" datatype="html">
|
||||
<source>Removal</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts</context>
|
||||
<context context-type="linenumber">133</context>
|
||||
<context context-type="linenumber">134</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="4206419737792796794" datatype="html">
|
||||
<source>Webhook</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts</context>
|
||||
<context context-type="linenumber">141</context>
|
||||
<context context-type="linenumber">142</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="4824906895380506720" datatype="html">
|
||||
<source>Password removal</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts</context>
|
||||
<context context-type="linenumber">145</context>
|
||||
<context context-type="linenumber">146</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="2048798344356757326" datatype="html">
|
||||
<source>Move to trash</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts</context>
|
||||
<context context-type="linenumber">149</context>
|
||||
<context context-type="linenumber">150</context>
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
@@ -6047,116 +6112,157 @@
|
||||
<source>Remote OCR</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts</context>
|
||||
<context context-type="linenumber">153</context>
|
||||
<context context-type="linenumber">154</context>
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/data/paperless-config.ts</context>
|
||||
<context context-type="linenumber">58</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="4101630920511881483" datatype="html">
|
||||
<source>Apply AI suggestions</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts</context>
|
||||
<context context-type="linenumber">158</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="5701618810648052610" datatype="html">
|
||||
<source>Title</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts</context>
|
||||
<context context-type="linenumber">165</context>
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.html</context>
|
||||
<context context-type="linenumber">167</context>
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-list/document-list.component.html</context>
|
||||
<context context-type="linenumber">218,219</context>
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-list/filter-editor/filter-editor.component.ts</context>
|
||||
<context context-type="linenumber">197</context>
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/data/document.ts</context>
|
||||
<context context-type="linenumber">30</context>
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/data/document.ts</context>
|
||||
<context context-type="linenumber">90</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="5224451373257709572" datatype="html">
|
||||
<source>Created date</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts</context>
|
||||
<context context-type="linenumber">185</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="4522609911791833187" datatype="html">
|
||||
<source>Has any of these tags</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts</context>
|
||||
<context context-type="linenumber">222</context>
|
||||
<context context-type="linenumber">254</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="4166903555074156852" datatype="html">
|
||||
<source>Has all of these tags</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts</context>
|
||||
<context context-type="linenumber">229</context>
|
||||
<context context-type="linenumber">261</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="6624363795312783141" datatype="html">
|
||||
<source>Does not have these tags</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts</context>
|
||||
<context context-type="linenumber">236</context>
|
||||
<context context-type="linenumber">268</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="7168528512669831184" datatype="html">
|
||||
<source>Has any of these correspondents</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts</context>
|
||||
<context context-type="linenumber">243</context>
|
||||
<context context-type="linenumber">275</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="5281365940563983618" datatype="html">
|
||||
<source>Has correspondent</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts</context>
|
||||
<context context-type="linenumber">251</context>
|
||||
<context context-type="linenumber">283</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="6884498632428600393" datatype="html">
|
||||
<source>Does not have correspondents</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts</context>
|
||||
<context context-type="linenumber">259</context>
|
||||
<context context-type="linenumber">291</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="4806713133917046341" datatype="html">
|
||||
<source>Has document type</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts</context>
|
||||
<context context-type="linenumber">267</context>
|
||||
<context context-type="linenumber">299</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="8801397520369995032" datatype="html">
|
||||
<source>Has any of these document types</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts</context>
|
||||
<context context-type="linenumber">275</context>
|
||||
<context context-type="linenumber">307</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="1507843981661822403" datatype="html">
|
||||
<source>Does not have document types</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts</context>
|
||||
<context context-type="linenumber">283</context>
|
||||
<context context-type="linenumber">315</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="4277260190522078330" datatype="html">
|
||||
<source>Has storage path</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts</context>
|
||||
<context context-type="linenumber">291</context>
|
||||
<context context-type="linenumber">323</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="8858580062214623097" datatype="html">
|
||||
<source>Has any of these storage paths</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts</context>
|
||||
<context context-type="linenumber">299</context>
|
||||
<context context-type="linenumber">331</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="6070943364927280151" datatype="html">
|
||||
<source>Does not have storage paths</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts</context>
|
||||
<context context-type="linenumber">307</context>
|
||||
<context context-type="linenumber">339</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="6250799006816371860" datatype="html">
|
||||
<source>Matches custom field query</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts</context>
|
||||
<context context-type="linenumber">315</context>
|
||||
<context context-type="linenumber">347</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="3138206142174978019" datatype="html">
|
||||
<source>Create new workflow</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts</context>
|
||||
<context context-type="linenumber">524</context>
|
||||
<context context-type="linenumber">556</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="5996779210524133604" datatype="html">
|
||||
<source>Edit workflow</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.ts</context>
|
||||
<context context-type="linenumber">528</context>
|
||||
<context context-type="linenumber">560</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="5457837313196342910" datatype="html">
|
||||
@@ -8208,29 +8314,6 @@
|
||||
<context context-type="linenumber">164,165</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="5701618810648052610" datatype="html">
|
||||
<source>Title</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.html</context>
|
||||
<context context-type="linenumber">167</context>
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-list/document-list.component.html</context>
|
||||
<context context-type="linenumber">218,219</context>
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-list/filter-editor/filter-editor.component.ts</context>
|
||||
<context context-type="linenumber">197</context>
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/data/document.ts</context>
|
||||
<context context-type="linenumber">30</context>
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/data/document.ts</context>
|
||||
<context context-type="linenumber">90</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="5114742157723900905" datatype="html">
|
||||
<source>Date created</source>
|
||||
<context-group purpose="location">
|
||||
|
||||
+39
@@ -462,6 +462,45 @@
|
||||
</div>
|
||||
</div>
|
||||
}
|
||||
@case (WorkflowActionType.ApplyAiSuggestions) {
|
||||
<div class="row">
|
||||
<div class="col">
|
||||
<p class="text-muted small" i18n>The document will be sent to the configured AI service for suggestions. Consider costs and privacy.</p>
|
||||
<pngx-input-select
|
||||
i18n-title
|
||||
title="Apply suggestions for"
|
||||
[items]="aiSuggestionFieldOptions"
|
||||
[multiple]="true"
|
||||
formControlName="ai_suggestion_fields"
|
||||
[error]="error?.actions?.[i]?.ai_suggestion_fields"
|
||||
hint="Suggestions for fields that are not selected are discarded."
|
||||
i18n-hint
|
||||
></pngx-input-select>
|
||||
</div>
|
||||
</div>
|
||||
<div class="row">
|
||||
<div class="col-md-6">
|
||||
<pngx-input-switch
|
||||
[horizontal]="true"
|
||||
i18n-title
|
||||
title="Create missing items"
|
||||
formControlName="ai_create_missing"
|
||||
hint="Create suggested tags, correspondents and document types that do not exist yet."
|
||||
i18n-hint
|
||||
></pngx-input-switch>
|
||||
</div>
|
||||
<div class="col-md-6">
|
||||
<pngx-input-switch
|
||||
[horizontal]="true"
|
||||
i18n-title
|
||||
title="Overwrite existing values"
|
||||
formControlName="ai_overwrite_existing"
|
||||
hint="Apply suggestions even if the document already has a value. Tags are always added, never replaced."
|
||||
i18n-hint
|
||||
></pngx-input-switch>
|
||||
</div>
|
||||
</div>
|
||||
}
|
||||
}
|
||||
</div>
|
||||
</ng-template>
|
||||
|
||||
+124
-2
@@ -22,6 +22,7 @@ import {
|
||||
} from 'src/app/data/matching-model'
|
||||
import { Workflow } from 'src/app/data/workflow'
|
||||
import {
|
||||
AISuggestionField,
|
||||
WorkflowAction,
|
||||
WorkflowActionType,
|
||||
} from 'src/app/data/workflow-action'
|
||||
@@ -49,6 +50,7 @@ import { TagsComponent } from '../../input/tags/tags.component'
|
||||
import { TextComponent } from '../../input/text/text.component'
|
||||
import { EditDialogMode } from '../edit-dialog.component'
|
||||
import {
|
||||
AI_SUGGESTION_FIELD_OPTIONS,
|
||||
DOCUMENT_SOURCE_OPTIONS,
|
||||
SCHEDULE_DATE_FIELD_OPTIONS,
|
||||
TriggerFilterType,
|
||||
@@ -239,14 +241,15 @@ describe('WorkflowEditDialogComponent', () => {
|
||||
SCHEDULE_DATE_FIELD_OPTIONS
|
||||
)
|
||||
|
||||
// Email disabled
|
||||
// Email, remote OCR and AI all disabled
|
||||
jest.spyOn(settingsService, 'get').mockReturnValue(false)
|
||||
component.ngOnInit()
|
||||
expect(component.actionTypeOptions).toEqual(
|
||||
WORKFLOW_ACTION_OPTIONS.filter(
|
||||
(a) =>
|
||||
a.id !== WorkflowActionType.Email &&
|
||||
a.id !== WorkflowActionType.RemoteOcr
|
||||
a.id !== WorkflowActionType.RemoteOcr &&
|
||||
a.id !== WorkflowActionType.ApplyAiSuggestions
|
||||
)
|
||||
)
|
||||
})
|
||||
@@ -344,6 +347,125 @@ describe('WorkflowEditDialogComponent', () => {
|
||||
)
|
||||
})
|
||||
|
||||
it('should offer apply AI suggestions unless every trigger is consumption', () => {
|
||||
jest.spyOn(settingsService, 'get').mockReturnValue(true)
|
||||
|
||||
// Consumption runs before the document has been parsed, so there would be
|
||||
// no content to make suggestions from
|
||||
component.object = {
|
||||
name: 'Workflow 1',
|
||||
order: 0,
|
||||
enabled: true,
|
||||
triggers: [{ type: WorkflowTriggerType.Consumption }],
|
||||
actions: [],
|
||||
} as Workflow
|
||||
component.ngOnInit()
|
||||
expect(component.actionTypeOptions.map((a) => a.id)).not.toContain(
|
||||
WorkflowActionType.ApplyAiSuggestions
|
||||
)
|
||||
|
||||
// A second, usable trigger is enough
|
||||
component.object = {
|
||||
name: 'Workflow 2',
|
||||
order: 0,
|
||||
enabled: true,
|
||||
triggers: [
|
||||
{ type: WorkflowTriggerType.Consumption },
|
||||
{ type: WorkflowTriggerType.DocumentAdded },
|
||||
],
|
||||
actions: [],
|
||||
} as Workflow
|
||||
component.ngOnInit()
|
||||
expect(component.actionTypeOptions.map((a) => a.id)).toContain(
|
||||
WorkflowActionType.ApplyAiSuggestions
|
||||
)
|
||||
})
|
||||
|
||||
it('should keep apply AI suggestions listed when an action already uses it', () => {
|
||||
jest.spyOn(settingsService, 'get').mockReturnValue(true)
|
||||
|
||||
// Otherwise changing the trigger would silently blank the selection
|
||||
component.object = {
|
||||
name: 'Workflow 1',
|
||||
order: 0,
|
||||
enabled: true,
|
||||
triggers: [{ type: WorkflowTriggerType.Consumption }],
|
||||
actions: [{ type: WorkflowActionType.ApplyAiSuggestions }],
|
||||
} as Workflow
|
||||
component.ngOnInit()
|
||||
|
||||
expect(component.actionTypeOptions.map((a) => a.id)).toContain(
|
||||
WorkflowActionType.ApplyAiSuggestions
|
||||
)
|
||||
})
|
||||
|
||||
it('should not offer apply AI suggestions when AI is disabled', () => {
|
||||
jest
|
||||
.spyOn(settingsService, 'get')
|
||||
.mockImplementation((key) => key !== SETTINGS_KEYS.AI_ENABLED)
|
||||
|
||||
component.object = {
|
||||
name: 'Workflow 1',
|
||||
order: 0,
|
||||
enabled: true,
|
||||
triggers: [{ type: WorkflowTriggerType.DocumentAdded }],
|
||||
actions: [],
|
||||
} as Workflow
|
||||
component.ngOnInit()
|
||||
|
||||
expect(component.actionTypeOptions.map((a) => a.id)).not.toContain(
|
||||
WorkflowActionType.ApplyAiSuggestions
|
||||
)
|
||||
})
|
||||
|
||||
it('should create form fields for apply AI suggestions options', () => {
|
||||
component.object = {
|
||||
name: 'Workflow 1',
|
||||
order: 0,
|
||||
enabled: true,
|
||||
triggers: [{ type: WorkflowTriggerType.DocumentAdded }],
|
||||
actions: [
|
||||
{
|
||||
type: WorkflowActionType.ApplyAiSuggestions,
|
||||
ai_suggestion_fields: [
|
||||
AISuggestionField.Title,
|
||||
AISuggestionField.Tags,
|
||||
],
|
||||
ai_create_missing: true,
|
||||
ai_overwrite_existing: true,
|
||||
},
|
||||
],
|
||||
} as Workflow
|
||||
component.ngOnInit()
|
||||
|
||||
const action = component.actionFields.at(0)
|
||||
expect(action.get('ai_suggestion_fields').value).toEqual([
|
||||
AISuggestionField.Title,
|
||||
AISuggestionField.Tags,
|
||||
])
|
||||
expect(action.get('ai_create_missing').value).toBeTruthy()
|
||||
expect(action.get('ai_overwrite_existing').value).toBeTruthy()
|
||||
expect(component.aiSuggestionFieldOptions).toEqual(
|
||||
AI_SUGGESTION_FIELD_OPTIONS
|
||||
)
|
||||
})
|
||||
|
||||
it('should default apply AI suggestions options on a new action', () => {
|
||||
component.object = {
|
||||
name: 'Workflow 1',
|
||||
order: 0,
|
||||
enabled: true,
|
||||
triggers: [{ type: WorkflowTriggerType.DocumentAdded }],
|
||||
actions: [],
|
||||
} as Workflow
|
||||
component.addAction()
|
||||
|
||||
const action = component.actionFields.at(component.actionFields.length - 1)
|
||||
expect(action.get('ai_suggestion_fields').value).toEqual([])
|
||||
expect(action.get('ai_create_missing').value).toBeFalsy()
|
||||
expect(action.get('ai_overwrite_existing').value).toBeFalsy()
|
||||
})
|
||||
|
||||
it('should support add and remove triggers and actions', () => {
|
||||
component.object = workflow
|
||||
component.addTrigger()
|
||||
|
||||
+62
@@ -30,6 +30,7 @@ import { StoragePath } from 'src/app/data/storage-path'
|
||||
import { SETTINGS_KEYS } from 'src/app/data/ui-settings'
|
||||
import { Workflow } from 'src/app/data/workflow'
|
||||
import {
|
||||
AISuggestionField,
|
||||
WorkflowAction,
|
||||
WorkflowActionType,
|
||||
} from 'src/app/data/workflow-action'
|
||||
@@ -152,6 +153,37 @@ export const WORKFLOW_ACTION_OPTIONS = [
|
||||
id: WorkflowActionType.RemoteOcr,
|
||||
name: $localize`Remote OCR`,
|
||||
},
|
||||
{
|
||||
id: WorkflowActionType.ApplyAiSuggestions,
|
||||
name: $localize`Apply AI suggestions`,
|
||||
},
|
||||
]
|
||||
|
||||
export const AI_SUGGESTION_FIELD_OPTIONS = [
|
||||
{
|
||||
id: AISuggestionField.Title,
|
||||
name: $localize`Title`,
|
||||
},
|
||||
{
|
||||
id: AISuggestionField.Tags,
|
||||
name: $localize`Tags`,
|
||||
},
|
||||
{
|
||||
id: AISuggestionField.Correspondent,
|
||||
name: $localize`Correspondent`,
|
||||
},
|
||||
{
|
||||
id: AISuggestionField.DocumentType,
|
||||
name: $localize`Document type`,
|
||||
},
|
||||
{
|
||||
id: AISuggestionField.StoragePath,
|
||||
name: $localize`Storage path`,
|
||||
},
|
||||
{
|
||||
id: AISuggestionField.Created,
|
||||
name: $localize`Created date`,
|
||||
},
|
||||
]
|
||||
|
||||
export enum TriggerFilterType {
|
||||
@@ -576,6 +608,24 @@ export class WorkflowEditDialogComponent
|
||||
allowed = allowed.filter((a) => a.id !== WorkflowActionType.RemoteOcr)
|
||||
}
|
||||
|
||||
// Only available after consumption. Unlike remote OCR this is hidden only
|
||||
// once every trigger is consumption, so it stays offered on a workflow
|
||||
// that has no triggers yet.
|
||||
const aiSuggestionsUsable =
|
||||
this.settingsService.get(SETTINGS_KEYS.AI_ENABLED) &&
|
||||
(!formWorkflow?.triggers?.length ||
|
||||
formWorkflow.triggers.some(
|
||||
(trigger) => trigger.type !== WorkflowTriggerType.Consumption
|
||||
) ||
|
||||
formWorkflow.actions?.some(
|
||||
(action) => action.type === WorkflowActionType.ApplyAiSuggestions
|
||||
))
|
||||
if (!aiSuggestionsUsable) {
|
||||
allowed = allowed.filter(
|
||||
(a) => a.id !== WorkflowActionType.ApplyAiSuggestions
|
||||
)
|
||||
}
|
||||
|
||||
if (
|
||||
this.allowedActionTypes?.length === allowed.length &&
|
||||
this.allowedActionTypes.every((a, i) => a.id === allowed[i].id)
|
||||
@@ -1227,6 +1277,11 @@ export class WorkflowEditDialogComponent
|
||||
passwords: new FormControl(
|
||||
this.formatPasswords(action.passwords ?? [])
|
||||
),
|
||||
ai_suggestion_fields: new FormControl(
|
||||
action.ai_suggestion_fields ?? []
|
||||
),
|
||||
ai_create_missing: new FormControl(!!action.ai_create_missing),
|
||||
ai_overwrite_existing: new FormControl(!!action.ai_overwrite_existing),
|
||||
}),
|
||||
{ emitEvent }
|
||||
)
|
||||
@@ -1316,6 +1371,10 @@ export class WorkflowEditDialogComponent
|
||||
return this.actionTypeOptions.find((t) => t.id === type)?.name ?? ''
|
||||
}
|
||||
|
||||
get aiSuggestionFieldOptions() {
|
||||
return AI_SUGGESTION_FIELD_OPTIONS
|
||||
}
|
||||
|
||||
addAction() {
|
||||
if (!this.object) {
|
||||
this.object = Object.assign({}, this.objectForm.value)
|
||||
@@ -1369,6 +1428,9 @@ export class WorkflowEditDialogComponent
|
||||
include_document: false,
|
||||
},
|
||||
passwords: [],
|
||||
ai_suggestion_fields: [],
|
||||
ai_create_missing: false,
|
||||
ai_overwrite_existing: false,
|
||||
}
|
||||
this.object.actions.push(action)
|
||||
this.createActionField(action)
|
||||
|
||||
@@ -8,6 +8,17 @@ export enum WorkflowActionType {
|
||||
PasswordRemoval = 5,
|
||||
MoveToTrash = 6,
|
||||
RemoteOcr = 7,
|
||||
ApplyAiSuggestions = 8,
|
||||
}
|
||||
|
||||
// see src/documents/models.py AISuggestionField
|
||||
export enum AISuggestionField {
|
||||
Title = 'title',
|
||||
Tags = 'tags',
|
||||
Correspondent = 'correspondent',
|
||||
DocumentType = 'document_type',
|
||||
StoragePath = 'storage_path',
|
||||
Created = 'created',
|
||||
}
|
||||
|
||||
export interface WorkflowActionEmail extends ObjectWithId {
|
||||
@@ -102,4 +113,10 @@ export interface WorkflowAction extends ObjectWithId {
|
||||
webhook?: WorkflowActionWebhook
|
||||
|
||||
passwords?: string[]
|
||||
|
||||
ai_suggestion_fields?: AISuggestionField[]
|
||||
|
||||
ai_create_missing?: boolean
|
||||
|
||||
ai_overwrite_existing?: boolean
|
||||
}
|
||||
|
||||
@@ -12,7 +12,6 @@ from typing import TYPE_CHECKING
|
||||
from typing import Any
|
||||
|
||||
from django.contrib.contenttypes.models import ContentType
|
||||
from django.core.exceptions import FieldError
|
||||
from django.db.models import Case
|
||||
from django.db.models import CharField
|
||||
from django.db.models import Count
|
||||
@@ -51,6 +50,7 @@ from documents.models import ShareLinkBundle
|
||||
from documents.models import StoragePath
|
||||
from documents.models import Tag
|
||||
from documents.permissions import permitted_object_ids
|
||||
from documents.versioning import ensure_effective_content
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
@@ -180,14 +180,9 @@ class TitleContentFilter(Filter):
|
||||
logger.warning(
|
||||
"Deprecated document filter parameter 'title_content' used; use `text` instead.",
|
||||
)
|
||||
try:
|
||||
return qs.filter(
|
||||
Q(title__icontains=value) | Q(effective_content__icontains=value),
|
||||
)
|
||||
except FieldError:
|
||||
return qs.filter(
|
||||
Q(title__icontains=value) | Q(content__icontains=value),
|
||||
)
|
||||
return ensure_effective_content(qs).filter(
|
||||
Q(title__icontains=value) | Q(effective_content__icontains=value),
|
||||
)
|
||||
else:
|
||||
return qs
|
||||
|
||||
@@ -198,14 +193,9 @@ class EffectiveContentFilter(Filter):
|
||||
value = value.strip() if isinstance(value, str) else value
|
||||
if not value:
|
||||
return qs
|
||||
try:
|
||||
return qs.filter(
|
||||
**{f"effective_content__{self.lookup_expr}": value},
|
||||
)
|
||||
except FieldError:
|
||||
return qs.filter(
|
||||
**{f"content__{self.lookup_expr}": value},
|
||||
)
|
||||
return ensure_effective_content(qs).filter(
|
||||
**{f"effective_content__{self.lookup_expr}": value},
|
||||
)
|
||||
|
||||
|
||||
@extend_schema_field(serializers.BooleanField)
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
# Generated by Django 5.2.16 on 2026-08-10 18:26
|
||||
|
||||
from django.db import migrations
|
||||
from django.db import models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("documents", "0024_alter_workflowaction_type"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name="workflowaction",
|
||||
name="ai_create_missing",
|
||||
field=models.BooleanField(
|
||||
default=False,
|
||||
help_text="Create suggested tags, correspondents, document types and storage paths that do not already exist instead of skipping them.",
|
||||
verbose_name="create missing objects",
|
||||
),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="workflowaction",
|
||||
name="ai_overwrite_existing",
|
||||
field=models.BooleanField(
|
||||
default=False,
|
||||
help_text="Apply suggestions even if the document already has a value for that field. Tags are always added to, never replaced.",
|
||||
verbose_name="overwrite existing values",
|
||||
),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="workflowaction",
|
||||
name="ai_suggestion_fields",
|
||||
field=models.JSONField(
|
||||
blank=True,
|
||||
help_text="Which of the AI-suggested fields to apply to the document.",
|
||||
null=True,
|
||||
verbose_name="AI suggestion fields",
|
||||
),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name="workflowaction",
|
||||
name="type",
|
||||
field=models.PositiveSmallIntegerField(
|
||||
choices=[
|
||||
(1, "Assignment"),
|
||||
(2, "Removal"),
|
||||
(3, "Email"),
|
||||
(4, "Webhook"),
|
||||
(5, "Password removal"),
|
||||
(6, "Move to trash"),
|
||||
(7, "Remote OCR"),
|
||||
(8, "Apply AI suggestions"),
|
||||
],
|
||||
default=1,
|
||||
verbose_name="Workflow Action Type",
|
||||
),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name="paperlesstask",
|
||||
name="task_type",
|
||||
field=models.CharField(
|
||||
choices=[
|
||||
("consume_file", "Consume File"),
|
||||
("train_classifier", "Train Classifier"),
|
||||
("sanity_check", "Sanity Check"),
|
||||
("index_optimize", "Index Optimize"),
|
||||
("mail_fetch", "Mail Fetch"),
|
||||
("llm_index", "LLM Index"),
|
||||
("empty_trash", "Empty Trash"),
|
||||
("check_workflows", "Check Workflows"),
|
||||
("bulk_update", "Bulk Update"),
|
||||
("reprocess_document", "Reprocess Document"),
|
||||
("build_share_link", "Build Share Link"),
|
||||
("bulk_delete", "Bulk Delete"),
|
||||
("apply_ai_suggestions", "Apply AI Suggestions"),
|
||||
],
|
||||
db_index=True,
|
||||
help_text="The kind of work being performed",
|
||||
max_length=50,
|
||||
verbose_name="Task Type",
|
||||
),
|
||||
),
|
||||
]
|
||||
@@ -766,6 +766,7 @@ class PaperlessTask(ModelWithOwner):
|
||||
REPROCESS_DOCUMENT = "reprocess_document", _("Reprocess Document")
|
||||
BUILD_SHARE_LINK = "build_share_link", _("Build Share Link")
|
||||
BULK_DELETE = "bulk_delete", _("Bulk Delete")
|
||||
APPLY_AI_SUGGESTIONS = "apply_ai_suggestions", _("Apply AI Suggestions")
|
||||
|
||||
COMPLETE_STATUSES = (
|
||||
Status.SUCCESS,
|
||||
@@ -1674,6 +1675,18 @@ class WorkflowAction(models.Model):
|
||||
7,
|
||||
_("Remote OCR"),
|
||||
)
|
||||
APPLY_AI_SUGGESTIONS = (
|
||||
8,
|
||||
_("Apply AI suggestions"),
|
||||
)
|
||||
|
||||
class AISuggestionField(models.TextChoices):
|
||||
TITLE = ("title", _("Title"))
|
||||
TAGS = ("tags", _("Tags"))
|
||||
CORRESPONDENT = ("correspondent", _("Correspondent"))
|
||||
DOCUMENT_TYPE = ("document_type", _("Document type"))
|
||||
STORAGE_PATH = ("storage_path", _("Storage path"))
|
||||
CREATED = ("created", _("Created date"))
|
||||
|
||||
type = models.PositiveSmallIntegerField(
|
||||
_("Workflow Action Type"),
|
||||
@@ -1912,6 +1925,33 @@ class WorkflowAction(models.Model):
|
||||
),
|
||||
)
|
||||
|
||||
ai_suggestion_fields = models.JSONField(
|
||||
_("AI suggestion fields"),
|
||||
null=True,
|
||||
blank=True,
|
||||
help_text=_(
|
||||
"Which of the AI-suggested fields to apply to the document.",
|
||||
),
|
||||
)
|
||||
|
||||
ai_create_missing = models.BooleanField(
|
||||
_("create missing objects"),
|
||||
default=False,
|
||||
help_text=_(
|
||||
"Create suggested tags, correspondents, document types and storage "
|
||||
"paths that do not already exist instead of skipping them.",
|
||||
),
|
||||
)
|
||||
|
||||
ai_overwrite_existing = models.BooleanField(
|
||||
_("overwrite existing values"),
|
||||
default=False,
|
||||
help_text=_(
|
||||
"Apply suggestions even if the document already has a value for that "
|
||||
"field. Tags are always added to, never replaced.",
|
||||
),
|
||||
)
|
||||
|
||||
class Meta:
|
||||
verbose_name = _("workflow action")
|
||||
verbose_name_plural = _("workflow actions")
|
||||
|
||||
@@ -3235,6 +3235,9 @@ class WorkflowActionSerializer(serializers.ModelSerializer[WorkflowAction]):
|
||||
"email",
|
||||
"webhook",
|
||||
"passwords",
|
||||
"ai_suggestion_fields",
|
||||
"ai_create_missing",
|
||||
"ai_overwrite_existing",
|
||||
]
|
||||
|
||||
def validate(self, attrs):
|
||||
@@ -3292,6 +3295,23 @@ class WorkflowActionSerializer(serializers.ModelSerializer[WorkflowAction]):
|
||||
"Passwords are required for password removal actions",
|
||||
)
|
||||
|
||||
if (
|
||||
"type" in attrs
|
||||
and attrs["type"] == WorkflowAction.WorkflowActionType.APPLY_AI_SUGGESTIONS
|
||||
):
|
||||
fields = attrs.get("ai_suggestion_fields")
|
||||
valid_fields = set(WorkflowAction.AISuggestionField.values)
|
||||
if (
|
||||
fields is None
|
||||
or not isinstance(fields, list)
|
||||
or len(fields) == 0
|
||||
or any(field not in valid_fields for field in fields)
|
||||
):
|
||||
raise serializers.ValidationError(
|
||||
"At least one valid field is required for apply AI "
|
||||
f"suggestions actions, options are: {sorted(valid_fields)}",
|
||||
)
|
||||
|
||||
return attrs
|
||||
|
||||
|
||||
@@ -3320,24 +3340,43 @@ class WorkflowSerializer(serializers.ModelSerializer[Workflow]):
|
||||
action.get("type") == WorkflowAction.WorkflowActionType.REMOTE_OCR
|
||||
for action in attrs["actions"]
|
||||
)
|
||||
has_ai_suggestions_action = any(
|
||||
action.get("type")
|
||||
== WorkflowAction.WorkflowActionType.APPLY_AI_SUGGESTIONS
|
||||
for action in attrs["actions"]
|
||||
)
|
||||
else:
|
||||
has_remote_ocr_action = self.instance is not None and (
|
||||
self.instance.actions.filter(
|
||||
type=WorkflowAction.WorkflowActionType.REMOTE_OCR,
|
||||
).exists()
|
||||
)
|
||||
has_ai_suggestions_action = self.instance is not None and (
|
||||
self.instance.actions.filter(
|
||||
type=WorkflowAction.WorkflowActionType.APPLY_AI_SUGGESTIONS,
|
||||
).exists()
|
||||
)
|
||||
|
||||
if "triggers" in attrs:
|
||||
has_consumption_trigger = any(
|
||||
trigger.get("type") == WorkflowTrigger.WorkflowTriggerType.CONSUMPTION
|
||||
for trigger in attrs["triggers"]
|
||||
)
|
||||
has_non_consumption_trigger = any(
|
||||
trigger.get("type") != WorkflowTrigger.WorkflowTriggerType.CONSUMPTION
|
||||
for trigger in attrs["triggers"]
|
||||
)
|
||||
else:
|
||||
has_consumption_trigger = self.instance is not None and (
|
||||
self.instance.triggers.filter(
|
||||
type=WorkflowTrigger.WorkflowTriggerType.CONSUMPTION,
|
||||
).exists()
|
||||
)
|
||||
has_non_consumption_trigger = self.instance is not None and (
|
||||
self.instance.triggers.exclude(
|
||||
type=WorkflowTrigger.WorkflowTriggerType.CONSUMPTION,
|
||||
).exists()
|
||||
)
|
||||
|
||||
# Remote OCR can only work with consumption triggers
|
||||
if has_remote_ocr_action and not has_consumption_trigger:
|
||||
@@ -3345,6 +3384,14 @@ class WorkflowSerializer(serializers.ModelSerializer[Workflow]):
|
||||
"Remote OCR actions require a consumption started trigger",
|
||||
)
|
||||
|
||||
# Suggestions are made from the document content, which does not exist
|
||||
# until after consumption has finished
|
||||
if has_ai_suggestions_action and not has_non_consumption_trigger:
|
||||
raise serializers.ValidationError(
|
||||
"Apply AI suggestions actions require a trigger other than "
|
||||
"consumption started",
|
||||
)
|
||||
|
||||
return attrs
|
||||
|
||||
def update_triggers_and_actions(
|
||||
|
||||
@@ -984,6 +984,28 @@ def run_workflows(
|
||||
"triggers, ignoring",
|
||||
extra={"group": logging_group},
|
||||
)
|
||||
elif (
|
||||
action.type
|
||||
== WorkflowAction.WorkflowActionType.APPLY_AI_SUGGESTIONS
|
||||
):
|
||||
if use_overrides:
|
||||
# The document has not been parsed yet, so there is no
|
||||
# content for the LLM to make suggestions from
|
||||
logger.debug(
|
||||
"Apply AI suggestions action does not apply to "
|
||||
"consumption triggers, ignoring",
|
||||
extra={"group": logging_group},
|
||||
)
|
||||
else:
|
||||
# Queued rather than run sync
|
||||
from documents.tasks import apply_ai_suggestions
|
||||
|
||||
# kwargs so the PaperlessTask record can note the
|
||||
# document, see _extract_input_data
|
||||
apply_ai_suggestions.delay(
|
||||
action_id=action.pk,
|
||||
document_id=document.pk,
|
||||
)
|
||||
|
||||
if not use_overrides:
|
||||
# limit title to 128 characters
|
||||
@@ -1039,6 +1061,7 @@ TRACKED_TASKS: dict[str, PaperlessTask.TaskType] = {
|
||||
"documents.tasks.update_document_content_maybe_archive_file": PaperlessTask.TaskType.REPROCESS_DOCUMENT,
|
||||
"documents.tasks.build_share_link_bundle": PaperlessTask.TaskType.BUILD_SHARE_LINK,
|
||||
"documents.bulk_edit.delete": PaperlessTask.TaskType.BULK_DELETE,
|
||||
"documents.tasks.apply_ai_suggestions": PaperlessTask.TaskType.APPLY_AI_SUGGESTIONS,
|
||||
}
|
||||
|
||||
_CELERY_STATE_TO_STATUS: dict[str, PaperlessTask.Status] = {
|
||||
@@ -1092,6 +1115,12 @@ def _extract_input_data(
|
||||
return {"account_ids": account_ids}
|
||||
return {}
|
||||
|
||||
if task_type == PaperlessTask.TaskType.APPLY_AI_SUGGESTIONS:
|
||||
document_id = task_kwargs.get("document_id")
|
||||
if document_id is not None:
|
||||
return {"document_id": document_id}
|
||||
return {}
|
||||
|
||||
return {}
|
||||
|
||||
|
||||
|
||||
@@ -71,6 +71,7 @@ from paperless.config import RemoteOCRConfig
|
||||
from paperless.logging import consume_task_id
|
||||
from paperless.parsers import ParserContext
|
||||
from paperless.parsers.registry import get_parser_registry
|
||||
from paperless_ai.exceptions import LLMTimeoutError
|
||||
from paperless_ai.indexing import llm_index_add_or_update_document
|
||||
from paperless_ai.indexing import llm_index_remove_document
|
||||
from paperless_ai.indexing import update_llm_index
|
||||
@@ -714,6 +715,45 @@ def llmindex_index(
|
||||
)
|
||||
|
||||
|
||||
@shared_task(
|
||||
bind=True,
|
||||
autoretry_for=(LLMTimeoutError,),
|
||||
max_retries=3,
|
||||
retry_backoff=60,
|
||||
retry_backoff_max=600,
|
||||
retry_jitter=True,
|
||||
)
|
||||
def apply_ai_suggestions(self, action_id: int, document_id: int) -> None:
|
||||
"""
|
||||
Deferred "apply AI suggestions" workflow action.
|
||||
"""
|
||||
from documents.models import WorkflowAction
|
||||
from documents.workflows.ai import apply_ai_suggestions_to_document
|
||||
|
||||
try:
|
||||
action = WorkflowAction.objects.get(pk=action_id)
|
||||
document = Document.objects.select_related("owner").get(pk=document_id)
|
||||
except (WorkflowAction.DoesNotExist, Document.DoesNotExist):
|
||||
logger.warning(
|
||||
"Workflow action %s or document %s no longer exists, "
|
||||
"not applying AI suggestions",
|
||||
action_id,
|
||||
document_id,
|
||||
)
|
||||
return
|
||||
|
||||
if not apply_ai_suggestions_to_document(action, document):
|
||||
return
|
||||
|
||||
# No document_updated signal to avoid loop
|
||||
clear_document_caches(document.pk)
|
||||
index_document.delay(document.pk)
|
||||
|
||||
ai_config = AIConfig()
|
||||
if ai_config.llm_index_enabled:
|
||||
update_document_in_llm_index.apply_async(kwargs={"document": document})
|
||||
|
||||
|
||||
@shared_task
|
||||
def update_document_in_llm_index(document) -> None:
|
||||
llm_index_add_or_update_document(document)
|
||||
|
||||
@@ -2,15 +2,14 @@ from __future__ import annotations
|
||||
|
||||
import datetime
|
||||
from typing import TYPE_CHECKING
|
||||
from unittest import TestCase
|
||||
from unittest import mock
|
||||
|
||||
from auditlog.models import LogEntry # type: ignore[import-untyped]
|
||||
from django.contrib.auth.models import Permission
|
||||
from django.contrib.auth.models import User
|
||||
from django.contrib.contenttypes.models import ContentType
|
||||
from django.core.exceptions import FieldError
|
||||
from django.core.files.uploadedfile import SimpleUploadedFile
|
||||
from django.test import TestCase as DjangoTestCase
|
||||
from django.utils import timezone
|
||||
from rest_framework import status
|
||||
from rest_framework.test import APITestCase
|
||||
@@ -21,6 +20,8 @@ from documents.filters import TitleContentFilter
|
||||
from documents.models import Document
|
||||
from documents.tests.utils import DirectoriesMixin
|
||||
from documents.tests.utils import read_streaming_response
|
||||
from documents.versioning import annotate_effective_content
|
||||
from documents.views import DocumentSelectionMixin
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
@@ -889,32 +890,103 @@ class TestDocumentVersioningApi(DirectoriesMixin, APITestCase):
|
||||
)
|
||||
|
||||
|
||||
class TestVersionAwareFilters(TestCase):
|
||||
def test_title_content_filter_falls_back_to_content(self) -> None:
|
||||
queryset = mock.Mock()
|
||||
fallback_queryset = mock.Mock()
|
||||
queryset.filter.side_effect = [FieldError("missing field"), fallback_queryset]
|
||||
class TestVersionAwareFilters(DjangoTestCase):
|
||||
"""
|
||||
The filters annotate effective_content themselves rather than relying on
|
||||
the caller's queryset carrying it, so they stay version-aware on a plain
|
||||
Document queryset (e.g. the bulk-edit "select all matching" path).
|
||||
"""
|
||||
|
||||
result = TitleContentFilter().filter(queryset, " latest ")
|
||||
def setUp(self) -> None:
|
||||
super().setUp()
|
||||
self.root = Document.objects.create(
|
||||
title="root",
|
||||
checksum="root",
|
||||
mime_type="application/pdf",
|
||||
content="superseded-content",
|
||||
)
|
||||
Document.objects.create(
|
||||
title="version",
|
||||
checksum="version",
|
||||
mime_type="application/pdf",
|
||||
root_document=self.root,
|
||||
version_index=1,
|
||||
content="latest-content",
|
||||
)
|
||||
self.unversioned = Document.objects.create(
|
||||
title="unversioned",
|
||||
checksum="unversioned",
|
||||
mime_type="application/pdf",
|
||||
content="latest-content",
|
||||
)
|
||||
|
||||
self.assertIs(result, fallback_queryset)
|
||||
self.assertEqual(queryset.filter.call_count, 2)
|
||||
|
||||
def test_effective_content_filter_falls_back_to_content_lookup(self) -> None:
|
||||
queryset = mock.Mock()
|
||||
fallback_queryset = mock.Mock()
|
||||
queryset.filter.side_effect = [FieldError("missing field"), fallback_queryset]
|
||||
|
||||
result = EffectiveContentFilter(lookup_expr="icontains").filter(
|
||||
queryset,
|
||||
def test_title_content_filter_matches_latest_version_content(self) -> None:
|
||||
result = TitleContentFilter().filter(
|
||||
Document.objects.filter(root_document__isnull=True),
|
||||
" latest ",
|
||||
)
|
||||
|
||||
self.assertIs(result, fallback_queryset)
|
||||
first_kwargs = queryset.filter.call_args_list[0].kwargs
|
||||
second_kwargs = queryset.filter.call_args_list[1].kwargs
|
||||
self.assertEqual(first_kwargs, {"effective_content__icontains": "latest"})
|
||||
self.assertEqual(second_kwargs, {"content__icontains": "latest"})
|
||||
self.assertCountEqual(
|
||||
[doc.id for doc in result],
|
||||
[self.root.id, self.unversioned.id],
|
||||
)
|
||||
|
||||
def test_effective_content_filter_matches_latest_version_content(self) -> None:
|
||||
result = EffectiveContentFilter(lookup_expr="icontains").filter(
|
||||
Document.objects.filter(root_document__isnull=True),
|
||||
" latest ",
|
||||
)
|
||||
|
||||
self.assertCountEqual(
|
||||
[doc.id for doc in result],
|
||||
[self.root.id, self.unversioned.id],
|
||||
)
|
||||
|
||||
def test_effective_content_filter_ignores_superseded_content(self) -> None:
|
||||
result = EffectiveContentFilter(lookup_expr="icontains").filter(
|
||||
Document.objects.filter(root_document__isnull=True),
|
||||
"superseded",
|
||||
)
|
||||
|
||||
self.assertEqual(list(result), [])
|
||||
|
||||
def test_filters_reuse_an_existing_annotation(self) -> None:
|
||||
"""
|
||||
Annotating twice under the same alias is an error, so an already
|
||||
annotated queryset (the search path) has to be left alone.
|
||||
"""
|
||||
annotated = annotate_effective_content(
|
||||
Document.objects.filter(root_document__isnull=True),
|
||||
)
|
||||
|
||||
result = EffectiveContentFilter(lookup_expr="icontains").filter(
|
||||
annotated,
|
||||
"latest",
|
||||
)
|
||||
|
||||
self.assertCountEqual(
|
||||
[doc.id for doc in result],
|
||||
[self.root.id, self.unversioned.id],
|
||||
)
|
||||
|
||||
def test_bulk_selection_does_not_match_superseded_content(self) -> None:
|
||||
"""
|
||||
Bulk edit's "select all matching" builds its own queryset, so before
|
||||
the filters annotated for themselves it matched the root document's
|
||||
superseded content -- selecting documents the list view, filtered by
|
||||
the same term, does not show.
|
||||
"""
|
||||
user = User.objects.create_superuser(username="bulk_selection")
|
||||
|
||||
selected = DocumentSelectionMixin()._resolve_document_ids(
|
||||
user=user,
|
||||
validated_data={
|
||||
"all": True,
|
||||
"filters": {"content__icontains": "superseded"},
|
||||
},
|
||||
)
|
||||
|
||||
self.assertEqual(selected, [])
|
||||
|
||||
def test_effective_content_filter_returns_input_for_empty_values(self) -> None:
|
||||
queryset = mock.Mock()
|
||||
@@ -923,3 +995,36 @@ class TestVersionAwareFilters(TestCase):
|
||||
|
||||
self.assertIs(result, queryset)
|
||||
queryset.filter.assert_not_called()
|
||||
|
||||
|
||||
class TestBulkSelectionExcludesVersions(DjangoTestCase):
|
||||
def test_select_all_matching_does_not_select_version_documents(self) -> None:
|
||||
"""
|
||||
"Select all matching" reconstructs the document list, which never
|
||||
contains version documents as rows of their own.
|
||||
"""
|
||||
user = User.objects.create_superuser(username="bulk_versions")
|
||||
root = Document.objects.create(
|
||||
title="shared-title root",
|
||||
checksum="bulk-root",
|
||||
mime_type="application/pdf",
|
||||
content="root",
|
||||
)
|
||||
Document.objects.create(
|
||||
title="shared-title version",
|
||||
checksum="bulk-version",
|
||||
mime_type="application/pdf",
|
||||
root_document=root,
|
||||
version_index=1,
|
||||
content="version",
|
||||
)
|
||||
|
||||
selected = DocumentSelectionMixin()._resolve_document_ids(
|
||||
user=user,
|
||||
validated_data={
|
||||
"all": True,
|
||||
"filters": {"title__icontains": "shared-title"},
|
||||
},
|
||||
)
|
||||
|
||||
self.assertEqual(selected, [root.id])
|
||||
|
||||
@@ -1917,6 +1917,29 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
|
||||
self.assertEqual(len(response.data["documents"]), 1)
|
||||
self.assertEqual(response.data["documents"][0]["id"], title_match.id)
|
||||
|
||||
def test_global_search_returns_latest_version_content(self) -> None:
|
||||
root = Document.objects.create(
|
||||
title="bank statement",
|
||||
content="superseded content",
|
||||
checksum="GSV1",
|
||||
pk=23,
|
||||
)
|
||||
Document.objects.create(
|
||||
title="bank statement v2",
|
||||
content="latest content",
|
||||
checksum="GSV2",
|
||||
pk=24,
|
||||
root_document=root,
|
||||
version_index=1,
|
||||
)
|
||||
|
||||
self.client.force_authenticate(self.user)
|
||||
|
||||
response = self.client.get("/api/search/?query=bank&db_only=true")
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
returned = {doc["id"]: doc["content"] for doc in response.data["documents"]}
|
||||
self.assertEqual(returned.get(root.id), "latest content")
|
||||
|
||||
def test_global_search_filters_owned_mail_objects(self) -> None:
|
||||
user1 = User.objects.create_user("mail-search-user")
|
||||
user2 = User.objects.create_user("other-mail-search-user")
|
||||
|
||||
@@ -641,6 +641,145 @@ class TestApiWorkflows(DirectoriesMixin, APITestCase):
|
||||
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
|
||||
self.assertEqual(self.workflow.triggers.get(), self.trigger)
|
||||
|
||||
def _post_ai_suggestions_workflow(self, *, trigger_types, action: dict):
|
||||
def trigger(trigger_type):
|
||||
# consumption triggers require a filter of their own
|
||||
if trigger_type == WorkflowTrigger.WorkflowTriggerType.CONSUMPTION:
|
||||
return {"type": trigger_type, "filter_filename": "*.pdf"}
|
||||
return {"type": trigger_type}
|
||||
|
||||
return self.client.post(
|
||||
self.ENDPOINT,
|
||||
json.dumps(
|
||||
{
|
||||
"name": "Apply AI suggestions",
|
||||
"order": 1,
|
||||
"triggers": [trigger(t) for t in trigger_types],
|
||||
"actions": [
|
||||
{
|
||||
"type": WorkflowAction.WorkflowActionType.APPLY_AI_SUGGESTIONS,
|
||||
**action,
|
||||
},
|
||||
],
|
||||
},
|
||||
),
|
||||
content_type="application/json",
|
||||
)
|
||||
|
||||
def test_api_create_apply_ai_suggestions_action(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- API request to create a workflow with an apply AI suggestions
|
||||
action and a valid set of fields
|
||||
WHEN:
|
||||
- API is called
|
||||
THEN:
|
||||
- The workflow is created with the chosen options
|
||||
"""
|
||||
response = self._post_ai_suggestions_workflow(
|
||||
trigger_types=[WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED],
|
||||
action={
|
||||
"ai_suggestion_fields": ["title", "tags", "correspondent"],
|
||||
"ai_create_missing": True,
|
||||
"ai_overwrite_existing": True,
|
||||
},
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
|
||||
action = Workflow.objects.get(name="Apply AI suggestions").actions.first()
|
||||
self.assertEqual(
|
||||
action.ai_suggestion_fields,
|
||||
["title", "tags", "correspondent"],
|
||||
)
|
||||
self.assertTrue(action.ai_create_missing)
|
||||
self.assertTrue(action.ai_overwrite_existing)
|
||||
|
||||
def test_api_create_apply_ai_suggestions_action_requires_fields(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- API request to create an apply AI suggestions action with no
|
||||
fields selected, which could never do anything
|
||||
WHEN:
|
||||
- API is called
|
||||
THEN:
|
||||
- Correct HTTP 400 response
|
||||
- No objects are created
|
||||
"""
|
||||
existing_count = Workflow.objects.count()
|
||||
|
||||
response = self._post_ai_suggestions_workflow(
|
||||
trigger_types=[WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED],
|
||||
action={"ai_suggestion_fields": []},
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
|
||||
self.assertEqual(Workflow.objects.count(), existing_count)
|
||||
|
||||
def test_api_create_apply_ai_suggestions_action_rejects_unknown_field(
|
||||
self,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- API request to create an apply AI suggestions action naming a
|
||||
field that does not exist
|
||||
WHEN:
|
||||
- API is called
|
||||
THEN:
|
||||
- Correct HTTP 400 response
|
||||
"""
|
||||
response = self._post_ai_suggestions_workflow(
|
||||
trigger_types=[WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED],
|
||||
action={"ai_suggestion_fields": ["title", "not_a_field"]},
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
|
||||
|
||||
def test_api_create_apply_ai_suggestions_action_rejects_consumption_only(
|
||||
self,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- API request to create an apply AI suggestions action whose only
|
||||
trigger is consumption started, so there is no document content
|
||||
to make suggestions from yet
|
||||
WHEN:
|
||||
- API is called
|
||||
THEN:
|
||||
- Correct HTTP 400 response
|
||||
- No objects are created
|
||||
"""
|
||||
existing_count = Workflow.objects.count()
|
||||
|
||||
response = self._post_ai_suggestions_workflow(
|
||||
trigger_types=[WorkflowTrigger.WorkflowTriggerType.CONSUMPTION],
|
||||
action={"ai_suggestion_fields": ["title"]},
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
|
||||
self.assertEqual(Workflow.objects.count(), existing_count)
|
||||
|
||||
def test_api_create_apply_ai_suggestions_action_allows_extra_consumption_trigger(
|
||||
self,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- API request to create an apply AI suggestions action with a
|
||||
consumption trigger alongside a usable one
|
||||
WHEN:
|
||||
- API is called
|
||||
THEN:
|
||||
- The workflow is created, the action applies to the other trigger
|
||||
"""
|
||||
response = self._post_ai_suggestions_workflow(
|
||||
trigger_types=[
|
||||
WorkflowTrigger.WorkflowTriggerType.CONSUMPTION,
|
||||
WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED,
|
||||
],
|
||||
action={"ai_suggestion_fields": ["title"]},
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
|
||||
|
||||
def test_api_create_workflow_trigger_action_empty_fields(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
|
||||
@@ -385,6 +385,25 @@ class TestTaskFailureHandler:
|
||||
task_failure_handler(task_id=None, exception=ValueError("x"), traceback=None)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestApplyAiSuggestionsTracking:
|
||||
def test_records_the_document_it_is_for(self) -> None:
|
||||
"""
|
||||
The action queues one task per document, so the tracked record notes
|
||||
which document it is for -- otherwise a bulk run is an indistinguishable
|
||||
wall of identical entries in the tasks list.
|
||||
"""
|
||||
task_id = send_publish(
|
||||
"documents.tasks.apply_ai_suggestions",
|
||||
(),
|
||||
{"action_id": 1, "document_id": 42},
|
||||
)
|
||||
|
||||
task = PaperlessTask.objects.get(task_id=task_id)
|
||||
assert task.task_type == PaperlessTask.TaskType.APPLY_AI_SUGGESTIONS
|
||||
assert task.input_data == {"document_id": 42}
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestTaskRevokedHandler:
|
||||
def test_marks_task_revoked(self, mocker: pytest_mock.MockerFixture) -> None:
|
||||
|
||||
@@ -14,6 +14,7 @@ from documents.models import Correspondent
|
||||
from documents.models import Document
|
||||
from documents.models import DocumentType
|
||||
from documents.models import Tag
|
||||
from documents.models import WorkflowAction
|
||||
from documents.sanity_checker import SanityCheckFailedException
|
||||
from documents.sanity_checker import SanityCheckMessages
|
||||
from documents.tests.test_classifier import dummy_preprocess
|
||||
@@ -447,3 +448,110 @@ class TestAIIndex(DirectoriesMixin, TestCase):
|
||||
rebuild=False,
|
||||
document_ids=doc_ids,
|
||||
)
|
||||
|
||||
|
||||
class TestApplyAISuggestionsTask(DirectoriesMixin, TestCase):
|
||||
def setUp(self) -> None:
|
||||
super().setUp()
|
||||
self.doc = Document.objects.create(
|
||||
title="doc",
|
||||
content="content",
|
||||
checksum="apply-ai-suggestions",
|
||||
)
|
||||
self.action = WorkflowAction.objects.create(
|
||||
type=WorkflowAction.WorkflowActionType.APPLY_AI_SUGGESTIONS,
|
||||
ai_suggestion_fields=[WorkflowAction.AISuggestionField.TITLE],
|
||||
)
|
||||
|
||||
def test_reindexes_without_sending_document_updated(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- An apply AI suggestions action that changes the document
|
||||
WHEN:
|
||||
- The task runs
|
||||
THEN:
|
||||
- The search index and caches are refreshed directly, deliberately
|
||||
not via the document_updated signal: that re-runs updated
|
||||
workflows, which for this action means queueing another LLM
|
||||
query for a document it just changed, forever
|
||||
"""
|
||||
with (
|
||||
mock.patch(
|
||||
"documents.workflows.ai.apply_ai_suggestions_to_document",
|
||||
return_value=["title"],
|
||||
),
|
||||
mock.patch("documents.tasks.index_document") as index_document,
|
||||
mock.patch("documents.tasks.clear_document_caches") as clear_caches,
|
||||
mock.patch("documents.tasks.document_updated") as document_updated,
|
||||
):
|
||||
tasks.apply_ai_suggestions(self.action.pk, self.doc.pk)
|
||||
|
||||
index_document.delay.assert_called_once_with(self.doc.pk)
|
||||
clear_caches.assert_called_once_with(self.doc.pk)
|
||||
document_updated.send.assert_not_called()
|
||||
|
||||
def test_no_changes_skips_reindex(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- An apply AI suggestions action that changes nothing
|
||||
WHEN:
|
||||
- The task runs
|
||||
THEN:
|
||||
- No reindexing work is queued
|
||||
"""
|
||||
with (
|
||||
mock.patch(
|
||||
"documents.workflows.ai.apply_ai_suggestions_to_document",
|
||||
return_value=[],
|
||||
),
|
||||
mock.patch("documents.tasks.index_document") as index_document,
|
||||
):
|
||||
tasks.apply_ai_suggestions(self.action.pk, self.doc.pk)
|
||||
|
||||
index_document.delay.assert_not_called()
|
||||
|
||||
@override_settings(AI_ENABLED=True, LLM_EMBEDDING_BACKEND="huggingface")
|
||||
def test_updates_llm_index_when_enabled(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- An apply AI suggestions action that changes the document
|
||||
- The LLM index is enabled
|
||||
WHEN:
|
||||
- The task runs
|
||||
THEN:
|
||||
- The document is updated in the LLM index too
|
||||
"""
|
||||
with (
|
||||
mock.patch(
|
||||
"documents.workflows.ai.apply_ai_suggestions_to_document",
|
||||
return_value=["title"],
|
||||
),
|
||||
mock.patch("documents.tasks.index_document"),
|
||||
mock.patch(
|
||||
"documents.tasks.update_document_in_llm_index",
|
||||
) as update_in_llm_index,
|
||||
):
|
||||
tasks.apply_ai_suggestions(self.action.pk, self.doc.pk)
|
||||
|
||||
update_in_llm_index.apply_async.assert_called_once()
|
||||
|
||||
def test_deleted_document_is_a_noop(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A document that was deleted between the workflow running and the
|
||||
queued task starting
|
||||
WHEN:
|
||||
- The task runs
|
||||
THEN:
|
||||
- It logs and exits rather than raising
|
||||
"""
|
||||
with (
|
||||
mock.patch(
|
||||
"documents.workflows.ai.apply_ai_suggestions_to_document",
|
||||
) as apply_suggestions,
|
||||
self.assertLogs("paperless.tasks", level="WARNING") as cm,
|
||||
):
|
||||
tasks.apply_ai_suggestions(self.action.pk, self.doc.pk + 1000)
|
||||
|
||||
apply_suggestions.assert_not_called()
|
||||
self.assertIn("no longer exists", "".join(cm.output))
|
||||
|
||||
@@ -31,7 +31,10 @@ from documents.file_handling import create_source_path_directory
|
||||
from documents.file_handling import generate_filename
|
||||
from documents.file_handling import generate_unique_filename
|
||||
from documents.signals.handlers import run_workflows
|
||||
from documents.workflows.ai import apply_ai_suggestions_to_document
|
||||
from documents.workflows.webhooks import send_webhook
|
||||
from paperless_ai.base_model import ClassificationSuggestions
|
||||
from paperless_ai.exceptions import LLMTimeoutError
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from django.db.models import QuerySet
|
||||
@@ -5488,3 +5491,481 @@ class TestRemoteOCRWorkflowAction(DirectoriesMixin, SampleDirMixin, APITestCase)
|
||||
)
|
||||
|
||||
self.assertIn("only applies to consumption triggers", "".join(cm.output))
|
||||
|
||||
|
||||
SUGGESTIONS: ClassificationSuggestions = {
|
||||
"title": "Suggested Title",
|
||||
"tags": {
|
||||
"existing_ids": [],
|
||||
"new_names": ["Existing Tag", "Suggested Tag"],
|
||||
},
|
||||
"correspondents": {
|
||||
"existing_ids": [],
|
||||
"new_names": ["Existing Correspondent", "Suggested Correspondent"],
|
||||
},
|
||||
"document_types": {
|
||||
"existing_ids": [],
|
||||
"new_names": ["Suggested Document Type"],
|
||||
},
|
||||
"storage_paths": {
|
||||
"existing_ids": [],
|
||||
"new_names": ["Suggested Storage Path"],
|
||||
},
|
||||
"dates": ["2024-03-05"],
|
||||
}
|
||||
|
||||
ALL_SUGGESTION_FIELDS = [
|
||||
WorkflowAction.AISuggestionField.TITLE,
|
||||
WorkflowAction.AISuggestionField.TAGS,
|
||||
WorkflowAction.AISuggestionField.CORRESPONDENT,
|
||||
WorkflowAction.AISuggestionField.DOCUMENT_TYPE,
|
||||
WorkflowAction.AISuggestionField.STORAGE_PATH,
|
||||
WorkflowAction.AISuggestionField.CREATED,
|
||||
]
|
||||
|
||||
|
||||
@override_settings(AI_ENABLED=True)
|
||||
class TestApplyAISuggestionsWorkflowAction(
|
||||
DirectoriesMixin,
|
||||
SampleDirMixin,
|
||||
APITestCase,
|
||||
):
|
||||
def setUp(self) -> None:
|
||||
super().setUp()
|
||||
self.user = User.objects.create(username="ai-user")
|
||||
self.doc = Document.objects.create(
|
||||
title="original.pdf",
|
||||
content="the document content",
|
||||
checksum="ai-suggestions-checksum",
|
||||
mime_type="application/pdf",
|
||||
created=datetime.date(2020, 1, 1),
|
||||
owner=self.user,
|
||||
)
|
||||
|
||||
def make_action(self, **kwargs) -> WorkflowAction:
|
||||
return WorkflowAction.objects.create(
|
||||
type=WorkflowAction.WorkflowActionType.APPLY_AI_SUGGESTIONS,
|
||||
ai_suggestion_fields=kwargs.pop(
|
||||
"ai_suggestion_fields",
|
||||
ALL_SUGGESTION_FIELDS,
|
||||
),
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def make_workflow(self, action: WorkflowAction, trigger_type) -> Workflow:
|
||||
trigger = WorkflowTrigger.objects.create(type=trigger_type)
|
||||
w = Workflow.objects.create(name="Apply AI suggestions", order=0)
|
||||
w.triggers.add(trigger)
|
||||
w.actions.add(action)
|
||||
w.save()
|
||||
return w
|
||||
|
||||
def apply(
|
||||
self,
|
||||
action: WorkflowAction,
|
||||
suggestions: ClassificationSuggestions = SUGGESTIONS,
|
||||
) -> list[str]:
|
||||
with mock.patch(
|
||||
"documents.workflows.ai.get_ai_document_classification",
|
||||
return_value=suggestions,
|
||||
):
|
||||
changed = apply_ai_suggestions_to_document(action, self.doc)
|
||||
self.doc.refresh_from_db()
|
||||
return changed
|
||||
|
||||
def test_document_added_trigger_queues_task(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A document added workflow with an apply AI suggestions action
|
||||
WHEN:
|
||||
- A matching document is added
|
||||
THEN:
|
||||
- The work is queued rather than run inline, so a slow LLM query
|
||||
cannot stall the rest of the workflow run
|
||||
"""
|
||||
action = self.make_action()
|
||||
self.make_workflow(action, WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED)
|
||||
|
||||
with mock.patch("documents.tasks.apply_ai_suggestions.delay") as delay:
|
||||
run_workflows(
|
||||
WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED,
|
||||
self.doc,
|
||||
)
|
||||
|
||||
delay.assert_called_once_with(action_id=action.pk, document_id=self.doc.pk)
|
||||
|
||||
def test_consumption_trigger_is_ignored(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A workflow with an apply AI suggestions action and a consumption
|
||||
trigger alongside a valid one
|
||||
WHEN:
|
||||
- The consumption trigger fires
|
||||
THEN:
|
||||
- The action is skipped, since the document has not been parsed
|
||||
yet and so has no content to make suggestions from
|
||||
"""
|
||||
action = self.make_action()
|
||||
w = self.make_workflow(
|
||||
action,
|
||||
WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED,
|
||||
)
|
||||
w.triggers.add(
|
||||
WorkflowTrigger.objects.create(
|
||||
type=WorkflowTrigger.WorkflowTriggerType.CONSUMPTION,
|
||||
),
|
||||
)
|
||||
|
||||
test_file = shutil.copy(
|
||||
self.SAMPLE_DIR / "simple.pdf",
|
||||
self.dirs.scratch_dir / "simple.pdf",
|
||||
)
|
||||
|
||||
with (
|
||||
mock.patch("documents.tasks.apply_ai_suggestions.delay") as delay,
|
||||
self.assertLogs("paperless.handlers", level="DEBUG") as cm,
|
||||
):
|
||||
run_workflows(
|
||||
WorkflowTrigger.WorkflowTriggerType.CONSUMPTION,
|
||||
ConsumableDocument(
|
||||
source=DocumentSource.ConsumeFolder,
|
||||
original_file=test_file,
|
||||
),
|
||||
overrides=DocumentMetadataOverrides(),
|
||||
)
|
||||
|
||||
delay.assert_not_called()
|
||||
self.assertIn("does not apply to consumption triggers", "".join(cm.output))
|
||||
|
||||
def test_no_selected_fields_does_nothing(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- An action with no suggestion fields selected
|
||||
WHEN:
|
||||
- The action is applied
|
||||
THEN:
|
||||
- Nothing is changed and it is logged
|
||||
"""
|
||||
action = self.make_action(ai_suggestion_fields=[])
|
||||
|
||||
with self.assertLogs("paperless.workflows.ai", level="WARNING") as cm:
|
||||
changed = self.apply(action)
|
||||
|
||||
self.assertEqual(changed, [])
|
||||
self.assertIn("no AI suggestion fields selected", "".join(cm.output))
|
||||
|
||||
@override_settings(AI_ENABLED=False)
|
||||
def test_ai_disabled_does_nothing(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- An action on an install where AI has since been disabled
|
||||
WHEN:
|
||||
- The action is applied
|
||||
THEN:
|
||||
- Nothing is changed and it is logged
|
||||
"""
|
||||
action = self.make_action()
|
||||
|
||||
with self.assertLogs("paperless.workflows.ai", level="ERROR") as cm:
|
||||
changed = self.apply(action)
|
||||
|
||||
self.assertEqual(changed, [])
|
||||
self.assertIn("AI is not enabled", "".join(cm.output))
|
||||
|
||||
def test_invalid_configuration_leaves_document_untouched(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- An AI backend that is misconfigured
|
||||
WHEN:
|
||||
- The action is applied
|
||||
THEN:
|
||||
- The failure is logged and the document is left alone. It is not
|
||||
re-raised, because retrying will not fix a bad configuration
|
||||
"""
|
||||
action = self.make_action()
|
||||
|
||||
with (
|
||||
mock.patch(
|
||||
"documents.workflows.ai.get_ai_document_classification",
|
||||
side_effect=ValueError("nope"),
|
||||
),
|
||||
self.assertLogs("paperless.workflows.ai", level="ERROR") as cm,
|
||||
):
|
||||
changed = apply_ai_suggestions_to_document(action, self.doc)
|
||||
|
||||
self.assertEqual(changed, [])
|
||||
self.doc.refresh_from_db()
|
||||
self.assertEqual(self.doc.title, "original.pdf")
|
||||
self.assertIn("Invalid AI configuration", "".join(cm.output))
|
||||
|
||||
def test_transient_llm_failure_is_raised_for_retry(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- An LLM backend that times out, or rate limits the request
|
||||
WHEN:
|
||||
- The action is applied
|
||||
THEN:
|
||||
- The error propagates so the queued task can back off and retry,
|
||||
rather than silently dropping this document's suggestions
|
||||
"""
|
||||
action = self.make_action()
|
||||
|
||||
with (
|
||||
mock.patch(
|
||||
"documents.workflows.ai.get_ai_document_classification",
|
||||
side_effect=LLMTimeoutError(),
|
||||
),
|
||||
self.assertRaises(LLMTimeoutError),
|
||||
):
|
||||
apply_ai_suggestions_to_document(action, self.doc)
|
||||
|
||||
self.doc.refresh_from_db()
|
||||
self.assertEqual(self.doc.title, "original.pdf")
|
||||
|
||||
def test_only_matching_objects_are_applied(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- An action without create missing, and only some of the suggested
|
||||
objects existing
|
||||
WHEN:
|
||||
- The action is applied
|
||||
THEN:
|
||||
- Only the existing objects are assigned, unmatched suggestions are
|
||||
dropped rather than creating anything
|
||||
"""
|
||||
tag = Tag.objects.create(name="Existing Tag", owner=self.user)
|
||||
correspondent = Correspondent.objects.create(
|
||||
name="Existing Correspondent",
|
||||
owner=self.user,
|
||||
)
|
||||
action = self.make_action(ai_overwrite_existing=True)
|
||||
|
||||
changed = self.apply(action)
|
||||
|
||||
self.assertEqual(self.doc.correspondent, correspondent)
|
||||
self.assertEqual(list(self.doc.tags.all()), [tag])
|
||||
# Nothing matched for these and create missing is off
|
||||
self.assertIsNone(self.doc.document_type)
|
||||
self.assertIsNone(self.doc.storage_path)
|
||||
self.assertNotIn("document_type", changed)
|
||||
self.assertEqual(Tag.objects.count(), 1)
|
||||
self.assertEqual(Correspondent.objects.count(), 1)
|
||||
|
||||
def test_existing_id_suggestions_are_applied(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- AI suggestions that select existing taxonomy candidates by ID
|
||||
WHEN:
|
||||
- The suggestions are applied
|
||||
THEN:
|
||||
- Each selected object is assigned to the document
|
||||
"""
|
||||
tag = Tag.objects.create(name="Existing Tag", owner=self.user)
|
||||
correspondent = Correspondent.objects.create(
|
||||
name="Existing Correspondent",
|
||||
owner=self.user,
|
||||
)
|
||||
document_type = DocumentType.objects.create(
|
||||
name="Existing Document Type",
|
||||
owner=self.user,
|
||||
)
|
||||
storage_path = StoragePath.objects.create(
|
||||
name="Existing Storage Path",
|
||||
path="{{ title }}",
|
||||
owner=self.user,
|
||||
)
|
||||
action = self.make_action(ai_overwrite_existing=True)
|
||||
suggestions: ClassificationSuggestions = {
|
||||
**SUGGESTIONS,
|
||||
"tags": {"existing_ids": [tag.pk], "new_names": []},
|
||||
"correspondents": {
|
||||
"existing_ids": [correspondent.pk],
|
||||
"new_names": [],
|
||||
},
|
||||
"document_types": {
|
||||
"existing_ids": [document_type.pk],
|
||||
"new_names": [],
|
||||
},
|
||||
"storage_paths": {
|
||||
"existing_ids": [storage_path.pk],
|
||||
"new_names": [],
|
||||
},
|
||||
}
|
||||
|
||||
changed = self.apply(action, suggestions)
|
||||
|
||||
self.assertEqual(list(self.doc.tags.all()), [tag])
|
||||
self.assertEqual(self.doc.correspondent, correspondent)
|
||||
self.assertEqual(self.doc.document_type, document_type)
|
||||
self.assertEqual(self.doc.storage_path, storage_path)
|
||||
self.assertTrue(
|
||||
{"tags", "correspondent", "document_type", "storage_path"} <= set(changed),
|
||||
)
|
||||
|
||||
def test_create_missing_creates_objects_owned_by_document_owner(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- An action with create missing enabled
|
||||
WHEN:
|
||||
- The action is applied and suggestions match nothing
|
||||
THEN:
|
||||
- Tags, correspondents and document types are created, owned by the
|
||||
document owner so they stay private to them
|
||||
- Storage paths are never created, since a path template cannot be
|
||||
inferred from a name
|
||||
"""
|
||||
action = self.make_action(
|
||||
ai_create_missing=True,
|
||||
ai_overwrite_existing=True,
|
||||
)
|
||||
|
||||
changed = self.apply(action)
|
||||
|
||||
self.assertEqual(
|
||||
sorted(t.name for t in self.doc.tags.all()),
|
||||
["Existing Tag", "Suggested Tag"],
|
||||
)
|
||||
self.assertEqual(self.doc.correspondent.name, "Existing Correspondent")
|
||||
self.assertEqual(self.doc.correspondent.owner, self.user)
|
||||
self.assertEqual(self.doc.document_type.name, "Suggested Document Type")
|
||||
self.assertEqual(self.doc.document_type.owner, self.user)
|
||||
|
||||
self.assertIsNone(self.doc.storage_path)
|
||||
self.assertFalse(StoragePath.objects.exists())
|
||||
self.assertNotIn("storage_path", changed)
|
||||
|
||||
def test_overwrite_disabled_keeps_existing_values(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- An action without overwrite existing
|
||||
- A document that already has a title, created date and
|
||||
correspondent
|
||||
WHEN:
|
||||
- The action is applied
|
||||
THEN:
|
||||
- The existing values are kept, only the empty document type is
|
||||
filled in
|
||||
"""
|
||||
existing = Correspondent.objects.create(name="Mine", owner=self.user)
|
||||
self.doc.correspondent = existing
|
||||
self.doc.save()
|
||||
action = self.make_action(ai_create_missing=True)
|
||||
|
||||
changed = self.apply(action)
|
||||
|
||||
self.assertEqual(self.doc.title, "original.pdf")
|
||||
self.assertEqual(self.doc.created, datetime.date(2020, 1, 1))
|
||||
self.assertEqual(self.doc.correspondent, existing)
|
||||
self.assertEqual(self.doc.document_type.name, "Suggested Document Type")
|
||||
self.assertNotIn("title", changed)
|
||||
self.assertNotIn("correspondent", changed)
|
||||
|
||||
def test_overwrite_enabled_replaces_existing_values(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- An action with overwrite existing
|
||||
- A document that already has a title and created date
|
||||
WHEN:
|
||||
- The action is applied
|
||||
THEN:
|
||||
- The suggested values replace them
|
||||
"""
|
||||
action = self.make_action(
|
||||
ai_create_missing=True,
|
||||
ai_overwrite_existing=True,
|
||||
)
|
||||
|
||||
changed = self.apply(action)
|
||||
|
||||
self.assertEqual(self.doc.title, "Suggested Title")
|
||||
self.assertEqual(self.doc.created, datetime.date(2024, 3, 5))
|
||||
self.assertIn("title", changed)
|
||||
self.assertIn("created", changed)
|
||||
|
||||
def test_tags_are_added_not_replaced(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A document that already has a tag unrelated to the suggestions
|
||||
WHEN:
|
||||
- The action is applied with overwrite existing enabled
|
||||
THEN:
|
||||
- The existing tag is kept, since suggested tags are always
|
||||
additive regardless of the overwrite setting
|
||||
"""
|
||||
kept = Tag.objects.create(name="Do Not Remove", owner=self.user)
|
||||
self.doc.tags.add(kept)
|
||||
Tag.objects.create(name="Existing Tag", owner=self.user)
|
||||
action = self.make_action(ai_overwrite_existing=True)
|
||||
|
||||
self.apply(action)
|
||||
|
||||
self.assertEqual(
|
||||
sorted(t.name for t in self.doc.tags.all()),
|
||||
["Do Not Remove", "Existing Tag"],
|
||||
)
|
||||
|
||||
def test_unselected_fields_are_untouched(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- An action that only selects the title
|
||||
WHEN:
|
||||
- The action is applied
|
||||
THEN:
|
||||
- Only the title changes, even though the LLM suggested everything
|
||||
"""
|
||||
action = self.make_action(
|
||||
ai_suggestion_fields=[WorkflowAction.AISuggestionField.TITLE],
|
||||
ai_create_missing=True,
|
||||
ai_overwrite_existing=True,
|
||||
)
|
||||
|
||||
changed = self.apply(action)
|
||||
|
||||
self.assertEqual(changed, ["title"])
|
||||
self.assertEqual(self.doc.title, "Suggested Title")
|
||||
self.assertEqual(self.doc.tags.count(), 0)
|
||||
self.assertIsNone(self.doc.correspondent)
|
||||
self.assertEqual(self.doc.created, datetime.date(2020, 1, 1))
|
||||
|
||||
def test_another_users_private_objects_are_not_matched(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A suggested tag name that exists, but is owned by someone else
|
||||
WHEN:
|
||||
- The action is applied
|
||||
THEN:
|
||||
- It is not assigned, because the document owner cannot see it
|
||||
"""
|
||||
other = User.objects.create(username="someone-else")
|
||||
Tag.objects.create(name="Existing Tag", owner=other)
|
||||
action = self.make_action(
|
||||
ai_suggestion_fields=[WorkflowAction.AISuggestionField.TAGS],
|
||||
)
|
||||
|
||||
self.apply(action)
|
||||
|
||||
self.assertEqual(self.doc.tags.count(), 0)
|
||||
|
||||
def test_unparsable_dates_are_skipped(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- Suggested dates that are not all valid
|
||||
WHEN:
|
||||
- The action is applied
|
||||
THEN:
|
||||
- The first usable date is applied and the rest ignored
|
||||
"""
|
||||
action = self.make_action(
|
||||
ai_suggestion_fields=[WorkflowAction.AISuggestionField.CREATED],
|
||||
ai_overwrite_existing=True,
|
||||
)
|
||||
|
||||
with mock.patch(
|
||||
"documents.workflows.ai.get_ai_document_classification",
|
||||
return_value={**SUGGESTIONS, "dates": ["not a date", "2019-07-04"]},
|
||||
):
|
||||
changed = apply_ai_suggestions_to_document(action, self.doc)
|
||||
|
||||
self.doc.refresh_from_db()
|
||||
self.assertEqual(changed, ["created"])
|
||||
self.assertEqual(self.doc.created, datetime.date(2019, 7, 4))
|
||||
|
||||
@@ -43,6 +43,21 @@ def annotate_effective_content(documents: QuerySet[Document]) -> QuerySet[Docume
|
||||
)
|
||||
|
||||
|
||||
def ensure_effective_content(documents: QuerySet[Document]) -> QuerySet[Document]:
|
||||
"""
|
||||
Annotates effective_content unless the queryset already carries it.
|
||||
|
||||
Lets a filter depend on effective_content without having to assume its
|
||||
caller annotated one -- annotating twice under the same alias is an error,
|
||||
and silently matching on the root document's own content instead is worse,
|
||||
because the same filter then selects different documents depending on which
|
||||
queryset it was handed.
|
||||
"""
|
||||
if "effective_content" in documents.query.annotations:
|
||||
return documents
|
||||
return annotate_effective_content(documents)
|
||||
|
||||
|
||||
def sort_versions_newest_first(documents: list[Document]) -> list[Document]:
|
||||
"""
|
||||
Same sorting as versions_newest_first()
|
||||
|
||||
+20
-18
@@ -230,6 +230,7 @@ from documents.tasks import train_classifier
|
||||
from documents.tasks import update_document_parent_tags
|
||||
from documents.utils import get_boolean
|
||||
from documents.versioning import VersionResolutionError
|
||||
from documents.versioning import annotate_effective_content
|
||||
from documents.versioning import get_latest_version_for_root
|
||||
from documents.versioning import get_request_version_param
|
||||
from documents.versioning import get_root_document
|
||||
@@ -247,6 +248,7 @@ from paperless.serialisers import GroupSerializer
|
||||
from paperless.serialisers import UserSerializer
|
||||
from paperless.views import StandardPagination
|
||||
from paperless_ai.ai_classifier import get_ai_document_classification
|
||||
from paperless_ai.ai_classifier import get_llm_output_language
|
||||
from paperless_ai.chat import stream_chat_with_documents
|
||||
from paperless_ai.exceptions import LLMTimeoutError
|
||||
from paperless_ai.matching import extract_unmatched_names
|
||||
@@ -665,20 +667,6 @@ class TagViewSet(PermissionsAwareDocumentCountMixin, ModelViewSet[Tag]):
|
||||
update_document_parent_tags(tag, new_parent)
|
||||
|
||||
|
||||
def _get_llm_output_language(ai_config: AIConfig, request) -> str | None:
|
||||
output_language = ai_config.llm_output_language
|
||||
if (
|
||||
not output_language
|
||||
and hasattr(request.user, "ui_settings")
|
||||
and isinstance(
|
||||
request.user.ui_settings.settings,
|
||||
dict,
|
||||
)
|
||||
):
|
||||
output_language = request.user.ui_settings.settings.get("language")
|
||||
return output_language
|
||||
|
||||
|
||||
@extend_schema_view(**generate_object_with_permissions_schema(DocumentTypeSerializer))
|
||||
class DocumentTypeViewSet(
|
||||
PermissionsAwareDocumentCountMixin,
|
||||
@@ -1542,7 +1530,10 @@ class DocumentViewSet(
|
||||
if not ai_config.ai_enabled:
|
||||
return HttpResponseBadRequest("AI is required for this feature")
|
||||
|
||||
output_language = _get_llm_output_language(ai_config=ai_config, request=request)
|
||||
output_language = get_llm_output_language(
|
||||
ai_config=ai_config,
|
||||
user=request.user,
|
||||
)
|
||||
llm_cache_backend = ":".join(
|
||||
part
|
||||
for part in (
|
||||
@@ -2331,7 +2322,10 @@ class ChatStreamingView(GenericAPIView[Any]):
|
||||
id__in=permitted_document_ids(request.user),
|
||||
)
|
||||
|
||||
output_language = _get_llm_output_language(ai_config=ai_config, request=request)
|
||||
output_language = get_llm_output_language(
|
||||
ai_config=ai_config,
|
||||
user=request.user,
|
||||
)
|
||||
|
||||
response = StreamingHttpResponse(
|
||||
stream_chat_with_documents(
|
||||
@@ -2795,8 +2789,11 @@ class DocumentSelectionMixin:
|
||||
for key, value in filters.items()
|
||||
if key not in _TANTIVY_SEARCH_PARAM_NAMES
|
||||
}
|
||||
# Operations are addressed to roots, a caller that wants
|
||||
# to act on a specific version passes its id explicitly instead
|
||||
permitted_documents = Document.objects.filter(
|
||||
id__in=permitted_document_ids(user),
|
||||
root_document__isnull=True,
|
||||
)
|
||||
# orm-filtered docs
|
||||
filtered_documents = DocumentFilterSet(
|
||||
@@ -3617,8 +3614,13 @@ class GlobalSearchView(PassUserMixin):
|
||||
OBJECT_LIMIT = 3
|
||||
docs = []
|
||||
if request.user.has_perm("documents.view_document"):
|
||||
all_docs = Document.objects.filter(
|
||||
id__in=permitted_document_ids(request.user),
|
||||
# Never more than OBJECT_LIMIT rows come back here, so annotating
|
||||
# is cheap -- and without it these results show the root
|
||||
# document's superseded content.
|
||||
all_docs = annotate_effective_content(
|
||||
Document.objects.filter(
|
||||
id__in=permitted_document_ids(request.user),
|
||||
),
|
||||
)
|
||||
if db_only:
|
||||
docs = all_docs.filter(title__icontains=query)[:OBJECT_LIMIT]
|
||||
|
||||
@@ -0,0 +1,259 @@
|
||||
import logging
|
||||
from datetime import date
|
||||
from datetime import datetime
|
||||
from typing import TypeVar
|
||||
|
||||
from django.contrib.auth.models import User
|
||||
|
||||
from documents.models import Correspondent
|
||||
from documents.models import Document
|
||||
from documents.models import DocumentType
|
||||
from documents.models import MatchingModel
|
||||
from documents.models import StoragePath
|
||||
from documents.models import Tag
|
||||
from documents.models import WorkflowAction
|
||||
from paperless.config import AIConfig
|
||||
from paperless_ai.ai_classifier import get_ai_document_classification
|
||||
from paperless_ai.ai_classifier import get_llm_output_language
|
||||
from paperless_ai.matching import extract_unmatched_names
|
||||
from paperless_ai.matching import match_correspondents_by_name
|
||||
from paperless_ai.matching import match_document_types_by_name
|
||||
from paperless_ai.matching import match_storage_paths_by_name
|
||||
from paperless_ai.matching import match_tags_by_name
|
||||
from paperless_ai.matching import resolve_correspondent_ids
|
||||
from paperless_ai.matching import resolve_document_type_ids
|
||||
from paperless_ai.matching import resolve_storage_path_ids
|
||||
from paperless_ai.matching import resolve_tag_ids
|
||||
|
||||
logger = logging.getLogger("paperless.workflows.ai")
|
||||
|
||||
AISuggestionField = WorkflowAction.AISuggestionField
|
||||
ObjT = TypeVar("ObjT", bound=MatchingModel)
|
||||
|
||||
# Tags use m2m relation instead
|
||||
DIRECT_FIELDS: dict[str, str] = {
|
||||
AISuggestionField.TITLE: "title",
|
||||
AISuggestionField.CORRESPONDENT: "correspondent",
|
||||
AISuggestionField.DOCUMENT_TYPE: "document_type",
|
||||
AISuggestionField.STORAGE_PATH: "storage_path",
|
||||
AISuggestionField.CREATED: "created",
|
||||
}
|
||||
|
||||
|
||||
def resolve_date(dates: list[str]) -> date | None:
|
||||
"""
|
||||
First usable date out of the suggestions, which are expected as
|
||||
YYYY-MM-DD. Document.created is a DateField, so only one can be applied.
|
||||
"""
|
||||
for value in dates:
|
||||
try:
|
||||
return datetime.strptime(value, "%Y-%m-%d").date()
|
||||
except (TypeError, ValueError):
|
||||
logger.debug("Ignoring unparsable suggested date %s", value)
|
||||
return None
|
||||
|
||||
|
||||
def resolve_object(
|
||||
model: type[ObjT],
|
||||
names: list[str],
|
||||
matched: list[ObjT],
|
||||
*,
|
||||
create_missing: bool,
|
||||
owner: User | None,
|
||||
) -> ObjT | None:
|
||||
"""
|
||||
Single object from a suggestion list. The best match if there was one, else
|
||||
optionally a newly-created object. StoragePaths are excluded.
|
||||
"""
|
||||
if matched:
|
||||
return matched[0]
|
||||
|
||||
if not create_missing or model is StoragePath:
|
||||
return None
|
||||
|
||||
unmatched = extract_unmatched_names(names, matched)
|
||||
if not unmatched:
|
||||
return None
|
||||
|
||||
# (name, owner) is what MatchingModel is unique on
|
||||
obj, created = model.objects.get_or_create(
|
||||
name=unmatched[0][:128],
|
||||
owner=owner,
|
||||
)
|
||||
if created:
|
||||
logger.info("Created %s '%s' from AI suggestion", model.__name__, obj.name)
|
||||
return obj
|
||||
|
||||
|
||||
def resolve_tags(
|
||||
names: list[str],
|
||||
matched: list[Tag],
|
||||
*,
|
||||
create_missing: bool,
|
||||
owner: User | None,
|
||||
) -> list[Tag]:
|
||||
"""
|
||||
Matched tags, plus newly created ones if create_missing is set.
|
||||
"""
|
||||
tags = list(matched)
|
||||
if not create_missing:
|
||||
return tags
|
||||
|
||||
for name in extract_unmatched_names(names, matched):
|
||||
tag, created = Tag.objects.get_or_create(
|
||||
name=name[:128],
|
||||
owner=owner,
|
||||
)
|
||||
if created:
|
||||
logger.info("Created tag '%s' from AI suggestion", tag.name)
|
||||
tags.append(tag)
|
||||
return tags
|
||||
|
||||
|
||||
def apply_ai_suggestions_to_document(
|
||||
action: WorkflowAction,
|
||||
document: Document,
|
||||
logging_group=None,
|
||||
) -> list[str]:
|
||||
"""
|
||||
Get suggestions about `document` and write the chosen fields.
|
||||
|
||||
Returns the names of the fields that were actually changed.
|
||||
"""
|
||||
selected = set(action.ai_suggestion_fields or [])
|
||||
if not selected:
|
||||
logger.warning(
|
||||
"Workflow action %s has no AI suggestion fields selected, skipping",
|
||||
action.pk,
|
||||
extra={"group": logging_group},
|
||||
)
|
||||
return []
|
||||
|
||||
ai_config = AIConfig()
|
||||
if not ai_config.ai_enabled:
|
||||
logger.error(
|
||||
"AI is not enabled, cannot apply AI suggestions for document %s",
|
||||
document.pk,
|
||||
extra={"group": logging_group},
|
||||
)
|
||||
return []
|
||||
|
||||
# Workflows run without a user, so we use the document owner
|
||||
owner = document.owner
|
||||
|
||||
try:
|
||||
suggestions = get_ai_document_classification(
|
||||
document,
|
||||
owner,
|
||||
get_llm_output_language(ai_config, owner),
|
||||
)
|
||||
except ValueError:
|
||||
# A bad AI config will not fix itself, so swallow it rather than
|
||||
# letting the caller retry. Timeouts, rate limits, network errors etc
|
||||
# propagate so the queued task can back off and try again.
|
||||
logger.exception(
|
||||
"Invalid AI configuration, cannot get suggestions for document %s",
|
||||
document.pk,
|
||||
extra={"group": logging_group},
|
||||
)
|
||||
return []
|
||||
|
||||
overwrite = action.ai_overwrite_existing
|
||||
create_missing = action.ai_create_missing
|
||||
updated_fields: list[str] = []
|
||||
|
||||
def should_set(field: str) -> bool:
|
||||
# The field is selected and (overwrite or it's empty)
|
||||
return field in selected and (
|
||||
overwrite or getattr(document, DIRECT_FIELDS[field]) in (None, "")
|
||||
)
|
||||
|
||||
if should_set(AISuggestionField.TITLE):
|
||||
title = suggestions["title"].strip()
|
||||
if title:
|
||||
# title is capped at 128 characters
|
||||
document.title = title[:128]
|
||||
updated_fields.append("title")
|
||||
|
||||
if should_set(AISuggestionField.CORRESPONDENT):
|
||||
choice = suggestions["correspondents"]
|
||||
names = choice["new_names"]
|
||||
correspondent = resolve_object(
|
||||
Correspondent,
|
||||
names,
|
||||
resolve_correspondent_ids(choice["existing_ids"], owner)
|
||||
+ match_correspondents_by_name(names, owner),
|
||||
create_missing=create_missing,
|
||||
owner=owner,
|
||||
)
|
||||
if correspondent:
|
||||
document.correspondent = correspondent
|
||||
updated_fields.append("correspondent")
|
||||
|
||||
if should_set(AISuggestionField.DOCUMENT_TYPE):
|
||||
choice = suggestions["document_types"]
|
||||
names = choice["new_names"]
|
||||
document_type = resolve_object(
|
||||
DocumentType,
|
||||
names,
|
||||
resolve_document_type_ids(choice["existing_ids"], owner)
|
||||
+ match_document_types_by_name(names, owner),
|
||||
create_missing=create_missing,
|
||||
owner=owner,
|
||||
)
|
||||
if document_type:
|
||||
document.document_type = document_type
|
||||
updated_fields.append("document_type")
|
||||
|
||||
if should_set(AISuggestionField.STORAGE_PATH):
|
||||
choice = suggestions["storage_paths"]
|
||||
names = choice["new_names"]
|
||||
storage_path = resolve_object(
|
||||
StoragePath,
|
||||
names,
|
||||
resolve_storage_path_ids(choice["existing_ids"], owner)
|
||||
+ match_storage_paths_by_name(names, owner),
|
||||
create_missing=create_missing,
|
||||
owner=owner,
|
||||
)
|
||||
if storage_path:
|
||||
document.storage_path = storage_path
|
||||
updated_fields.append("storage_path")
|
||||
|
||||
if should_set(AISuggestionField.CREATED):
|
||||
created = resolve_date(suggestions["dates"])
|
||||
if created:
|
||||
document.created = created
|
||||
updated_fields.append("created")
|
||||
|
||||
if AISuggestionField.TAGS in selected:
|
||||
choice = suggestions["tags"]
|
||||
names = choice["new_names"]
|
||||
tags = resolve_tags(
|
||||
names,
|
||||
resolve_tag_ids(choice["existing_ids"], owner)
|
||||
+ match_tags_by_name(names, owner),
|
||||
create_missing=create_missing,
|
||||
owner=owner,
|
||||
)
|
||||
if tags:
|
||||
# Suggested tags are always added, so overwrite_existing
|
||||
# does not really apply here
|
||||
document.add_nested_tags(tags)
|
||||
updated_fields.append("tags")
|
||||
|
||||
if updated_fields:
|
||||
# save fields and update modified (excluding m2m tags from update_fields)
|
||||
direct_updated_fields = [
|
||||
field for field in updated_fields if field in DIRECT_FIELDS.values()
|
||||
]
|
||||
document.save(update_fields=[*direct_updated_fields, "modified"])
|
||||
|
||||
logger.info(
|
||||
"Applied AI suggestions %s to document %s",
|
||||
updated_fields or "(none)",
|
||||
document.pk,
|
||||
extra={"group": logging_group},
|
||||
)
|
||||
|
||||
return updated_fields
|
||||
File diff suppressed because it is too large
Load Diff
@@ -5,12 +5,13 @@ from django.conf import settings
|
||||
from django.contrib.auth.models import User
|
||||
|
||||
from documents.models import Document
|
||||
from documents.permissions import permitted_object_ids
|
||||
from documents.permissions import get_objects_for_user_owner_aware
|
||||
from paperless.config import AIConfig
|
||||
from paperless_ai.base_model import ClassificationSuggestions
|
||||
from paperless_ai.base_model import TaxonomyChoiceDict
|
||||
from paperless_ai.client import AIClient
|
||||
from paperless_ai.db import db_connection_released
|
||||
from paperless_ai.indexing import _node_document_ids
|
||||
from paperless_ai.indexing import retrieve_similar_nodes
|
||||
from paperless_ai.indexing import truncate_content
|
||||
from paperless_ai.prompts.context import ClassificationPromptContext
|
||||
@@ -18,9 +19,7 @@ from paperless_ai.prompts.context import LocalizationPromptContext
|
||||
from paperless_ai.prompts.context import RagContextPromptContext
|
||||
from paperless_ai.prompts.render import render_prompt
|
||||
from paperless_ai.taxonomy import AssignedMetadata
|
||||
from paperless_ai.taxonomy import SimilarDocument
|
||||
from paperless_ai.taxonomy import TaxonomyCandidates
|
||||
from paperless_ai.taxonomy import _node_document_weights
|
||||
from paperless_ai.taxonomy import build_taxonomy_candidates
|
||||
from paperless_ai.taxonomy import empty_taxonomy_candidates
|
||||
from paperless_ai.taxonomy import format_taxonomy_for_prompt
|
||||
@@ -40,35 +39,6 @@ logger = logging.getLogger("paperless_ai.rag_classifier")
|
||||
TAXONOMY_CANDIDATE_TOP_K = 15
|
||||
|
||||
|
||||
def _fulltext_similar_documents(
|
||||
document: Document,
|
||||
user: User | None,
|
||||
top_k: int,
|
||||
) -> list[SimilarDocument]:
|
||||
"""Rank-based fallback when no embedding backend is configured. Uses
|
||||
Tantivy's "More Like This" (term-overlap similarity) instead of vector
|
||||
similarity - cruder, but far better than no candidates at all.
|
||||
more_like_this_ids returns only a ranked ID list, no scores, so weight is
|
||||
synthesized from rank (descending from top_k) rather than claiming a
|
||||
similarity magnitude that doesn't exist. A superuser is normalized to
|
||||
``None`` before calling, since the backend's permission filter has no
|
||||
superuser short-circuit of its own.
|
||||
"""
|
||||
from documents.search import get_backend
|
||||
|
||||
search_user = None if user is not None and user.is_superuser else user
|
||||
backend = get_backend()
|
||||
similar_ids = backend.more_like_this_ids(
|
||||
document.pk,
|
||||
user=search_user,
|
||||
limit=top_k,
|
||||
)
|
||||
return [
|
||||
SimilarDocument(document_id=doc_id, weight=float(top_k - rank))
|
||||
for rank, doc_id in enumerate(similar_ids)
|
||||
]
|
||||
|
||||
|
||||
def get_language_name(language_code: str) -> str:
|
||||
normalized_language_code = language_code.lower()
|
||||
for code, name in settings.LANGUAGES:
|
||||
@@ -77,6 +47,22 @@ def get_language_name(language_code: str) -> str:
|
||||
return language_code
|
||||
|
||||
|
||||
def get_llm_output_language(ai_config: AIConfig, user: User | None) -> str | None:
|
||||
"""
|
||||
Language to localize LLM output into: the configured language, falling back
|
||||
to the user's own UI language when unset.
|
||||
"""
|
||||
output_language = ai_config.llm_output_language
|
||||
if (
|
||||
not output_language
|
||||
and user is not None
|
||||
and hasattr(user, "ui_settings")
|
||||
and isinstance(user.ui_settings.settings, dict)
|
||||
):
|
||||
output_language = user.ui_settings.settings.get("language")
|
||||
return output_language
|
||||
|
||||
|
||||
def build_prompt_without_rag(
|
||||
document: Document,
|
||||
config: AIConfig,
|
||||
@@ -161,52 +147,44 @@ def get_taxonomy_context(
|
||||
user: User | None = None,
|
||||
max_docs: int = 5,
|
||||
) -> tuple[TaxonomyCandidates, AssignedMetadata, str]:
|
||||
"""One retrieval feeds both taxonomy candidates and RAG text context. Uses
|
||||
vector similarity when an embedding backend is configured, otherwise
|
||||
falls back to Tantivy full-text "More Like This" similarity - see
|
||||
_fulltext_similar_documents. On any retrieval failure, degrades to empty
|
||||
candidates/context rather than propagating the exception - neither a
|
||||
vector-store outage nor a search-index issue should block classification,
|
||||
only its context-assisted enrichment.
|
||||
"""One retrieval feeds both taxonomy candidates and RAG text context.
|
||||
On any retrieval failure, degrades to empty candidates/context rather than
|
||||
propagating the exception - a vector-store outage should not block
|
||||
classification, only its RAG-assisted enrichment.
|
||||
"""
|
||||
assigned = get_assigned_metadata(document, user)
|
||||
ai_config = AIConfig()
|
||||
try:
|
||||
if ai_config.llm_embedding_backend:
|
||||
# None means "no restriction" to retrieve_similar_nodes. A superuser
|
||||
# (like no user at all) can see every document, so skip materializing
|
||||
# every visible pk into a Python list and passing it through as an IN
|
||||
# filter: for a large library that is a wasted quadratic scan in the
|
||||
# vector store at best, and past ~32,763 documents a hard
|
||||
# sqlite3.OperationalError (SQLite's bound-parameter limit) at worst.
|
||||
# permitted_object_ids() has its own superuser shortcut that would
|
||||
# return every Document's id anyway, so this changes nothing about
|
||||
# which documents are considered -- only how we get there.
|
||||
visible_document_ids = (
|
||||
None
|
||||
if user is None or user.is_superuser
|
||||
else list(permitted_object_ids(user, Document, "view_document"))
|
||||
)
|
||||
nodes = retrieve_similar_nodes(
|
||||
document,
|
||||
top_k=TAXONOMY_CANDIDATE_TOP_K,
|
||||
document_ids=visible_document_ids,
|
||||
)
|
||||
similar_documents = _node_document_weights(nodes)
|
||||
else:
|
||||
# See _fulltext_similar_documents: it applies its own permission
|
||||
# filter via `user`, so no visible-document-id list is needed here.
|
||||
similar_documents = _fulltext_similar_documents(
|
||||
document,
|
||||
user,
|
||||
top_k=TAXONOMY_CANDIDATE_TOP_K,
|
||||
# None means "no restriction" to retrieve_similar_nodes. A superuser
|
||||
# (like no user at all) can see every document, so skip materializing
|
||||
# every visible pk into a Python list and passing it through as an IN
|
||||
# filter: for a large library that is a wasted quadratic scan in the
|
||||
# vector store at best, and past ~32,763 documents a hard
|
||||
# sqlite3.OperationalError (SQLite's bound-parameter limit) at worst.
|
||||
# get_objects_for_user_owner_aware() would return every Document for a
|
||||
# superuser anyway (guardian's own with_superuser shortcut), so this
|
||||
# changes nothing about which documents are considered -- only how we
|
||||
# get there.
|
||||
visible_document_ids = (
|
||||
None
|
||||
if user is None or user.is_superuser
|
||||
else list(
|
||||
get_objects_for_user_owner_aware(
|
||||
user,
|
||||
"view_document",
|
||||
Document,
|
||||
).values_list("pk", flat=True),
|
||||
)
|
||||
)
|
||||
nodes = retrieve_similar_nodes(
|
||||
document,
|
||||
top_k=TAXONOMY_CANDIDATE_TOP_K,
|
||||
document_ids=visible_document_ids,
|
||||
)
|
||||
|
||||
candidates = build_taxonomy_candidates(similar_documents, user)
|
||||
candidates = build_taxonomy_candidates(nodes, user)
|
||||
|
||||
similar_doc_ids = [s["document_id"] for s in similar_documents]
|
||||
similar_docs = list(
|
||||
Document.objects.filter(pk__in=similar_doc_ids)[:max_docs],
|
||||
Document.objects.filter(pk__in=_node_document_ids(nodes))[:max_docs],
|
||||
)
|
||||
context_blocks = []
|
||||
for similar in similar_docs:
|
||||
@@ -215,8 +193,8 @@ def get_taxonomy_context(
|
||||
context_blocks.append(f"TITLE: {title}\n{text}")
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Failed to retrieve similar-document context for document %s; "
|
||||
"continuing without taxonomy candidates or similar-document context.",
|
||||
"Failed to retrieve RAG neighbours for document %s; continuing "
|
||||
"without taxonomy candidates or similar-document context.",
|
||||
document.pk,
|
||||
)
|
||||
return empty_taxonomy_candidates(), assigned, ""
|
||||
@@ -299,14 +277,23 @@ def get_ai_document_classification(
|
||||
) -> ClassificationSuggestions:
|
||||
ai_config = AIConfig()
|
||||
|
||||
candidates, assigned, context = get_taxonomy_context(document, user)
|
||||
prompt = build_prompt_with_rag(
|
||||
document,
|
||||
ai_config,
|
||||
candidates=candidates,
|
||||
assigned=assigned,
|
||||
context=context,
|
||||
)
|
||||
if ai_config.llm_embedding_backend:
|
||||
candidates, assigned, context = get_taxonomy_context(document, user)
|
||||
prompt = build_prompt_with_rag(
|
||||
document,
|
||||
ai_config,
|
||||
candidates=candidates,
|
||||
assigned=assigned,
|
||||
context=context,
|
||||
)
|
||||
else:
|
||||
candidates = empty_taxonomy_candidates()
|
||||
prompt = build_prompt_without_rag(
|
||||
document,
|
||||
ai_config,
|
||||
candidates=candidates,
|
||||
assigned=get_assigned_metadata(document, user),
|
||||
)
|
||||
|
||||
client = AIClient()
|
||||
# Hand the pooled DB connection back while the (slow) LLM query runs so it
|
||||
|
||||
@@ -11,7 +11,7 @@ from documents.models import Correspondent
|
||||
from documents.models import DocumentType
|
||||
from documents.models import StoragePath
|
||||
from documents.models import Tag
|
||||
from documents.permissions import get_objects_for_user_owner_aware
|
||||
from documents.permissions import permitted_object_ids
|
||||
from documents.permissions import restrict_queryset_to_visible
|
||||
|
||||
MATCH_THRESHOLD = 0.8
|
||||
@@ -63,30 +63,40 @@ def resolve_storage_path_ids(ids: list[int], user: User | None) -> list[StorageP
|
||||
|
||||
def _match_by_name(
|
||||
names: list[str],
|
||||
user: User,
|
||||
user: User | None,
|
||||
model: type[ModelT],
|
||||
perm: str,
|
||||
) -> list[ModelT]:
|
||||
queryset = get_objects_for_user_owner_aware(user, [perm], model)
|
||||
# A workflow may have no user. In that case permitted_object_ids limits
|
||||
# matching to unowned objects, avoiding another user's private taxonomy.
|
||||
queryset = model.objects.filter(
|
||||
pk__in=permitted_object_ids(user, model, perm),
|
||||
)
|
||||
return _match_names_to_queryset(names, queryset)
|
||||
|
||||
|
||||
def match_tags_by_name(names: list[str], user: User) -> list[Tag]:
|
||||
def match_tags_by_name(names: list[str], user: User | None) -> list[Tag]:
|
||||
return _match_by_name(names, user, Tag, "view_tag")
|
||||
|
||||
|
||||
def match_correspondents_by_name(
|
||||
names: list[str],
|
||||
user: User,
|
||||
user: User | None,
|
||||
) -> list[Correspondent]:
|
||||
return _match_by_name(names, user, Correspondent, "view_correspondent")
|
||||
|
||||
|
||||
def match_document_types_by_name(names: list[str], user: User) -> list[DocumentType]:
|
||||
def match_document_types_by_name(
|
||||
names: list[str],
|
||||
user: User | None,
|
||||
) -> list[DocumentType]:
|
||||
return _match_by_name(names, user, DocumentType, "view_documenttype")
|
||||
|
||||
|
||||
def match_storage_paths_by_name(names: list[str], user: User) -> list[StoragePath]:
|
||||
def match_storage_paths_by_name(
|
||||
names: list[str],
|
||||
user: User | None,
|
||||
) -> list[StoragePath]:
|
||||
return _match_by_name(names, user, StoragePath, "view_storagepath")
|
||||
|
||||
|
||||
|
||||
@@ -33,11 +33,6 @@ class TaxonomyCandidate(TypedDict):
|
||||
weight: float
|
||||
|
||||
|
||||
class SimilarDocument(TypedDict):
|
||||
document_id: int
|
||||
weight: float
|
||||
|
||||
|
||||
class TaxonomyCandidates(TypedDict):
|
||||
tags: list[TaxonomyCandidate]
|
||||
document_types: list[TaxonomyCandidate]
|
||||
@@ -110,10 +105,10 @@ def get_assigned_metadata(document: Document, user: User | None) -> AssignedMeta
|
||||
)
|
||||
|
||||
|
||||
def _node_document_weights(nodes: list["NodeWithScore"]) -> list[SimilarDocument]:
|
||||
"""Sum each node's similarity score into its document_id (a document can
|
||||
appear via multiple chunks/nodes) and return one SimilarDocument per
|
||||
distinct document_id."""
|
||||
def _node_document_weights(nodes: list["NodeWithScore"]) -> dict[int, float]:
|
||||
"""document_id -> that node's similarity score, summed if a document_id
|
||||
appears more than once across the retrieved nodes (e.g. multiple chunks
|
||||
of the same source document)."""
|
||||
weights: dict[int, float] = defaultdict(float)
|
||||
for node in nodes:
|
||||
document_id = node.metadata.get("document_id")
|
||||
@@ -126,10 +121,7 @@ def _node_document_weights(nodes: list["NodeWithScore"]) -> list[SimilarDocument
|
||||
weights[int(document_id)] += float(node.score or 0.0)
|
||||
except (TypeError, ValueError): # pragma: no cover
|
||||
continue
|
||||
return [
|
||||
SimilarDocument(document_id=document_id, weight=weight)
|
||||
for document_id, weight in weights.items()
|
||||
]
|
||||
return weights
|
||||
|
||||
|
||||
def _visible_ranked_candidates(
|
||||
@@ -165,25 +157,20 @@ def _visible_ranked_candidates(
|
||||
|
||||
|
||||
def build_taxonomy_candidates(
|
||||
similar_documents: list[SimilarDocument],
|
||||
nodes: list["NodeWithScore"],
|
||||
user: User | None,
|
||||
) -> TaxonomyCandidates:
|
||||
"""Resolve each similar document's id to a live Document, read its
|
||||
*current* tags/type/correspondent/storage_path via the ORM (never any
|
||||
possibly-stale names an adapter's source might have cached), weight each
|
||||
distinct taxonomy object by aggregate similarity weight, permission-filter
|
||||
"""Resolve each neighbour node's document_id to a live Document, read its
|
||||
*current* tags/type/correspondent/storage_path via the ORM (never the
|
||||
possibly-stale names cached in vector-index node metadata), weight each
|
||||
distinct taxonomy object by aggregate neighbour similarity, permission-filter
|
||||
against what ``user`` can see, and return each category ranked by weight
|
||||
and capped. ``similar_documents`` may come from either the vector-RAG
|
||||
adapter or the full-text fallback adapter - both produce this same shape.
|
||||
and capped.
|
||||
"""
|
||||
if not similar_documents:
|
||||
return empty_taxonomy_candidates()
|
||||
|
||||
# Both adapters guarantee at most one SimilarDocument per document_id, so
|
||||
# this never silently drops a duplicate's weight.
|
||||
document_weights: dict[int, float] = {
|
||||
s["document_id"]: s["weight"] for s in similar_documents
|
||||
}
|
||||
document_weights = _node_document_weights(nodes)
|
||||
if not document_weights:
|
||||
return empty_taxonomy_candidates()
|
||||
|
||||
# Only .tags.all() needs prefetching (a reverse M2M, one extra query for
|
||||
# the whole batch). document_type/correspondent/storage_path are read
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
from collections.abc import Generator
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
from unittest.mock import patch
|
||||
@@ -8,13 +7,10 @@ import pytest_mock
|
||||
from django.test import override_settings
|
||||
|
||||
from documents.models import Document
|
||||
from documents.search import TantivyBackend
|
||||
from documents.tests.factories import DocumentFactory
|
||||
from documents.tests.factories import TagFactory
|
||||
from documents.tests.factories import UserFactory
|
||||
from paperless.config import AIConfig
|
||||
from paperless_ai.ai_classifier import TAXONOMY_CANDIDATE_TOP_K
|
||||
from paperless_ai.ai_classifier import _fulltext_similar_documents
|
||||
from paperless_ai.ai_classifier import _restrict_to_shown_candidates
|
||||
from paperless_ai.ai_classifier import build_localization_prompt
|
||||
from paperless_ai.ai_classifier import build_prompt_with_rag
|
||||
@@ -24,7 +20,6 @@ from paperless_ai.ai_classifier import get_language_name
|
||||
from paperless_ai.ai_classifier import get_taxonomy_context
|
||||
from paperless_ai.base_model import ClassificationSuggestions
|
||||
from paperless_ai.base_model import TaxonomyChoiceDict
|
||||
from paperless_ai.taxonomy import SimilarDocument
|
||||
from paperless_ai.taxonomy import TaxonomyCandidate
|
||||
from paperless_ai.taxonomy import TaxonomyCandidates
|
||||
from paperless_ai.taxonomy import empty_taxonomy_candidates
|
||||
@@ -209,10 +204,12 @@ def test_use_rag_if_configured(
|
||||
|
||||
@pytest.mark.django_db
|
||||
@patch("paperless_ai.client.AIClient.run_llm_query")
|
||||
@patch("paperless_ai.ai_classifier.build_prompt_with_rag")
|
||||
@patch("paperless_ai.ai_classifier.build_prompt_without_rag")
|
||||
@patch("paperless_ai.ai_classifier.AIConfig")
|
||||
@override_settings(LLM_BACKEND="ollama", LLM_MODEL="some_model")
|
||||
def test_use_rag_prompt_even_without_embedding_backend(
|
||||
mock_build_prompt_with_rag,
|
||||
def test_use_without_rag_if_not_configured(
|
||||
mock_ai_config,
|
||||
mock_build_prompt_without_rag,
|
||||
mock_run_llm_query,
|
||||
mock_document,
|
||||
):
|
||||
@@ -222,13 +219,13 @@ def test_use_rag_prompt_even_without_embedding_backend(
|
||||
WHEN:
|
||||
- get_ai_document_classification() is called
|
||||
THEN:
|
||||
- The RAG-context prompt builder is still used (fed by the full-text
|
||||
fallback's context/candidates instead of the vector store's)
|
||||
- The non-RAG prompt builder is used
|
||||
"""
|
||||
mock_build_prompt_with_rag.return_value = "Prompt with RAG"
|
||||
mock_ai_config.return_value.llm_embedding_backend = None
|
||||
mock_build_prompt_without_rag.return_value = "Prompt without RAG"
|
||||
mock_run_llm_query.return_value = NESTED_SUGGESTIONS
|
||||
get_ai_document_classification(mock_document)
|
||||
mock_build_prompt_with_rag.assert_called_once()
|
||||
mock_build_prompt_without_rag.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@@ -306,7 +303,6 @@ def test_build_localization_prompt_preserves_unicode_characters():
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
|
||||
def test_get_taxonomy_context_assembles_rag_text_and_candidates():
|
||||
"""
|
||||
GIVEN:
|
||||
@@ -348,7 +344,6 @@ def test_get_taxonomy_context_assembles_rag_text_and_candidates():
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
|
||||
def test_get_taxonomy_context_no_similar_docs():
|
||||
"""
|
||||
GIVEN:
|
||||
@@ -372,67 +367,6 @@ def test_get_taxonomy_context_no_similar_docs():
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_get_taxonomy_context_uses_fulltext_fallback_when_no_embedding_backend(
|
||||
mocker: pytest_mock.MockerFixture,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- No LLM embedding backend is configured (the default test settings)
|
||||
WHEN:
|
||||
- get_taxonomy_context() is called
|
||||
THEN:
|
||||
- _fulltext_similar_documents() is called with the document, the user
|
||||
and TAXONOMY_CANDIDATE_TOP_K
|
||||
- retrieve_similar_nodes() (the vector path) is never called
|
||||
"""
|
||||
document = DocumentFactory.create(content="Some content")
|
||||
mock_fulltext = mocker.patch(
|
||||
"paperless_ai.ai_classifier._fulltext_similar_documents",
|
||||
return_value=[],
|
||||
)
|
||||
mock_retrieve = mocker.patch("paperless_ai.ai_classifier.retrieve_similar_nodes")
|
||||
|
||||
get_taxonomy_context(document, user=None)
|
||||
|
||||
mock_fulltext.assert_called_once_with(
|
||||
document,
|
||||
None,
|
||||
top_k=TAXONOMY_CANDIDATE_TOP_K,
|
||||
)
|
||||
mock_retrieve.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
|
||||
def test_get_taxonomy_context_uses_vector_path_when_embedding_backend_configured(
|
||||
mocker: pytest_mock.MockerFixture,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- An LLM embedding backend is configured
|
||||
WHEN:
|
||||
- get_taxonomy_context() is called
|
||||
THEN:
|
||||
- retrieve_similar_nodes() (the vector path) is called
|
||||
- _fulltext_similar_documents() (the no-embedding-backend fallback)
|
||||
is never called
|
||||
"""
|
||||
document = DocumentFactory.create(content="Some content")
|
||||
mock_retrieve = mocker.patch(
|
||||
"paperless_ai.ai_classifier.retrieve_similar_nodes",
|
||||
return_value=[],
|
||||
)
|
||||
mock_fulltext = mocker.patch(
|
||||
"paperless_ai.ai_classifier._fulltext_similar_documents",
|
||||
)
|
||||
|
||||
get_taxonomy_context(document, user=None)
|
||||
|
||||
mock_retrieve.assert_called_once()
|
||||
mock_fulltext.assert_not_called()
|
||||
|
||||
|
||||
class TestGetTaxonomyContextVisibility:
|
||||
"""get_taxonomy_context must not materialize every visible document id
|
||||
for a user who can already see the whole library: a superuser (like no
|
||||
@@ -445,7 +379,6 @@ class TestGetTaxonomyContextVisibility:
|
||||
"""
|
||||
|
||||
@pytest.mark.django_db
|
||||
@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
|
||||
def test_skips_permission_lookup_for_superuser(
|
||||
self,
|
||||
mocker: pytest_mock.MockerFixture,
|
||||
@@ -464,18 +397,17 @@ class TestGetTaxonomyContextVisibility:
|
||||
"paperless_ai.ai_classifier.retrieve_similar_nodes",
|
||||
return_value=[],
|
||||
)
|
||||
mock_permitted = mocker.patch(
|
||||
"paperless_ai.ai_classifier.permitted_object_ids",
|
||||
mock_get_objects = mocker.patch(
|
||||
"paperless_ai.ai_classifier.get_objects_for_user_owner_aware",
|
||||
)
|
||||
user = UserFactory.create(is_superuser=True)
|
||||
|
||||
get_taxonomy_context(document, user)
|
||||
|
||||
mock_permitted.assert_not_called()
|
||||
mock_get_objects.assert_not_called()
|
||||
assert mock_retrieve.call_args.kwargs["document_ids"] is None
|
||||
|
||||
@pytest.mark.django_db
|
||||
@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
|
||||
def test_skips_permission_lookup_when_no_user(
|
||||
self,
|
||||
mocker: pytest_mock.MockerFixture,
|
||||
@@ -494,17 +426,16 @@ class TestGetTaxonomyContextVisibility:
|
||||
"paperless_ai.ai_classifier.retrieve_similar_nodes",
|
||||
return_value=[],
|
||||
)
|
||||
mock_permitted = mocker.patch(
|
||||
"paperless_ai.ai_classifier.permitted_object_ids",
|
||||
mock_get_objects = mocker.patch(
|
||||
"paperless_ai.ai_classifier.get_objects_for_user_owner_aware",
|
||||
)
|
||||
|
||||
get_taxonomy_context(document, None)
|
||||
|
||||
mock_permitted.assert_not_called()
|
||||
mock_get_objects.assert_not_called()
|
||||
assert mock_retrieve.call_args.kwargs["document_ids"] is None
|
||||
|
||||
@pytest.mark.django_db
|
||||
@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
|
||||
def test_restricts_to_visible_documents_for_non_superuser(
|
||||
self,
|
||||
mocker: pytest_mock.MockerFixture,
|
||||
@@ -515,7 +446,7 @@ class TestGetTaxonomyContextVisibility:
|
||||
WHEN:
|
||||
- get_taxonomy_context() is called
|
||||
THEN:
|
||||
- The user's permitted document ids are looked up and passed to
|
||||
- The user's visible document ids are looked up and passed to
|
||||
retrieve_similar_nodes() as a restriction
|
||||
"""
|
||||
document = DocumentFactory.create(content="Some content")
|
||||
@@ -523,185 +454,21 @@ class TestGetTaxonomyContextVisibility:
|
||||
"paperless_ai.ai_classifier.retrieve_similar_nodes",
|
||||
return_value=[],
|
||||
)
|
||||
mock_permitted = mocker.patch(
|
||||
"paperless_ai.ai_classifier.permitted_object_ids",
|
||||
return_value=[1, 2, 3],
|
||||
mock_queryset = mocker.MagicMock()
|
||||
mock_queryset.values_list.return_value = [1, 2, 3]
|
||||
mock_get_objects = mocker.patch(
|
||||
"paperless_ai.ai_classifier.get_objects_for_user_owner_aware",
|
||||
return_value=mock_queryset,
|
||||
)
|
||||
user = UserFactory.create(is_superuser=False)
|
||||
|
||||
get_taxonomy_context(document, user)
|
||||
|
||||
mock_permitted.assert_called_once_with(user, Document, "view_document")
|
||||
mock_get_objects.assert_called_once_with(user, "view_document", Document)
|
||||
assert mock_retrieve.call_args.kwargs["document_ids"] == [1, 2, 3]
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestFulltextSimilarDocuments:
|
||||
"""_fulltext_similar_documents is the no-embedding-backend fallback: it
|
||||
asks the Tantivy full-text index for "More Like This" neighbours instead
|
||||
of the vector store, and synthesizes a rank-based weight since Tantivy's
|
||||
more_like_this_ids returns only an ordered id list, no scores.
|
||||
"""
|
||||
|
||||
@pytest.fixture
|
||||
def fulltext_backend(
|
||||
self,
|
||||
mocker: pytest_mock.MockerFixture,
|
||||
) -> Generator[TantivyBackend, None, None]:
|
||||
"""An in-memory Tantivy backend, wired up as the module-level
|
||||
singleton _fulltext_similar_documents resolves via get_backend()."""
|
||||
backend = TantivyBackend(path=None)
|
||||
backend.open()
|
||||
mocker.patch("documents.search.get_backend", return_value=backend)
|
||||
try:
|
||||
yield backend
|
||||
finally:
|
||||
backend.close()
|
||||
|
||||
def test_ranks_by_rank_based_weight_descending(
|
||||
self,
|
||||
fulltext_backend: TantivyBackend,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A source document and two similar documents indexed in Tantivy
|
||||
WHEN:
|
||||
- _fulltext_similar_documents() is called
|
||||
THEN:
|
||||
- Each result's weight reflects its rank (first result weighted
|
||||
higher than the second), not a raw similarity score
|
||||
"""
|
||||
source = DocumentFactory.create(content="quarterly financial report details")
|
||||
first = DocumentFactory.create(content="quarterly financial report details")
|
||||
second = DocumentFactory.create(content="financial report")
|
||||
for doc in (source, first, second):
|
||||
fulltext_backend.add_or_update(doc)
|
||||
|
||||
result = _fulltext_similar_documents(source, user=None, top_k=5)
|
||||
|
||||
assert len(result) == 2
|
||||
weight_by_id = {s["document_id"]: s["weight"] for s in result}
|
||||
assert weight_by_id[first.pk] > weight_by_id[second.pk]
|
||||
|
||||
def test_excludes_source_document(
|
||||
self,
|
||||
fulltext_backend: TantivyBackend,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A source document indexed in Tantivy with no other documents
|
||||
WHEN:
|
||||
- _fulltext_similar_documents() is called
|
||||
THEN:
|
||||
- An empty list is returned - the source document is never its
|
||||
own similar document
|
||||
"""
|
||||
source = DocumentFactory.create(content="unique unrelated content")
|
||||
fulltext_backend.add_or_update(source)
|
||||
|
||||
result = _fulltext_similar_documents(source, user=None, top_k=5)
|
||||
|
||||
assert result == []
|
||||
|
||||
def test_empty_index_returns_empty_list(
|
||||
self,
|
||||
fulltext_backend: TantivyBackend,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A document that has never been indexed (fresh/empty Tantivy index)
|
||||
WHEN:
|
||||
- _fulltext_similar_documents() is called
|
||||
THEN:
|
||||
- An empty list is returned rather than raising
|
||||
"""
|
||||
source = DocumentFactory.create(content="never indexed")
|
||||
|
||||
result = _fulltext_similar_documents(source, user=None, top_k=5)
|
||||
|
||||
assert result == []
|
||||
|
||||
def test_respects_top_k_limit(
|
||||
self,
|
||||
fulltext_backend: TantivyBackend,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A source document and four similar documents indexed
|
||||
WHEN:
|
||||
- _fulltext_similar_documents() is called with top_k=2
|
||||
THEN:
|
||||
- At most 2 results are returned
|
||||
"""
|
||||
source = DocumentFactory.create(content="shared overlapping keyword text")
|
||||
for _ in range(4):
|
||||
fulltext_backend.add_or_update(
|
||||
DocumentFactory.create(content="shared overlapping keyword text"),
|
||||
)
|
||||
|
||||
result = _fulltext_similar_documents(source, user=None, top_k=2)
|
||||
|
||||
assert len(result) <= 2
|
||||
|
||||
def test_result_shape_is_similar_document(
|
||||
self,
|
||||
fulltext_backend: TantivyBackend,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A source document and one similar document indexed
|
||||
WHEN:
|
||||
- _fulltext_similar_documents() is called
|
||||
THEN:
|
||||
- Each result is a SimilarDocument (document_id + weight only)
|
||||
"""
|
||||
source = DocumentFactory.create(content="shared content phrase")
|
||||
other = DocumentFactory.create(content="shared content phrase")
|
||||
fulltext_backend.add_or_update(source)
|
||||
fulltext_backend.add_or_update(other)
|
||||
|
||||
result = _fulltext_similar_documents(source, user=None, top_k=5)
|
||||
|
||||
# rank 0 (the only/best result) with top_k=5 -> weight = top_k - rank = 5.0,
|
||||
# per the "first result gets top_k, the last gets 1" formula.
|
||||
assert result == [SimilarDocument(document_id=other.pk, weight=5.0)]
|
||||
|
||||
def test_superuser_sees_other_users_documents(
|
||||
self,
|
||||
fulltext_backend: TantivyBackend,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A source document owned by one user and a similar document
|
||||
owned by a different user, with no sharing between them
|
||||
WHEN:
|
||||
- _fulltext_similar_documents() is called with a superuser
|
||||
THEN:
|
||||
- The other user's document is still returned as a similar
|
||||
document - a superuser must not be narrowed by the backend's
|
||||
owner-based permission filter
|
||||
"""
|
||||
owner = UserFactory.create()
|
||||
other_owner = UserFactory.create()
|
||||
superuser = UserFactory.create(is_superuser=True)
|
||||
source = DocumentFactory.create(
|
||||
content="shared content phrase",
|
||||
owner=owner,
|
||||
)
|
||||
other = DocumentFactory.create(
|
||||
content="shared content phrase",
|
||||
owner=other_owner,
|
||||
)
|
||||
fulltext_backend.add_or_update(source)
|
||||
fulltext_backend.add_or_update(other)
|
||||
|
||||
result = _fulltext_similar_documents(source, user=superuser, top_k=5)
|
||||
|
||||
assert [s["document_id"] for s in result] == [other.pk]
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
|
||||
@patch("paperless_ai.ai_classifier.retrieve_similar_nodes")
|
||||
def test_get_taxonomy_context_retrieval_failure_degrades_to_no_hints(mock_retrieve):
|
||||
"""
|
||||
@@ -728,7 +495,6 @@ def test_get_taxonomy_context_retrieval_failure_degrades_to_no_hints(mock_retrie
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
|
||||
@patch("paperless_ai.ai_classifier.build_taxonomy_candidates")
|
||||
@patch("paperless_ai.ai_classifier.retrieve_similar_nodes")
|
||||
def test_get_taxonomy_context_candidate_building_failure_degrades_to_no_hints(
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
from collections.abc import Callable
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import pytest_mock
|
||||
@@ -45,33 +44,25 @@ class TestAIMatching(TestCase):
|
||||
self.storage_path1 = StoragePath.objects.create(name="Test Storage Path 1")
|
||||
self.storage_path2 = StoragePath.objects.create(name="Test Storage Path 2")
|
||||
|
||||
@patch("paperless_ai.matching.get_objects_for_user_owner_aware")
|
||||
def test_match_tags_by_name(self, mock_get_objects) -> None:
|
||||
mock_get_objects.return_value = Tag.objects.all()
|
||||
def test_match_tags_by_name(self) -> None:
|
||||
names = ["Test Tag 1", "Nonexistent Tag"]
|
||||
result = match_tags_by_name(names, user=None)
|
||||
self.assertEqual(len(result), 1)
|
||||
self.assertEqual(result[0].name, "Test Tag 1")
|
||||
|
||||
@patch("paperless_ai.matching.get_objects_for_user_owner_aware")
|
||||
def test_match_correspondents_by_name(self, mock_get_objects) -> None:
|
||||
mock_get_objects.return_value = Correspondent.objects.all()
|
||||
def test_match_correspondents_by_name(self) -> None:
|
||||
names = ["Test Correspondent 1", "Nonexistent Correspondent"]
|
||||
result = match_correspondents_by_name(names, user=None)
|
||||
self.assertEqual(len(result), 1)
|
||||
self.assertEqual(result[0].name, "Test Correspondent 1")
|
||||
|
||||
@patch("paperless_ai.matching.get_objects_for_user_owner_aware")
|
||||
def test_match_document_types_by_name(self, mock_get_objects) -> None:
|
||||
mock_get_objects.return_value = DocumentType.objects.all()
|
||||
def test_match_document_types_by_name(self) -> None:
|
||||
names = ["Test Document Type 1", "Nonexistent Document Type"]
|
||||
result = match_document_types_by_name(names, user=None)
|
||||
self.assertEqual(len(result), 1)
|
||||
self.assertEqual(result[0].name, "Test Document Type 1")
|
||||
|
||||
@patch("paperless_ai.matching.get_objects_for_user_owner_aware")
|
||||
def test_match_storage_paths_by_name(self, mock_get_objects) -> None:
|
||||
mock_get_objects.return_value = StoragePath.objects.all()
|
||||
def test_match_storage_paths_by_name(self) -> None:
|
||||
names = ["Test Storage Path 1", "Nonexistent Storage Path"]
|
||||
result = match_storage_paths_by_name(names, user=None)
|
||||
self.assertEqual(len(result), 1)
|
||||
@@ -83,16 +74,12 @@ class TestAIMatching(TestCase):
|
||||
unmatched_names = extract_unmatched_names(llm_names, matched_objects)
|
||||
self.assertEqual(unmatched_names, ["Nonexistent Tag"])
|
||||
|
||||
@patch("paperless_ai.matching.get_objects_for_user_owner_aware")
|
||||
def test_match_tags_by_name_with_empty_names(self, mock_get_objects) -> None:
|
||||
mock_get_objects.return_value = Tag.objects.all()
|
||||
def test_match_tags_by_name_with_empty_names(self) -> None:
|
||||
names = [None, "", " "]
|
||||
result = match_tags_by_name(names, user=None)
|
||||
self.assertEqual(result, [])
|
||||
|
||||
@patch("paperless_ai.matching.get_objects_for_user_owner_aware")
|
||||
def test_match_tags_with_fuzzy_matching(self, mock_get_objects) -> None:
|
||||
mock_get_objects.return_value = Tag.objects.all()
|
||||
def test_match_tags_with_fuzzy_matching(self) -> None:
|
||||
names = ["Test Taag 1", "Teest Tag 2"]
|
||||
result = match_tags_by_name(names, user=None)
|
||||
self.assertEqual(len(result), 2)
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import pytest_mock
|
||||
@@ -10,7 +11,6 @@ from documents.tests.factories import StoragePathFactory
|
||||
from documents.tests.factories import TagFactory
|
||||
from documents.tests.factories import UserFactory
|
||||
from paperless_ai.taxonomy import AssignedMetadata
|
||||
from paperless_ai.taxonomy import SimilarDocument
|
||||
from paperless_ai.taxonomy import TaxonomyCandidates
|
||||
from paperless_ai.taxonomy import build_taxonomy_candidates
|
||||
from paperless_ai.taxonomy import format_taxonomy_for_prompt
|
||||
@@ -132,8 +132,9 @@ class TestGetAssignedMetadata:
|
||||
assert result["tags"] == ["Owned By Someone Else"]
|
||||
|
||||
|
||||
def make_similar(document_id: int, weight: float) -> SimilarDocument:
|
||||
return SimilarDocument(document_id=document_id, weight=weight)
|
||||
def make_node(document_id: int, score: float) -> SimpleNamespace:
|
||||
"""A stand-in for NodeWithScore: only ``.metadata``/``.score`` are read."""
|
||||
return SimpleNamespace(metadata={"document_id": str(document_id)}, score=score)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@@ -169,9 +170,9 @@ class TestBuildTaxonomyCandidates:
|
||||
doc_a.tags.add(tag)
|
||||
doc_b = DocumentFactory.create()
|
||||
doc_b.tags.add(tag)
|
||||
similar_documents = [make_similar(doc_a.pk, 0.9), make_similar(doc_b.pk, 0.4)]
|
||||
nodes = [make_node(doc_a.pk, 0.9), make_node(doc_b.pk, 0.4)]
|
||||
|
||||
result = build_taxonomy_candidates(similar_documents, user=None)
|
||||
result = build_taxonomy_candidates(nodes, user=None)
|
||||
|
||||
assert len(result["tags"]) == 1
|
||||
assert result["tags"][0]["id"] == tag.pk
|
||||
@@ -196,9 +197,9 @@ class TestBuildTaxonomyCandidates:
|
||||
document.tags.add(tag)
|
||||
tag.name = "New Name"
|
||||
tag.save()
|
||||
similar_documents = [make_similar(document.pk, 0.5)]
|
||||
nodes = [make_node(document.pk, 0.5)]
|
||||
|
||||
result = build_taxonomy_candidates(similar_documents, user=None)
|
||||
result = build_taxonomy_candidates(nodes, user=None)
|
||||
|
||||
assert result["tags"][0]["name"] == "New Name"
|
||||
|
||||
@@ -218,9 +219,9 @@ class TestBuildTaxonomyCandidates:
|
||||
document = DocumentFactory.create()
|
||||
document.tags.add(tag)
|
||||
tag.delete()
|
||||
similar_documents = [make_similar(document.pk, 0.5)]
|
||||
nodes = [make_node(document.pk, 0.5)]
|
||||
|
||||
result = build_taxonomy_candidates(similar_documents, user=None)
|
||||
result = build_taxonomy_candidates(nodes, user=None)
|
||||
|
||||
assert result["tags"] == []
|
||||
|
||||
@@ -239,12 +240,9 @@ class TestBuildTaxonomyCandidates:
|
||||
strong_doc.tags.add(strong_tag)
|
||||
weak_doc = DocumentFactory.create()
|
||||
weak_doc.tags.add(weak_tag)
|
||||
similar_documents = [
|
||||
make_similar(strong_doc.pk, 0.9),
|
||||
make_similar(weak_doc.pk, 0.1),
|
||||
]
|
||||
nodes = [make_node(strong_doc.pk, 0.9), make_node(weak_doc.pk, 0.1)]
|
||||
|
||||
result = build_taxonomy_candidates(similar_documents, user=None)
|
||||
result = build_taxonomy_candidates(nodes, user=None)
|
||||
|
||||
assert [c["name"] for c in result["tags"]] == ["Strong", "Weak"]
|
||||
|
||||
@@ -260,9 +258,9 @@ class TestBuildTaxonomyCandidates:
|
||||
document = DocumentFactory.create()
|
||||
for i in range(15):
|
||||
document.tags.add(TagFactory.create(name=f"Tag{i}"))
|
||||
similar_documents = [make_similar(document.pk, 0.5)]
|
||||
nodes = [make_node(document.pk, 0.5)]
|
||||
|
||||
result = build_taxonomy_candidates(similar_documents, user=None)
|
||||
result = build_taxonomy_candidates(nodes, user=None)
|
||||
|
||||
assert len(result["tags"]) == 10
|
||||
|
||||
@@ -276,12 +274,12 @@ class TestBuildTaxonomyCandidates:
|
||||
- Only 5 correspondents are returned
|
||||
"""
|
||||
correspondents = CorrespondentFactory.create_batch(7)
|
||||
similar_documents = [
|
||||
make_similar(DocumentFactory.create(correspondent=c).pk, 0.5)
|
||||
nodes = [
|
||||
make_node(DocumentFactory.create(correspondent=c).pk, 0.5)
|
||||
for c in correspondents
|
||||
]
|
||||
|
||||
result = build_taxonomy_candidates(similar_documents, user=None)
|
||||
result = build_taxonomy_candidates(nodes, user=None)
|
||||
|
||||
assert len(result["correspondents"]) == 5
|
||||
|
||||
@@ -296,9 +294,9 @@ class TestBuildTaxonomyCandidates:
|
||||
"""
|
||||
document_type = DocumentTypeFactory.create(name="Invoice")
|
||||
document = DocumentFactory.create(document_type=document_type)
|
||||
similar_documents = [make_similar(document.pk, 0.5)]
|
||||
nodes = [make_node(document.pk, 0.5)]
|
||||
|
||||
result = build_taxonomy_candidates(similar_documents, user=None)
|
||||
result = build_taxonomy_candidates(nodes, user=None)
|
||||
|
||||
assert len(result["document_types"]) == 1
|
||||
assert result["document_types"][0]["id"] == document_type.pk
|
||||
@@ -314,12 +312,12 @@ class TestBuildTaxonomyCandidates:
|
||||
- Only 5 document_types are returned
|
||||
"""
|
||||
document_types = DocumentTypeFactory.create_batch(7)
|
||||
similar_documents = [
|
||||
make_similar(DocumentFactory.create(document_type=dt).pk, 0.5)
|
||||
nodes = [
|
||||
make_node(DocumentFactory.create(document_type=dt).pk, 0.5)
|
||||
for dt in document_types
|
||||
]
|
||||
|
||||
result = build_taxonomy_candidates(similar_documents, user=None)
|
||||
result = build_taxonomy_candidates(nodes, user=None)
|
||||
|
||||
assert len(result["document_types"]) == 5
|
||||
|
||||
@@ -334,9 +332,9 @@ class TestBuildTaxonomyCandidates:
|
||||
"""
|
||||
storage_path = StoragePathFactory.create(name="Invoices")
|
||||
document = DocumentFactory.create(storage_path=storage_path)
|
||||
similar_documents = [make_similar(document.pk, 0.5)]
|
||||
nodes = [make_node(document.pk, 0.5)]
|
||||
|
||||
result = build_taxonomy_candidates(similar_documents, user=None)
|
||||
result = build_taxonomy_candidates(nodes, user=None)
|
||||
|
||||
assert len(result["storage_paths"]) == 1
|
||||
assert result["storage_paths"][0]["id"] == storage_path.pk
|
||||
@@ -352,12 +350,12 @@ class TestBuildTaxonomyCandidates:
|
||||
- Only 5 storage_paths are returned
|
||||
"""
|
||||
storage_paths = StoragePathFactory.create_batch(7)
|
||||
similar_documents = [
|
||||
make_similar(DocumentFactory.create(storage_path=sp).pk, 0.5)
|
||||
nodes = [
|
||||
make_node(DocumentFactory.create(storage_path=sp).pk, 0.5)
|
||||
for sp in storage_paths
|
||||
]
|
||||
|
||||
result = build_taxonomy_candidates(similar_documents, user=None)
|
||||
result = build_taxonomy_candidates(nodes, user=None)
|
||||
|
||||
assert len(result["storage_paths"]) == 5
|
||||
|
||||
@@ -377,14 +375,14 @@ class TestBuildTaxonomyCandidates:
|
||||
tag = TagFactory.create(name="Restricted")
|
||||
document = DocumentFactory.create()
|
||||
document.tags.add(tag)
|
||||
similar_documents = [make_similar(document.pk, 0.5)]
|
||||
nodes = [make_node(document.pk, 0.5)]
|
||||
user = UserFactory.create()
|
||||
mocker.patch(
|
||||
"documents.permissions.permitted_object_ids",
|
||||
return_value=[], # user cannot see this tag
|
||||
)
|
||||
|
||||
result = build_taxonomy_candidates(similar_documents, user=user)
|
||||
result = build_taxonomy_candidates(nodes, user=user)
|
||||
|
||||
assert result["tags"] == []
|
||||
|
||||
@@ -414,10 +412,10 @@ class TestBuildTaxonomyCandidates:
|
||||
tag.save()
|
||||
document = DocumentFactory.create()
|
||||
document.tags.add(tag)
|
||||
similar_documents = [make_similar(document.pk, 0.5)]
|
||||
nodes = [make_node(document.pk, 0.5)]
|
||||
spy = mocker.patch("documents.permissions.permitted_object_ids")
|
||||
|
||||
result = build_taxonomy_candidates(similar_documents, user=None)
|
||||
result = build_taxonomy_candidates(nodes, user=None)
|
||||
|
||||
assert result["tags"][0]["name"] == "Owned"
|
||||
spy.assert_not_called()
|
||||
|
||||
Reference in New Issue
Block a user