Compare commits

..
Author SHA1 Message Date
stumpylog 8e65ba0efa fix: address review feedback on effective_content annotation skip
- _needs_effective_content_annotation() now checks for a non-blank,
  stripped param value rather than mere key presence, matching how
  SearchFilter/TitleContentFilter/EffectiveContentFilter themselves
  no-op on a blank value. An empty ?search= or a saved view with a
  cleared text filter no longer re-triggers the annotation.

- The "versions" prefetch on DocumentViewSet no longer carries content
  for every historical version of every document -- that's unused
  bloat for version-heavy documents. Added
  latest_version_content_prefetch() (versioning.py), a separate,
  windowed prefetch scoped to just the newest version's content per
  root, and taught Document.get_effective_content() to check it first.

- DocumentSerializer.to_representation() no longer unconditionally
  calls get_effective_content(). Added has_prefetched_effective_content()
  (versioning.py) as a cheap upfront check: only resolve version-aware
  content when an SQL annotation or a versions prefetch is already on
  the instance. TrashView and GlobalSearchView build their own
  querysets independently of DocumentViewSet and never display
  document content at all (checked both frontend components), so they
  now keep showing the document's own, unresolved content with zero
  extra queries -- the same behavior as before effective_content
  resolution existed, just generalized past the narrow hasattr() check
  it replaced.
2026-08-25 14:18:50 -07:00
stumpylog 3221c3a3e8 perf: skip effective_content annotation on document list unless filtered on
DocumentViewSet.get_queryset() always attached a correlated subquery
resolving each document's latest version content, even though it's only
needed for the deprecated search/title_content/content__* filter params.
Evaluated for every candidate row before pagination's LIMIT, this is
pathological on MariaDB: its default cardinality estimate for the mostly-
NULL root_document_id self-join drives it to a near-full-table scan per
row instead of using the FK index, turning a normal filtered list request
into a multi-second query (root cause of paperless-ngx#13778's report).

Only attach the annotation when a request actually filters on it. The
common case now relies on Document.get_effective_content()'s existing
prefetch-based fallback instead (extended the "versions" prefetch to
include content), which DocumentSerializer.to_representation() now calls
directly instead of checking for the annotation via hasattr().
2026-08-25 13:18:49 -07:00
18 changed files with 582 additions and 537 deletions
+63 -63
View File
@@ -296,11 +296,11 @@
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">89</context>
<context context-type="linenumber">93</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">91</context>
<context context-type="linenumber">95</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/dashboard/dashboard.component.html</context>
@@ -323,11 +323,11 @@
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">98</context>
<context context-type="linenumber">102</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">100</context>
<context context-type="linenumber">104</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/share-link-bundle-dialog/share-link-bundle-dialog.component.html</context>
@@ -374,15 +374,15 @@
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">56</context>
<context context-type="linenumber">60</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">280</context>
<context context-type="linenumber">284</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">282</context>
<context context-type="linenumber">286</context>
</context-group>
</trans-unit>
<trans-unit id="5890330709052835856" datatype="html">
@@ -727,11 +727,11 @@
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">316</context>
<context context-type="linenumber">320</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">318</context>
<context context-type="linenumber">322</context>
</context-group>
</trans-unit>
<trans-unit id="2272120016352772836" datatype="html">
@@ -1142,11 +1142,11 @@
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">240</context>
<context context-type="linenumber">244</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">242</context>
<context context-type="linenumber">246</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/saved-views/saved-views.component.html</context>
@@ -1711,7 +1711,7 @@
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">211</context>
<context context-type="linenumber">215</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/input/tags/tags.component.ts</context>
@@ -1801,15 +1801,15 @@
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.ts</context>
<context context-type="linenumber">164</context>
<context context-type="linenumber">162</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.ts</context>
<context context-type="linenumber">276</context>
<context context-type="linenumber">274</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.ts</context>
<context context-type="linenumber">306</context>
<context context-type="linenumber">304</context>
</context-group>
</trans-unit>
<trans-unit id="6904866445262015585" datatype="html">
@@ -1820,11 +1820,11 @@
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">303</context>
<context context-type="linenumber">307</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">305</context>
<context context-type="linenumber">309</context>
</context-group>
</trans-unit>
<trans-unit id="8492095365580052820" datatype="html">
@@ -2601,11 +2601,11 @@
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">264</context>
<context context-type="linenumber">268</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">266</context>
<context context-type="linenumber">270</context>
</context-group>
</trans-unit>
<trans-unit id="3818027200170621545" datatype="html">
@@ -2962,11 +2962,11 @@
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">294</context>
<context context-type="linenumber">298</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">296</context>
<context context-type="linenumber">300</context>
</context-group>
</trans-unit>
<trans-unit id="4569276013106377105" datatype="html">
@@ -3275,104 +3275,104 @@
<source>by Paperless-ngx</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">23,24</context>
<context context-type="linenumber">27,28</context>
</context-group>
</trans-unit>
<trans-unit id="7228136119811576789" datatype="html">
<source>User menu</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">40</context>
<context context-type="linenumber">44</context>
</context-group>
</trans-unit>
<trans-unit id="2448391510242468907" datatype="html">
<source>Logged in as <x id="INTERPOLATION" equiv-text="{{this.settingsService.displayName}}"/></source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">48,49</context>
<context context-type="linenumber">52,53</context>
</context-group>
</trans-unit>
<trans-unit id="2127032578120864096" datatype="html">
<source>My Profile</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">52</context>
<context context-type="linenumber">56</context>
</context-group>
</trans-unit>
<trans-unit id="3797778920049399855" datatype="html">
<source>Logout</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">59</context>
<context context-type="linenumber">63</context>
</context-group>
</trans-unit>
<trans-unit id="4895326106573044490" datatype="html">
<source>Documentation</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">64</context>
<context context-type="linenumber">68</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">324,325</context>
<context context-type="linenumber">328,329</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">327</context>
<context context-type="linenumber">331</context>
</context-group>
</trans-unit>
<trans-unit id="472206565520537964" datatype="html">
<source>Saved views</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">108,109</context>
<context context-type="linenumber">112,113</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">138,139</context>
<context context-type="linenumber">142,143</context>
</context-group>
</trans-unit>
<trans-unit id="6988090220128974198" datatype="html">
<source>Open documents</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">147,148</context>
<context context-type="linenumber">151,152</context>
</context-group>
</trans-unit>
<trans-unit id="5687256342387781369" datatype="html">
<source>Close all</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">169</context>
<context context-type="linenumber">173</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">171</context>
<context context-type="linenumber">175</context>
</context-group>
</trans-unit>
<trans-unit id="3897348120591552265" datatype="html">
<source>Manage</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">180,181</context>
<context context-type="linenumber">184,185</context>
</context-group>
</trans-unit>
<trans-unit id="8008131619909556709" datatype="html">
<source>Attributes</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">188</context>
<context context-type="linenumber">192</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">190</context>
<context context-type="linenumber">194</context>
</context-group>
</trans-unit>
<trans-unit id="7437910965833684826" datatype="html">
<source>Correspondents</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">216</context>
<context context-type="linenumber">220</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/suggestions-dropdown/suggestions-dropdown.component.html</context>
@@ -3391,7 +3391,7 @@
<source>Document types</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">221</context>
<context context-type="linenumber">225</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/document-attributes/document-attributes.component.ts</context>
@@ -3402,7 +3402,7 @@
<source>Storage paths</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">226</context>
<context context-type="linenumber">230</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/document-attributes/document-attributes.component.ts</context>
@@ -3413,7 +3413,7 @@
<source>Custom fields</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">231</context>
<context context-type="linenumber">235</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-list/bulk-editor/bulk-editor.component.html</context>
@@ -3432,11 +3432,11 @@
<source>Workflows</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">249</context>
<context context-type="linenumber">253</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">251</context>
<context context-type="linenumber">255</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/manage/workflows/workflows.component.html</context>
@@ -3447,99 +3447,99 @@
<source>Mail</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">257</context>
<context context-type="linenumber">261</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">259</context>
<context context-type="linenumber">263</context>
</context-group>
</trans-unit>
<trans-unit id="7844706011418789951" datatype="html">
<source>Administration</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">274,275</context>
<context context-type="linenumber">278,279</context>
</context-group>
</trans-unit>
<trans-unit id="3008420115644088420" datatype="html">
<source>Configuration</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">287</context>
<context context-type="linenumber">291</context>
</context-group>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">289</context>
<context context-type="linenumber">293</context>
</context-group>
</trans-unit>
<trans-unit id="1534029177398918729" datatype="html">
<source>GitHub</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">334</context>
<context context-type="linenumber">338</context>
</context-group>
</trans-unit>
<trans-unit id="4112664765954374539" datatype="html">
<source>is available.</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">344</context>
<context context-type="linenumber">348</context>
</context-group>
</trans-unit>
<trans-unit id="1175891574282637937" datatype="html">
<source>Click to view.</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">344</context>
<context context-type="linenumber">348</context>
</context-group>
</trans-unit>
<trans-unit id="9811291095862612" datatype="html">
<source>Paperless-ngx can automatically check for updates</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">348</context>
<context context-type="linenumber">352</context>
</context-group>
</trans-unit>
<trans-unit id="894819944961861800" datatype="html">
<source> How does this work? </source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">356</context>
<context context-type="linenumber">360</context>
</context-group>
</trans-unit>
<trans-unit id="509090351011426949" datatype="html">
<source>Update available</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">368</context>
<context context-type="linenumber">372</context>
</context-group>
</trans-unit>
<trans-unit id="1329827712962827905" datatype="html">
<source>Configure update checking</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.html</context>
<context context-type="linenumber">373</context>
<context context-type="linenumber">377</context>
</context-group>
</trans-unit>
<trans-unit id="1542489069631984294" datatype="html">
<source>Sidebar views updated</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.ts</context>
<context context-type="linenumber">427</context>
<context context-type="linenumber">425</context>
</context-group>
</trans-unit>
<trans-unit id="3547923076537026828" datatype="html">
<source>Error updating sidebar views</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.ts</context>
<context context-type="linenumber">430</context>
<context context-type="linenumber">428</context>
</context-group>
</trans-unit>
<trans-unit id="2526035785704676448" datatype="html">
<source>An error occurred while saving update checking settings.</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/app-frame/app-frame.component.ts</context>
<context context-type="linenumber">451</context>
<context context-type="linenumber">449</context>
</context-group>
</trans-unit>
<trans-unit id="4580988005648117665" datatype="html">
@@ -8021,35 +8021,35 @@
<source>Paperless-ngx is running!</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/dashboard/widgets/welcome-widget/welcome-widget.component.html</context>
<context context-type="linenumber">6</context>
<context context-type="linenumber">2</context>
</context-group>
</trans-unit>
<trans-unit id="3326049540711826572" datatype="html">
<source>You&apos;re ready to start uploading documents! Explore the various features of this web app on your own, or start a quick tour using the button below.</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/dashboard/widgets/welcome-widget/welcome-widget.component.html</context>
<context context-type="linenumber">8,9</context>
<context context-type="linenumber">3,4</context>
</context-group>
</trans-unit>
<trans-unit id="4474647174688421179" datatype="html">
<source>More detail on how to use and configure Paperless-ngx is always available in the <x id="START_LINK" ctype="x-a" equiv-text="&lt;a href=&quot;https://docs.paperless-ngx.com&quot; target=&quot;_blank&quot;&gt;"/>documentation<x id="CLOSE_LINK" ctype="x-a" equiv-text="&lt;/a&gt;"/>.</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/dashboard/widgets/welcome-widget/welcome-widget.component.html</context>
<context context-type="linenumber">9,10</context>
<context context-type="linenumber">4,6</context>
</context-group>
</trans-unit>
<trans-unit id="4294899532887357745" datatype="html">
<source>Thanks for being a part of the Paperless-ngx community!</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/dashboard/widgets/welcome-widget/welcome-widget.component.html</context>
<context context-type="linenumber">13,14</context>
<context context-type="linenumber">7,8</context>
</context-group>
</trans-unit>
<trans-unit id="1415832194529539652" datatype="html">
<source>Start the tour</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/dashboard/widgets/welcome-widget/welcome-widget.component.html</context>
<context context-type="linenumber">14</context>
<context context-type="linenumber">8</context>
</context-group>
</trans-unit>
<trans-unit id="7822640317427130239" datatype="html">
@@ -10,12 +10,16 @@
tourAnchor="tour.intro">
@if (!hasCustomBranding) {
<pngx-logo extra_classes="navbar-official-logo px-1" height="2.4rem"></pngx-logo>
<pngx-brand-mark class="brand-mark brand-mark-slim d-none"></pngx-brand-mark>
<svg class="brand-mark brand-mark-slim d-none" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1000 1000" width="1.5em" height="1.5em" fill="currentColor">
<path d="M341,949.1c-6.9-20.3-20.7-61.2-21.9-61-199.6-88.9-182.5-229.8-134.3-347.5,30,137.2,268.8,148.9,146.2,336-.9,2.2,10,27.8,19.5,51.3,22.7-51.9,58.6-115.5,55.8-120.8C178,398.7,724.9,299,807.1,18.5c83,251.5,53.1,659.8-377.4,814.9-2,1.4-63.5,148.6-66.9,150.2-.2-2.1-33.2,2.9-30.1-8.7,1.6-7,4.8-16.2,8.2-25.6h0v-.2h.1ZM323.1,846.2c48.3-71.9-12.7-120.8-56.9-152.2,81.2,107.4,66.4,120.8,56.9,152.2h0Z"/>
</svg>
} @else {
@if (customAppLogo) {
<img class="brand-logo" [src]="customAppLogo" alt="" />
} @else {
<pngx-brand-mark class="brand-mark"></pngx-brand-mark>
<svg class="brand-mark" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1000 1000" width="1.5em" height="1.5em" fill="currentColor">
<path d="M341,949.1c-6.9-20.3-20.7-61.2-21.9-61-199.6-88.9-182.5-229.8-134.3-347.5,30,137.2,268.8,148.9,146.2,336-.9,2.2,10,27.8,19.5,51.3,22.7-51.9,58.6-115.5,55.8-120.8C178,398.7,724.9,299,807.1,18.5c83,251.5,53.1,659.8-377.4,814.9-2,1.4-63.5,148.6-66.9,150.2-.2-2.1-33.2,2.9-30.1-8.7,1.6-7,4.8-16.2,8.2-25.6h0v-.2h.1ZM323.1,846.2c48.3-71.9-12.7-120.8-56.9-152.2,81.2,107.4,66.4,120.8,56.9,152.2h0Z"/>
</svg>
}
<div class="brand-copy ms-2 text-truncate" [class.d-md-none]="slimSidebarEnabled">
<span class="brand-title text-truncate">{{ appTitle }}</span>
@@ -45,7 +45,6 @@ import { TasksService } from 'src/app/services/tasks.service'
import { ToastService } from 'src/app/services/toast.service'
import { environment } from 'src/environments/environment'
import { ChatComponent } from '../chat/chat/chat.component'
import { BrandMarkComponent } from '../common/logo/brand-mark/brand-mark.component'
import { LogoComponent } from '../common/logo/logo.component'
import { ProfileEditDialogComponent } from '../common/profile-edit-dialog/profile-edit-dialog.component'
import { DocumentDetailComponent } from '../document-detail/document-detail.component'
@@ -62,7 +61,6 @@ const SCROLL_THRESHOLD = 16
imports: [
GlobalSearchComponent,
LogoComponent,
BrandMarkComponent,
DocumentTitlePipe,
IfPermissionsDirective,
ToastsDropdownComponent,
@@ -1,3 +0,0 @@
<svg class="w-100 h-100 d-block" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1000 1000" fill="currentColor" aria-hidden="true">
<path d="M341,949.1c-6.9-20.3-20.7-61.2-21.9-61-199.6-88.9-182.5-229.8-134.3-347.5,30,137.2,268.8,148.9,146.2,336-.9,2.2,10,27.8,19.5,51.3,22.7-51.9,58.6-115.5,55.8-120.8C178,398.7,724.9,299,807.1,18.5c83,251.5,53.1,659.8-377.4,814.9-2,1.4-63.5,148.6-66.9,150.2-.2-2.1-33.2,2.9-30.1-8.7,1.6-7,4.8-16.2,8.2-25.6h0v-.2h.1ZM323.1,846.2c48.3-71.9-12.7-120.8-56.9-152.2,81.2,107.4,66.4,120.8,56.9,152.2h0Z"/>
</svg>

Before

Width:  |  Height:  |  Size: 545 B

@@ -1,14 +0,0 @@
import { Component, input } from '@angular/core'
@Component({
selector: 'pngx-brand-mark',
templateUrl: './brand-mark.component.html',
host: {
'[style.width]': 'width()',
'[style.height]': 'height()',
},
})
export class BrandMarkComponent {
readonly width = input<string>(null)
readonly height = input<string>(null)
}
@@ -1,18 +1,10 @@
<ngb-alert class="welcome-widget bg-light shadow-sm pe-3 position-relative overflow-hidden" type="primary" [dismissible]="true" (closed)="dismiss.emit(true)">
<pngx-brand-mark class="welcome-watermark d-none d-md-block position-absolute top-50 translate-middle-y text-primary pe-none" width="26rem" height="26rem"></pngx-brand-mark>
<div class="d-flex align-items-start gap-3 gap-md-4 position-relative">
<pngx-brand-mark class="d-none d-sm-block flex-shrink-0 text-primary" width="3rem" height="3rem"></pngx-brand-mark>
<div class="flex-grow-1">
<h4 class="alert-heading text-primary fw-semibold"><ng-container i18n>Paperless-ngx is running!</ng-container> 🎉</h4>
<div class="welcome-copy text-dark">
<p i18n>You're ready to start uploading documents! Explore the various features of this web app on your own, or start a quick tour using the button below.</p>
<p class="mb-0" i18n>More detail on how to use and configure Paperless-ngx is always available in the <a href="https://docs.paperless-ngx.com" target="_blank">documentation</a>.</p>
</div>
<hr class="opacity-100">
<div class="d-flex flex-column flex-sm-row align-items-sm-center gap-3">
<p class="m-0 text-secondary"><em i18n>Thanks for being a part of the Paperless-ngx community!</em></p>
<button class="btn btn-primary ms-sm-auto flex-shrink-0 align-self-start align-self-sm-auto" (click)="tourService.start()"><ng-container i18n>Start the tour</ng-container> &rarr;</button>
</div>
</div>
<ngb-alert class="pe-3" type="primary" [dismissible]="true" (closed)="dismiss.emit(true)">
<h4 class="alert-heading"><ng-container i18n>Paperless-ngx is running!</ng-container> 🎉</h4>
<p i18n>You're ready to start uploading documents! Explore the various features of this web app on your own, or start a quick tour using the button below.</p>
<p i18n>More detail on how to use and configure Paperless-ngx is always available in the <a href="https://docs.paperless-ngx.com" target="_blank">documentation</a>.</p>
<hr>
<div class="d-flex align-items-end">
<p class="lead fs-6 m-0"><em i18n>Thanks for being a part of the Paperless-ngx community!</em></p>
<button class="btn btn-primary ms-auto flex-shrink-0" (click)="tourService.start()"><ng-container i18n>Start the tour</ng-container> &rarr;</button>
</div>
</ngb-alert>
@@ -1,16 +0,0 @@
.welcome-widget {
border-color: color-mix(in srgb, var(--bs-primary) 50%, transparent);
hr {
border-top-color: color-mix(in srgb, var(--bs-primary) 25%, transparent);
}
}
.welcome-copy {
max-width: 68ch;
}
.welcome-watermark {
right: -6rem;
opacity: .07;
}
@@ -1,13 +1,12 @@
import { Component, EventEmitter, Output, inject } from '@angular/core'
import { NgbAlertModule } from '@ng-bootstrap/ng-bootstrap'
import { TourService } from 'ngx-ui-tour-ng-bootstrap'
import { BrandMarkComponent } from '../../../common/logo/brand-mark/brand-mark.component'
@Component({
selector: 'pngx-welcome-widget',
templateUrl: './welcome-widget.component.html',
styleUrls: ['./welcome-widget.component.scss'],
imports: [NgbAlertModule, BrandMarkComponent],
imports: [NgbAlertModule],
})
export class WelcomeWidgetComponent {
readonly tourService = inject(TourService)
+14
View File
@@ -373,6 +373,7 @@ class Document(SoftDeleteModel, ModelWithOwner): # type: ignore[django-manager-
If the queryset already annotated ``effective_content``, that value is used.
"""
# Here to avoid circular import
from documents.versioning import LATEST_VERSION_CONTENT_PREFETCH_ATTR
from documents.versioning import sort_versions_newest_first
from documents.versioning import versions_newest_first
@@ -382,6 +383,19 @@ class Document(SoftDeleteModel, ModelWithOwner): # type: ignore[django-manager-
if self.root_document_id is not None or self.pk is None:
return self.content
latest_version_prefetch = getattr(
self,
LATEST_VERSION_CONTENT_PREFETCH_ATTR,
None,
)
if latest_version_prefetch is not None:
# Empty list means prefetch ran and found no versions — use own content.
return (
latest_version_prefetch[0].content
if latest_version_prefetch
else self.content
)
prefetched_cache = getattr(self, "_prefetched_objects_cache", None)
prefetched_versions = (
prefetched_cache.get("versions")
+9 -2
View File
@@ -88,6 +88,7 @@ from documents.templating.utils import convert_format_str_to_template_format
from documents.templating.workflows import validate_workflow_template
from documents.validators import uri_validator
from documents.validators import url_validator
from documents.versioning import has_prefetched_effective_content
from documents.versioning import sort_versions_newest_first
if TYPE_CHECKING:
@@ -1146,8 +1147,14 @@ class DocumentSerializer(
def to_representation(self, instance):
doc = super().to_representation(instance)
if "content" in self.fields and hasattr(instance, "effective_content"):
doc["content"] = getattr(instance, "effective_content") or ""
if "content" in self.fields and has_prefetched_effective_content(instance):
# Only resolve version-aware content when it's cheap: an SQL
# annotation or a versions prefetch is already on the instance.
# A caller that set up neither (e.g. TrashView, GlobalSearchView,
# which build their own querysets) gets the document's own,
# unresolved content instead of paying for an extra per-instance
# query -- same as before effective_content resolution existed.
doc["content"] = instance.get_effective_content() or ""
if self.truncate_content and "content" in self.fields:
doc["content"] = doc.get("content")[0:550]
return doc
@@ -0,0 +1,239 @@
from __future__ import annotations
from types import SimpleNamespace
from typing import TYPE_CHECKING
import pytest
from django.db import connection
from django.test.utils import CaptureQueriesContext
from rest_framework import status
from documents.models import Document
from documents.tests.factories import DocumentFactory
from documents.versioning import LATEST_VERSION_CONTENT_PREFETCH_ATTR
from documents.versioning import has_prefetched_effective_content
from documents.versioning import latest_version_content_prefetch
from documents.views import DocumentViewSet
if TYPE_CHECKING:
from rest_framework.test import APIClient
class TestNeedsEffectiveContentAnnotation:
"""
DocumentViewSet._needs_effective_content_annotation() decides whether
the effective_content correlated subquery is worth attaching to the
queryset at all -- see TestDocumentListEffectiveContentAnnotation below
for why. This only checks that decision's own logic (a plain query-param
membership test), not that Django/DRF's filtering machinery works.
"""
@pytest.mark.parametrize(
("params", "expected"),
[
({}, False),
({"ordering": "-added"}, False),
({"tags__id__in": "1,2"}, False),
({"search": ""}, False),
({"search": " "}, False),
({"content__icontains": ""}, False),
({"search": "foo"}, True),
({"title_content": "foo"}, True),
({"content__istartswith": "foo"}, True),
({"content__iendswith": "foo"}, True),
({"content__icontains": "foo"}, True),
({"content__iexact": "foo"}, True),
],
)
def test_detects_content_filter_params(
self,
params: dict[str, str],
expected: bool, # noqa: FBT001
) -> None:
# GIVEN a view bound to a request carrying the given query params
view = DocumentViewSet()
view.request = SimpleNamespace(query_params=params)
# WHEN checking whether the effective_content annotation is needed
# THEN it's needed only for requests that actually filter on it
assert view._needs_effective_content_annotation() is expected
@pytest.mark.django_db
class TestDocumentListEffectiveContentAnnotation:
"""
DocumentViewSet.get_queryset() only attaches the effective_content
correlated subquery when a request actually filters on it. Attaching it
unconditionally re-executes it once per candidate row before the page's
LIMIT is applied -- fine on SQLite/Postgres, but pathological on
MariaDB's default cardinality estimation for the root_document_id
self-join once candidate counts get large (see the root_document_id /
effective_content perf investigation).
"""
def test_list_without_content_filter_skips_annotation_but_returns_latest_content(
self,
admin_client: APIClient,
) -> None:
# GIVEN a root document whose latest version has different content
root = DocumentFactory(content="old-root-content")
DocumentFactory(
root_document=root,
version_index=1,
content="new-version-content",
)
# WHEN listing documents with no search/content-filter param
with CaptureQueriesContext(connection) as ctx:
response = admin_client.get("/api/documents/?fields=id,content")
# THEN the response still reflects the latest version's content...
assert response.status_code == status.HTTP_200_OK
assert response.data["results"] == [
{"id": root.id, "content": "new-version-content"},
]
# ...without the database ever evaluating effective_content per row
assert not any(
"effective_content" in query["sql"] for query in ctx.captured_queries
)
def test_latest_version_content_prefetch_carries_only_the_newest_version(
self,
) -> None:
# GIVEN a root document with two versions
root = DocumentFactory(content="root-content")
DocumentFactory(
root_document=root,
version_index=1,
content="older-version-content",
)
DocumentFactory(
root_document=root,
version_index=2,
content="newest-version-content",
)
# WHEN fetching the root through latest_version_content_prefetch()
fetched_root = (
Document.objects.filter(pk=root.pk)
.prefetch_related(
latest_version_content_prefetch(),
)
.get()
)
# THEN the prefetch carries only the single newest version, not
# every historical version's content (the whole point of not
# reusing the metadata-only "versions" prefetch for this)
latest = getattr(fetched_root, LATEST_VERSION_CONTENT_PREFETCH_ATTR)
assert [v.content for v in latest] == ["newest-version-content"]
class TestHasPrefetchedEffectiveContent:
"""
DocumentSerializer.to_representation() only calls get_effective_content()
when has_prefetched_effective_content() says it's cheap -- otherwise a
caller that never set up an annotation or prefetch (TrashView,
GlobalSearchView, which build their own querysets and don't display
content at all) would pay for a per-instance query nobody asked for.
"""
def test_false_with_no_annotation_or_prefetch(self) -> None:
document = Document()
assert has_prefetched_effective_content(document) is False
def test_true_with_effective_content_annotation(self) -> None:
document = Document()
document.effective_content = "resolved"
assert has_prefetched_effective_content(document) is True
def test_true_with_lean_prefetch_attr_even_when_empty(self) -> None:
document = Document()
setattr(document, LATEST_VERSION_CONTENT_PREFETCH_ATTR, [])
assert has_prefetched_effective_content(document) is True
def test_true_with_metadata_versions_prefetch_cache(self) -> None:
document = Document()
document._prefetched_objects_cache = {"versions": []}
assert has_prefetched_effective_content(document) is True
def _get_effective_content_fallback_queries(
ctx: CaptureQueriesContext,
) -> list[dict[str, str]]:
"""
Document.get_effective_content()'s per-instance fallback (no annotation,
no prefetch) is a `.values_list("content", flat=True).first()` query --
a SELECT of just the content column. Distinct from get_versions()'s own,
unrelated per-instance metadata query (id/checksum/added/etc, no
content) run to build the "versions" response field, which isn't part
of what this test file covers.
"""
return [
q
for q in ctx.captured_queries
if q["sql"].startswith('SELECT "documents_document"."content" FROM')
]
@pytest.mark.django_db
class TestTrashAndGlobalSearchDoNotResolveEffectiveContent:
"""
TrashView and GlobalSearchView serialize Document instances with
DocumentSerializer too, but build their querysets independently of
DocumentViewSet.get_queryset() -- and neither actually displays
document content. They should keep showing the document's own,
unresolved content with no extra query, exactly as before
effective_content resolution existed.
"""
def test_trash_list_shows_unresolved_content_with_no_extra_query(
self,
admin_client: APIClient,
) -> None:
# GIVEN a trashed root document whose own content differs from what
# a (also trashed, since deletion cascades) version would have had
root = DocumentFactory(content="own-content")
DocumentFactory(
root_document=root,
version_index=1,
content="version-content",
)
root.delete()
# WHEN listing trash
with CaptureQueriesContext(connection) as ctx:
response = admin_client.get("/api/trash/")
# THEN the response shows the document's own content...
assert response.status_code == status.HTTP_200_OK
[result] = [r for r in response.data["results"] if r["id"] == root.id]
assert result["content"] == "own-content"
# ...without ever querying for versions to resolve it
assert _get_effective_content_fallback_queries(ctx) == []
def test_global_search_db_only_shows_unresolved_content_with_no_extra_query(
self,
admin_client: APIClient,
) -> None:
# GIVEN a root document, findable by title, whose own content
# differs from its latest version's
root = DocumentFactory(title="findme", content="own-content")
DocumentFactory(
root_document=root,
version_index=1,
content="version-content",
)
# WHEN using the global search endpoint's db_only mode
with CaptureQueriesContext(connection) as ctx:
response = admin_client.get(
"/api/search/?query=findme&db_only=true",
)
# THEN the response shows the document's own content...
assert response.status_code == status.HTTP_200_OK
[result] = [d for d in response.data["documents"] if d["id"] == root.id]
assert result["content"] == "own-content"
# ...without ever querying for versions to resolve it
assert _get_effective_content_fallback_queries(ctx) == []
+65
View File
@@ -7,9 +7,12 @@ from typing import Any
from django.db.models import F
from django.db.models import OuterRef
from django.db.models import Prefetch
from django.db.models import QuerySet
from django.db.models import Subquery
from django.db.models import Window
from django.db.models.functions import Coalesce
from django.db.models.functions import RowNumber
from documents.models import Document
@@ -43,6 +46,68 @@ def annotate_effective_content(documents: QuerySet[Document]) -> QuerySet[Docume
)
LATEST_VERSION_CONTENT_PREFETCH_ATTR = "_latest_version_content_prefetch"
def latest_version_content_prefetch() -> Prefetch:
"""
A Prefetch for Document.versions scoped to just the newest version's
content, for get_effective_content()'s fallback when no SQL annotation
is present.
Deliberately not merged into a metadata-only "versions" prefetch (the one
used for the serialized versions list): that one fetches every historical
version of every document, and pulling full OCR content for versions
nobody will read wastes DB transfer/memory at scale. This one is windowed
down to a single row per root, then bounded by Prefetch's own IN-list to
whatever page/result set it's attached to -- one cheap bulk query total,
not one per document and not one per version.
"""
return Prefetch(
"versions",
queryset=(
Document.objects.filter(
root_document_id__isnull=False,
deleted_at__isnull=True,
)
.annotate(
rn=Window(
RowNumber(),
partition_by=F("root_document_id"),
order_by=[
F("version_index").desc(nulls_last=True),
F("id").desc(),
],
),
)
.filter(rn=1)
.only("id", "root_document_id", "content")
),
to_attr=LATEST_VERSION_CONTENT_PREFETCH_ATTR,
)
def has_prefetched_effective_content(document: Document) -> bool:
"""
True if document.get_effective_content() can answer without an extra
per-instance query -- an SQL ``effective_content`` annotation, the lean
latest_version_content_prefetch(), or the metadata-only "versions"
prefetch is already present on the instance.
Callers that haven't set any of those up (e.g. views that build their
own querysets independently of DocumentViewSet.get_queryset(), like
TrashView or GlobalSearchView) intentionally don't pay for version-aware
content resolution -- see DocumentSerializer.to_representation(), which
uses this to decide whether to call get_effective_content() at all.
"""
if hasattr(document, "effective_content"):
return True
if getattr(document, LATEST_VERSION_CONTENT_PREFETCH_ATTR, None) is not None:
return True
prefetched_cache = getattr(document, "_prefetched_objects_cache", None)
return isinstance(prefetched_cache, dict) and "versions" in prefetched_cache
def sort_versions_newest_first(documents: list[Document]) -> list[Document]:
"""
Same sorting as versions_newest_first()
+47 -9
View File
@@ -233,6 +233,7 @@ from documents.versioning import VersionResolutionError
from documents.versioning import get_latest_version_for_root
from documents.versioning import get_request_version_param
from documents.versioning import get_root_document
from documents.versioning import latest_version_content_prefetch
from documents.versioning import resolve_requested_version_for_root
from documents.versioning import versions_newest_first
from paperless import version
@@ -1085,12 +1086,40 @@ class DocumentViewSet(
],
}
def get_queryset(self):
latest_version_content = Subquery(
versions_newest_first(
Document.objects.filter(root_document=OuterRef("pk")),
).values("content")[:1],
# Query params whose filtering needs effective_content evaluated in SQL
# against every candidate row -- see _needs_effective_content_annotation().
_CONTENT_FILTER_PARAMS = (
"search", # DRF SearchFilter's search_fields includes effective_content
"title_content",
"content__istartswith",
"content__iendswith",
"content__icontains",
"content__iexact",
)
def _needs_effective_content_annotation(self) -> bool:
# effective_content is a per-row correlated subquery resolving each
# document's latest version. Cheap when evaluated only for the page
# that survives filtering/sorting/pagination (the common case, via
# the "versions" prefetch + Document.get_effective_content()'s
# fallback), but if anything filters *on* it, the database has to
# evaluate it for every candidate row before the LIMIT is reached --
# pathological on MariaDB specifically for the root_document_id
# self-join once real candidate counts get large. Everything on this
# list is deprecated in favor of the Tantivy-backed search endpoint
# (see filters.py's TitleContentFilter/EffectiveContentFilter docs),
# so keep paying that cost only when one is actually used. Checked as
# a stripped, non-blank value (not just key presence) to match how
# DRF's SearchFilter and TitleContentFilter/EffectiveContentFilter
# themselves no-op on a blank value -- otherwise an empty `?search=`
# or a saved view with a cleared text filter would still pay for the
# annotation despite applying no actual predicate.
params = self.request.query_params
return any(
params.get(param, "").strip() for param in self._CONTENT_FILTER_PARAMS
)
def get_queryset(self):
# A correlated subquery avoids the LEFT JOIN + Count() this used to
# be, which forced a GROUP BY aggregate over every matching document
# before the query could even be sorted or limited.
@@ -1110,10 +1139,9 @@ class DocumentViewSet(
# ObjectFilter.filter(). A blanket .distinct() here forces the
# database to fully sort and dedupe every visible document before
# it can apply LIMIT, which is disastrous at scale.
return (
queryset = (
Document.objects.filter(root_document__isnull=True)
.order_by("-created", "-id")
.annotate(effective_content=Coalesce(latest_version_content, F("content")))
.annotate(num_notes=Coalesce(note_count, 0))
.select_related("correspondent", "storage_path", "document_type", "owner")
.prefetch_related(
@@ -1128,15 +1156,25 @@ class DocumentViewSet(
"version_index",
),
),
latest_version_content_prefetch(),
"tags",
Prefetch(
"custom_fields",
queryset=CustomFieldInstance.objects.select_related("field"),
),
# NotesSerializer nests the author, this avoids query per note
Prefetch("notes", queryset=Note.objects.select_related("user")),
"notes",
)
)
if self._needs_effective_content_annotation():
latest_version_content = Subquery(
versions_newest_first(
Document.objects.filter(root_document=OuterRef("pk")),
).values("content")[:1],
)
queryset = queryset.annotate(
effective_content=Coalesce(latest_version_content, F("content")),
)
return queryset
def get_serializer(self, *args, **kwargs):
fields_param = self.request.query_params.get("fields", None)
+11 -11
View File
@@ -2,7 +2,7 @@ msgid ""
msgstr ""
"Project-Id-Version: paperless-ngx\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2026-08-25 21:36+0000\n"
"POT-Creation-Date: 2026-08-24 21:43+0000\n"
"PO-Revision-Date: 2022-02-17 04:17\n"
"Last-Translator: \n"
"Language-Team: English\n"
@@ -1580,7 +1580,7 @@ msgid "workflow runs"
msgstr ""
#: documents/serialisers.py:523 documents/serialisers.py:875
#: documents/serialisers.py:2827 documents/views.py:311 documents/views.py:2613
#: documents/serialisers.py:2827 documents/views.py:311 documents/views.py:2612
#: paperless_mail/serialisers.py:155
msgid "Insufficient permissions."
msgstr ""
@@ -1621,7 +1621,7 @@ msgstr ""
msgid "Duplicate document identifiers are not allowed."
msgstr ""
#: documents/serialisers.py:2913 documents/views.py:4607
#: documents/serialisers.py:2913 documents/views.py:4606
#, python-format
msgid "Documents not found: %(ids)s"
msgstr ""
@@ -1889,36 +1889,36 @@ msgstr ""
msgid "Unable to parse URI {value}"
msgstr ""
#: documents/views.py:304 documents/views.py:2610
#: documents/views.py:304 documents/views.py:2609
msgid "Invalid more_like_id"
msgstr ""
#: documents/views.py:1587
#: documents/views.py:1586
msgid "Invalid AI configuration."
msgstr ""
#: documents/views.py:1598
#: documents/views.py:1597
msgid "AI backend request timed out."
msgstr ""
#: documents/views.py:2435 documents/views.py:2756
#: documents/views.py:2434 documents/views.py:2755
msgid "Specify only one of text, title_search, query, or more_like_id."
msgstr ""
#: documents/views.py:4620
#: documents/views.py:4619
#, python-format
msgid "Insufficient permissions to share document %(id)s."
msgstr ""
#: documents/views.py:4666
#: documents/views.py:4665
msgid "Bundle is already being processed."
msgstr ""
#: documents/views.py:4727
#: documents/views.py:4726
msgid "The share link bundle is still being prepared. Please try again later."
msgstr ""
#: documents/views.py:4737
#: documents/views.py:4736
msgid "The share link bundle is unavailable."
msgstr ""
+52 -81
View File
@@ -5,12 +5,13 @@ from django.conf import settings
from django.contrib.auth.models import User
from documents.models import Document
from documents.permissions import permitted_object_ids
from documents.permissions import get_objects_for_user_owner_aware
from paperless.config import AIConfig
from paperless_ai.base_model import ClassificationSuggestions
from paperless_ai.base_model import TaxonomyChoiceDict
from paperless_ai.client import AIClient
from paperless_ai.db import db_connection_released
from paperless_ai.indexing import _node_document_ids
from paperless_ai.indexing import retrieve_similar_nodes
from paperless_ai.indexing import truncate_content
from paperless_ai.prompts.context import ClassificationPromptContext
@@ -18,9 +19,7 @@ from paperless_ai.prompts.context import LocalizationPromptContext
from paperless_ai.prompts.context import RagContextPromptContext
from paperless_ai.prompts.render import render_prompt
from paperless_ai.taxonomy import AssignedMetadata
from paperless_ai.taxonomy import SimilarDocument
from paperless_ai.taxonomy import TaxonomyCandidates
from paperless_ai.taxonomy import _node_document_weights
from paperless_ai.taxonomy import build_taxonomy_candidates
from paperless_ai.taxonomy import empty_taxonomy_candidates
from paperless_ai.taxonomy import format_taxonomy_for_prompt
@@ -40,35 +39,6 @@ logger = logging.getLogger("paperless_ai.rag_classifier")
TAXONOMY_CANDIDATE_TOP_K = 15
def _fulltext_similar_documents(
document: Document,
user: User | None,
top_k: int,
) -> list[SimilarDocument]:
"""Rank-based fallback when no embedding backend is configured. Uses
Tantivy's "More Like This" (term-overlap similarity) instead of vector
similarity - cruder, but far better than no candidates at all.
more_like_this_ids returns only a ranked ID list, no scores, so weight is
synthesized from rank (descending from top_k) rather than claiming a
similarity magnitude that doesn't exist. A superuser is normalized to
``None`` before calling, since the backend's permission filter has no
superuser short-circuit of its own.
"""
from documents.search import get_backend
search_user = None if user is not None and user.is_superuser else user
backend = get_backend()
similar_ids = backend.more_like_this_ids(
document.pk,
user=search_user,
limit=top_k,
)
return [
SimilarDocument(document_id=doc_id, weight=float(top_k - rank))
for rank, doc_id in enumerate(similar_ids)
]
def get_language_name(language_code: str) -> str:
normalized_language_code = language_code.lower()
for code, name in settings.LANGUAGES:
@@ -161,52 +131,44 @@ def get_taxonomy_context(
user: User | None = None,
max_docs: int = 5,
) -> tuple[TaxonomyCandidates, AssignedMetadata, str]:
"""One retrieval feeds both taxonomy candidates and RAG text context. Uses
vector similarity when an embedding backend is configured, otherwise
falls back to Tantivy full-text "More Like This" similarity - see
_fulltext_similar_documents. On any retrieval failure, degrades to empty
candidates/context rather than propagating the exception - neither a
vector-store outage nor a search-index issue should block classification,
only its context-assisted enrichment.
"""One retrieval feeds both taxonomy candidates and RAG text context.
On any retrieval failure, degrades to empty candidates/context rather than
propagating the exception - a vector-store outage should not block
classification, only its RAG-assisted enrichment.
"""
assigned = get_assigned_metadata(document, user)
ai_config = AIConfig()
try:
if ai_config.llm_embedding_backend:
# None means "no restriction" to retrieve_similar_nodes. A superuser
# (like no user at all) can see every document, so skip materializing
# every visible pk into a Python list and passing it through as an IN
# filter: for a large library that is a wasted quadratic scan in the
# vector store at best, and past ~32,763 documents a hard
# sqlite3.OperationalError (SQLite's bound-parameter limit) at worst.
# permitted_object_ids() has its own superuser shortcut that would
# return every Document's id anyway, so this changes nothing about
# which documents are considered -- only how we get there.
visible_document_ids = (
None
if user is None or user.is_superuser
else list(permitted_object_ids(user, Document, "view_document"))
)
nodes = retrieve_similar_nodes(
document,
top_k=TAXONOMY_CANDIDATE_TOP_K,
document_ids=visible_document_ids,
)
similar_documents = _node_document_weights(nodes)
else:
# See _fulltext_similar_documents: it applies its own permission
# filter via `user`, so no visible-document-id list is needed here.
similar_documents = _fulltext_similar_documents(
document,
user,
top_k=TAXONOMY_CANDIDATE_TOP_K,
# None means "no restriction" to retrieve_similar_nodes. A superuser
# (like no user at all) can see every document, so skip materializing
# every visible pk into a Python list and passing it through as an IN
# filter: for a large library that is a wasted quadratic scan in the
# vector store at best, and past ~32,763 documents a hard
# sqlite3.OperationalError (SQLite's bound-parameter limit) at worst.
# get_objects_for_user_owner_aware() would return every Document for a
# superuser anyway (guardian's own with_superuser shortcut), so this
# changes nothing about which documents are considered -- only how we
# get there.
visible_document_ids = (
None
if user is None or user.is_superuser
else list(
get_objects_for_user_owner_aware(
user,
"view_document",
Document,
).values_list("pk", flat=True),
)
)
nodes = retrieve_similar_nodes(
document,
top_k=TAXONOMY_CANDIDATE_TOP_K,
document_ids=visible_document_ids,
)
candidates = build_taxonomy_candidates(similar_documents, user)
candidates = build_taxonomy_candidates(nodes, user)
similar_doc_ids = [s["document_id"] for s in similar_documents]
similar_docs = list(
Document.objects.filter(pk__in=similar_doc_ids)[:max_docs],
Document.objects.filter(pk__in=_node_document_ids(nodes))[:max_docs],
)
context_blocks = []
for similar in similar_docs:
@@ -215,8 +177,8 @@ def get_taxonomy_context(
context_blocks.append(f"TITLE: {title}\n{text}")
except Exception:
logger.exception(
"Failed to retrieve similar-document context for document %s; "
"continuing without taxonomy candidates or similar-document context.",
"Failed to retrieve RAG neighbours for document %s; continuing "
"without taxonomy candidates or similar-document context.",
document.pk,
)
return empty_taxonomy_candidates(), assigned, ""
@@ -299,14 +261,23 @@ def get_ai_document_classification(
) -> ClassificationSuggestions:
ai_config = AIConfig()
candidates, assigned, context = get_taxonomy_context(document, user)
prompt = build_prompt_with_rag(
document,
ai_config,
candidates=candidates,
assigned=assigned,
context=context,
)
if ai_config.llm_embedding_backend:
candidates, assigned, context = get_taxonomy_context(document, user)
prompt = build_prompt_with_rag(
document,
ai_config,
candidates=candidates,
assigned=assigned,
context=context,
)
else:
candidates = empty_taxonomy_candidates()
prompt = build_prompt_without_rag(
document,
ai_config,
candidates=candidates,
assigned=get_assigned_metadata(document, user),
)
client = AIClient()
# Hand the pooled DB connection back while the (slow) LLM query runs so it
+14 -27
View File
@@ -33,11 +33,6 @@ class TaxonomyCandidate(TypedDict):
weight: float
class SimilarDocument(TypedDict):
document_id: int
weight: float
class TaxonomyCandidates(TypedDict):
tags: list[TaxonomyCandidate]
document_types: list[TaxonomyCandidate]
@@ -110,10 +105,10 @@ def get_assigned_metadata(document: Document, user: User | None) -> AssignedMeta
)
def _node_document_weights(nodes: list["NodeWithScore"]) -> list[SimilarDocument]:
"""Sum each node's similarity score into its document_id (a document can
appear via multiple chunks/nodes) and return one SimilarDocument per
distinct document_id."""
def _node_document_weights(nodes: list["NodeWithScore"]) -> dict[int, float]:
"""document_id -> that node's similarity score, summed if a document_id
appears more than once across the retrieved nodes (e.g. multiple chunks
of the same source document)."""
weights: dict[int, float] = defaultdict(float)
for node in nodes:
document_id = node.metadata.get("document_id")
@@ -126,10 +121,7 @@ def _node_document_weights(nodes: list["NodeWithScore"]) -> list[SimilarDocument
weights[int(document_id)] += float(node.score or 0.0)
except (TypeError, ValueError): # pragma: no cover
continue
return [
SimilarDocument(document_id=document_id, weight=weight)
for document_id, weight in weights.items()
]
return weights
def _visible_ranked_candidates(
@@ -165,25 +157,20 @@ def _visible_ranked_candidates(
def build_taxonomy_candidates(
similar_documents: list[SimilarDocument],
nodes: list["NodeWithScore"],
user: User | None,
) -> TaxonomyCandidates:
"""Resolve each similar document's id to a live Document, read its
*current* tags/type/correspondent/storage_path via the ORM (never any
possibly-stale names an adapter's source might have cached), weight each
distinct taxonomy object by aggregate similarity weight, permission-filter
"""Resolve each neighbour node's document_id to a live Document, read its
*current* tags/type/correspondent/storage_path via the ORM (never the
possibly-stale names cached in vector-index node metadata), weight each
distinct taxonomy object by aggregate neighbour similarity, permission-filter
against what ``user`` can see, and return each category ranked by weight
and capped. ``similar_documents`` may come from either the vector-RAG
adapter or the full-text fallback adapter - both produce this same shape.
and capped.
"""
if not similar_documents:
return empty_taxonomy_candidates()
# Both adapters guarantee at most one SimilarDocument per document_id, so
# this never silently drops a duplicate's weight.
document_weights: dict[int, float] = {
s["document_id"]: s["weight"] for s in similar_documents
}
document_weights = _node_document_weights(nodes)
if not document_weights:
return empty_taxonomy_candidates()
# Only .tags.all() needs prefetching (a reverse M2M, one extra query for
# the whole batch). document_type/correspondent/storage_path are read
+22 -256
View File
@@ -1,4 +1,3 @@
from collections.abc import Generator
from types import SimpleNamespace
from unittest.mock import MagicMock
from unittest.mock import patch
@@ -8,13 +7,10 @@ import pytest_mock
from django.test import override_settings
from documents.models import Document
from documents.search import TantivyBackend
from documents.tests.factories import DocumentFactory
from documents.tests.factories import TagFactory
from documents.tests.factories import UserFactory
from paperless.config import AIConfig
from paperless_ai.ai_classifier import TAXONOMY_CANDIDATE_TOP_K
from paperless_ai.ai_classifier import _fulltext_similar_documents
from paperless_ai.ai_classifier import _restrict_to_shown_candidates
from paperless_ai.ai_classifier import build_localization_prompt
from paperless_ai.ai_classifier import build_prompt_with_rag
@@ -24,7 +20,6 @@ from paperless_ai.ai_classifier import get_language_name
from paperless_ai.ai_classifier import get_taxonomy_context
from paperless_ai.base_model import ClassificationSuggestions
from paperless_ai.base_model import TaxonomyChoiceDict
from paperless_ai.taxonomy import SimilarDocument
from paperless_ai.taxonomy import TaxonomyCandidate
from paperless_ai.taxonomy import TaxonomyCandidates
from paperless_ai.taxonomy import empty_taxonomy_candidates
@@ -209,10 +204,12 @@ def test_use_rag_if_configured(
@pytest.mark.django_db
@patch("paperless_ai.client.AIClient.run_llm_query")
@patch("paperless_ai.ai_classifier.build_prompt_with_rag")
@patch("paperless_ai.ai_classifier.build_prompt_without_rag")
@patch("paperless_ai.ai_classifier.AIConfig")
@override_settings(LLM_BACKEND="ollama", LLM_MODEL="some_model")
def test_use_rag_prompt_even_without_embedding_backend(
mock_build_prompt_with_rag,
def test_use_without_rag_if_not_configured(
mock_ai_config,
mock_build_prompt_without_rag,
mock_run_llm_query,
mock_document,
):
@@ -222,13 +219,13 @@ def test_use_rag_prompt_even_without_embedding_backend(
WHEN:
- get_ai_document_classification() is called
THEN:
- The RAG-context prompt builder is still used (fed by the full-text
fallback's context/candidates instead of the vector store's)
- The non-RAG prompt builder is used
"""
mock_build_prompt_with_rag.return_value = "Prompt with RAG"
mock_ai_config.return_value.llm_embedding_backend = None
mock_build_prompt_without_rag.return_value = "Prompt without RAG"
mock_run_llm_query.return_value = NESTED_SUGGESTIONS
get_ai_document_classification(mock_document)
mock_build_prompt_with_rag.assert_called_once()
mock_build_prompt_without_rag.assert_called_once()
@pytest.mark.django_db
@@ -306,7 +303,6 @@ def test_build_localization_prompt_preserves_unicode_characters():
@pytest.mark.django_db
@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
def test_get_taxonomy_context_assembles_rag_text_and_candidates():
"""
GIVEN:
@@ -348,7 +344,6 @@ def test_get_taxonomy_context_assembles_rag_text_and_candidates():
@pytest.mark.django_db
@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
def test_get_taxonomy_context_no_similar_docs():
"""
GIVEN:
@@ -372,67 +367,6 @@ def test_get_taxonomy_context_no_similar_docs():
}
@pytest.mark.django_db
def test_get_taxonomy_context_uses_fulltext_fallback_when_no_embedding_backend(
mocker: pytest_mock.MockerFixture,
) -> None:
"""
GIVEN:
- No LLM embedding backend is configured (the default test settings)
WHEN:
- get_taxonomy_context() is called
THEN:
- _fulltext_similar_documents() is called with the document, the user
and TAXONOMY_CANDIDATE_TOP_K
- retrieve_similar_nodes() (the vector path) is never called
"""
document = DocumentFactory.create(content="Some content")
mock_fulltext = mocker.patch(
"paperless_ai.ai_classifier._fulltext_similar_documents",
return_value=[],
)
mock_retrieve = mocker.patch("paperless_ai.ai_classifier.retrieve_similar_nodes")
get_taxonomy_context(document, user=None)
mock_fulltext.assert_called_once_with(
document,
None,
top_k=TAXONOMY_CANDIDATE_TOP_K,
)
mock_retrieve.assert_not_called()
@pytest.mark.django_db
@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
def test_get_taxonomy_context_uses_vector_path_when_embedding_backend_configured(
mocker: pytest_mock.MockerFixture,
) -> None:
"""
GIVEN:
- An LLM embedding backend is configured
WHEN:
- get_taxonomy_context() is called
THEN:
- retrieve_similar_nodes() (the vector path) is called
- _fulltext_similar_documents() (the no-embedding-backend fallback)
is never called
"""
document = DocumentFactory.create(content="Some content")
mock_retrieve = mocker.patch(
"paperless_ai.ai_classifier.retrieve_similar_nodes",
return_value=[],
)
mock_fulltext = mocker.patch(
"paperless_ai.ai_classifier._fulltext_similar_documents",
)
get_taxonomy_context(document, user=None)
mock_retrieve.assert_called_once()
mock_fulltext.assert_not_called()
class TestGetTaxonomyContextVisibility:
"""get_taxonomy_context must not materialize every visible document id
for a user who can already see the whole library: a superuser (like no
@@ -445,7 +379,6 @@ class TestGetTaxonomyContextVisibility:
"""
@pytest.mark.django_db
@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
def test_skips_permission_lookup_for_superuser(
self,
mocker: pytest_mock.MockerFixture,
@@ -464,18 +397,17 @@ class TestGetTaxonomyContextVisibility:
"paperless_ai.ai_classifier.retrieve_similar_nodes",
return_value=[],
)
mock_permitted = mocker.patch(
"paperless_ai.ai_classifier.permitted_object_ids",
mock_get_objects = mocker.patch(
"paperless_ai.ai_classifier.get_objects_for_user_owner_aware",
)
user = UserFactory.create(is_superuser=True)
get_taxonomy_context(document, user)
mock_permitted.assert_not_called()
mock_get_objects.assert_not_called()
assert mock_retrieve.call_args.kwargs["document_ids"] is None
@pytest.mark.django_db
@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
def test_skips_permission_lookup_when_no_user(
self,
mocker: pytest_mock.MockerFixture,
@@ -494,17 +426,16 @@ class TestGetTaxonomyContextVisibility:
"paperless_ai.ai_classifier.retrieve_similar_nodes",
return_value=[],
)
mock_permitted = mocker.patch(
"paperless_ai.ai_classifier.permitted_object_ids",
mock_get_objects = mocker.patch(
"paperless_ai.ai_classifier.get_objects_for_user_owner_aware",
)
get_taxonomy_context(document, None)
mock_permitted.assert_not_called()
mock_get_objects.assert_not_called()
assert mock_retrieve.call_args.kwargs["document_ids"] is None
@pytest.mark.django_db
@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
def test_restricts_to_visible_documents_for_non_superuser(
self,
mocker: pytest_mock.MockerFixture,
@@ -515,7 +446,7 @@ class TestGetTaxonomyContextVisibility:
WHEN:
- get_taxonomy_context() is called
THEN:
- The user's permitted document ids are looked up and passed to
- The user's visible document ids are looked up and passed to
retrieve_similar_nodes() as a restriction
"""
document = DocumentFactory.create(content="Some content")
@@ -523,185 +454,21 @@ class TestGetTaxonomyContextVisibility:
"paperless_ai.ai_classifier.retrieve_similar_nodes",
return_value=[],
)
mock_permitted = mocker.patch(
"paperless_ai.ai_classifier.permitted_object_ids",
return_value=[1, 2, 3],
mock_queryset = mocker.MagicMock()
mock_queryset.values_list.return_value = [1, 2, 3]
mock_get_objects = mocker.patch(
"paperless_ai.ai_classifier.get_objects_for_user_owner_aware",
return_value=mock_queryset,
)
user = UserFactory.create(is_superuser=False)
get_taxonomy_context(document, user)
mock_permitted.assert_called_once_with(user, Document, "view_document")
mock_get_objects.assert_called_once_with(user, "view_document", Document)
assert mock_retrieve.call_args.kwargs["document_ids"] == [1, 2, 3]
@pytest.mark.django_db
class TestFulltextSimilarDocuments:
"""_fulltext_similar_documents is the no-embedding-backend fallback: it
asks the Tantivy full-text index for "More Like This" neighbours instead
of the vector store, and synthesizes a rank-based weight since Tantivy's
more_like_this_ids returns only an ordered id list, no scores.
"""
@pytest.fixture
def fulltext_backend(
self,
mocker: pytest_mock.MockerFixture,
) -> Generator[TantivyBackend, None, None]:
"""An in-memory Tantivy backend, wired up as the module-level
singleton _fulltext_similar_documents resolves via get_backend()."""
backend = TantivyBackend(path=None)
backend.open()
mocker.patch("documents.search.get_backend", return_value=backend)
try:
yield backend
finally:
backend.close()
def test_ranks_by_rank_based_weight_descending(
self,
fulltext_backend: TantivyBackend,
) -> None:
"""
GIVEN:
- A source document and two similar documents indexed in Tantivy
WHEN:
- _fulltext_similar_documents() is called
THEN:
- Each result's weight reflects its rank (first result weighted
higher than the second), not a raw similarity score
"""
source = DocumentFactory.create(content="quarterly financial report details")
first = DocumentFactory.create(content="quarterly financial report details")
second = DocumentFactory.create(content="financial report")
for doc in (source, first, second):
fulltext_backend.add_or_update(doc)
result = _fulltext_similar_documents(source, user=None, top_k=5)
assert len(result) == 2
weight_by_id = {s["document_id"]: s["weight"] for s in result}
assert weight_by_id[first.pk] > weight_by_id[second.pk]
def test_excludes_source_document(
self,
fulltext_backend: TantivyBackend,
) -> None:
"""
GIVEN:
- A source document indexed in Tantivy with no other documents
WHEN:
- _fulltext_similar_documents() is called
THEN:
- An empty list is returned - the source document is never its
own similar document
"""
source = DocumentFactory.create(content="unique unrelated content")
fulltext_backend.add_or_update(source)
result = _fulltext_similar_documents(source, user=None, top_k=5)
assert result == []
def test_empty_index_returns_empty_list(
self,
fulltext_backend: TantivyBackend,
) -> None:
"""
GIVEN:
- A document that has never been indexed (fresh/empty Tantivy index)
WHEN:
- _fulltext_similar_documents() is called
THEN:
- An empty list is returned rather than raising
"""
source = DocumentFactory.create(content="never indexed")
result = _fulltext_similar_documents(source, user=None, top_k=5)
assert result == []
def test_respects_top_k_limit(
self,
fulltext_backend: TantivyBackend,
) -> None:
"""
GIVEN:
- A source document and four similar documents indexed
WHEN:
- _fulltext_similar_documents() is called with top_k=2
THEN:
- At most 2 results are returned
"""
source = DocumentFactory.create(content="shared overlapping keyword text")
for _ in range(4):
fulltext_backend.add_or_update(
DocumentFactory.create(content="shared overlapping keyword text"),
)
result = _fulltext_similar_documents(source, user=None, top_k=2)
assert len(result) <= 2
def test_result_shape_is_similar_document(
self,
fulltext_backend: TantivyBackend,
) -> None:
"""
GIVEN:
- A source document and one similar document indexed
WHEN:
- _fulltext_similar_documents() is called
THEN:
- Each result is a SimilarDocument (document_id + weight only)
"""
source = DocumentFactory.create(content="shared content phrase")
other = DocumentFactory.create(content="shared content phrase")
fulltext_backend.add_or_update(source)
fulltext_backend.add_or_update(other)
result = _fulltext_similar_documents(source, user=None, top_k=5)
# rank 0 (the only/best result) with top_k=5 -> weight = top_k - rank = 5.0,
# per the "first result gets top_k, the last gets 1" formula.
assert result == [SimilarDocument(document_id=other.pk, weight=5.0)]
def test_superuser_sees_other_users_documents(
self,
fulltext_backend: TantivyBackend,
) -> None:
"""
GIVEN:
- A source document owned by one user and a similar document
owned by a different user, with no sharing between them
WHEN:
- _fulltext_similar_documents() is called with a superuser
THEN:
- The other user's document is still returned as a similar
document - a superuser must not be narrowed by the backend's
owner-based permission filter
"""
owner = UserFactory.create()
other_owner = UserFactory.create()
superuser = UserFactory.create(is_superuser=True)
source = DocumentFactory.create(
content="shared content phrase",
owner=owner,
)
other = DocumentFactory.create(
content="shared content phrase",
owner=other_owner,
)
fulltext_backend.add_or_update(source)
fulltext_backend.add_or_update(other)
result = _fulltext_similar_documents(source, user=superuser, top_k=5)
assert [s["document_id"] for s in result] == [other.pk]
@pytest.mark.django_db
@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
@patch("paperless_ai.ai_classifier.retrieve_similar_nodes")
def test_get_taxonomy_context_retrieval_failure_degrades_to_no_hints(mock_retrieve):
"""
@@ -728,7 +495,6 @@ def test_get_taxonomy_context_retrieval_failure_degrades_to_no_hints(mock_retrie
@pytest.mark.django_db
@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
@patch("paperless_ai.ai_classifier.build_taxonomy_candidates")
@patch("paperless_ai.ai_classifier.retrieve_similar_nodes")
def test_get_taxonomy_context_candidate_building_failure_degrades_to_no_hints(
+31 -33
View File
@@ -1,4 +1,5 @@
import json
from types import SimpleNamespace
import pytest
import pytest_mock
@@ -10,7 +11,6 @@ from documents.tests.factories import StoragePathFactory
from documents.tests.factories import TagFactory
from documents.tests.factories import UserFactory
from paperless_ai.taxonomy import AssignedMetadata
from paperless_ai.taxonomy import SimilarDocument
from paperless_ai.taxonomy import TaxonomyCandidates
from paperless_ai.taxonomy import build_taxonomy_candidates
from paperless_ai.taxonomy import format_taxonomy_for_prompt
@@ -132,8 +132,9 @@ class TestGetAssignedMetadata:
assert result["tags"] == ["Owned By Someone Else"]
def make_similar(document_id: int, weight: float) -> SimilarDocument:
return SimilarDocument(document_id=document_id, weight=weight)
def make_node(document_id: int, score: float) -> SimpleNamespace:
"""A stand-in for NodeWithScore: only ``.metadata``/``.score`` are read."""
return SimpleNamespace(metadata={"document_id": str(document_id)}, score=score)
@pytest.mark.django_db
@@ -169,9 +170,9 @@ class TestBuildTaxonomyCandidates:
doc_a.tags.add(tag)
doc_b = DocumentFactory.create()
doc_b.tags.add(tag)
similar_documents = [make_similar(doc_a.pk, 0.9), make_similar(doc_b.pk, 0.4)]
nodes = [make_node(doc_a.pk, 0.9), make_node(doc_b.pk, 0.4)]
result = build_taxonomy_candidates(similar_documents, user=None)
result = build_taxonomy_candidates(nodes, user=None)
assert len(result["tags"]) == 1
assert result["tags"][0]["id"] == tag.pk
@@ -196,9 +197,9 @@ class TestBuildTaxonomyCandidates:
document.tags.add(tag)
tag.name = "New Name"
tag.save()
similar_documents = [make_similar(document.pk, 0.5)]
nodes = [make_node(document.pk, 0.5)]
result = build_taxonomy_candidates(similar_documents, user=None)
result = build_taxonomy_candidates(nodes, user=None)
assert result["tags"][0]["name"] == "New Name"
@@ -218,9 +219,9 @@ class TestBuildTaxonomyCandidates:
document = DocumentFactory.create()
document.tags.add(tag)
tag.delete()
similar_documents = [make_similar(document.pk, 0.5)]
nodes = [make_node(document.pk, 0.5)]
result = build_taxonomy_candidates(similar_documents, user=None)
result = build_taxonomy_candidates(nodes, user=None)
assert result["tags"] == []
@@ -239,12 +240,9 @@ class TestBuildTaxonomyCandidates:
strong_doc.tags.add(strong_tag)
weak_doc = DocumentFactory.create()
weak_doc.tags.add(weak_tag)
similar_documents = [
make_similar(strong_doc.pk, 0.9),
make_similar(weak_doc.pk, 0.1),
]
nodes = [make_node(strong_doc.pk, 0.9), make_node(weak_doc.pk, 0.1)]
result = build_taxonomy_candidates(similar_documents, user=None)
result = build_taxonomy_candidates(nodes, user=None)
assert [c["name"] for c in result["tags"]] == ["Strong", "Weak"]
@@ -260,9 +258,9 @@ class TestBuildTaxonomyCandidates:
document = DocumentFactory.create()
for i in range(15):
document.tags.add(TagFactory.create(name=f"Tag{i}"))
similar_documents = [make_similar(document.pk, 0.5)]
nodes = [make_node(document.pk, 0.5)]
result = build_taxonomy_candidates(similar_documents, user=None)
result = build_taxonomy_candidates(nodes, user=None)
assert len(result["tags"]) == 10
@@ -276,12 +274,12 @@ class TestBuildTaxonomyCandidates:
- Only 5 correspondents are returned
"""
correspondents = CorrespondentFactory.create_batch(7)
similar_documents = [
make_similar(DocumentFactory.create(correspondent=c).pk, 0.5)
nodes = [
make_node(DocumentFactory.create(correspondent=c).pk, 0.5)
for c in correspondents
]
result = build_taxonomy_candidates(similar_documents, user=None)
result = build_taxonomy_candidates(nodes, user=None)
assert len(result["correspondents"]) == 5
@@ -296,9 +294,9 @@ class TestBuildTaxonomyCandidates:
"""
document_type = DocumentTypeFactory.create(name="Invoice")
document = DocumentFactory.create(document_type=document_type)
similar_documents = [make_similar(document.pk, 0.5)]
nodes = [make_node(document.pk, 0.5)]
result = build_taxonomy_candidates(similar_documents, user=None)
result = build_taxonomy_candidates(nodes, user=None)
assert len(result["document_types"]) == 1
assert result["document_types"][0]["id"] == document_type.pk
@@ -314,12 +312,12 @@ class TestBuildTaxonomyCandidates:
- Only 5 document_types are returned
"""
document_types = DocumentTypeFactory.create_batch(7)
similar_documents = [
make_similar(DocumentFactory.create(document_type=dt).pk, 0.5)
nodes = [
make_node(DocumentFactory.create(document_type=dt).pk, 0.5)
for dt in document_types
]
result = build_taxonomy_candidates(similar_documents, user=None)
result = build_taxonomy_candidates(nodes, user=None)
assert len(result["document_types"]) == 5
@@ -334,9 +332,9 @@ class TestBuildTaxonomyCandidates:
"""
storage_path = StoragePathFactory.create(name="Invoices")
document = DocumentFactory.create(storage_path=storage_path)
similar_documents = [make_similar(document.pk, 0.5)]
nodes = [make_node(document.pk, 0.5)]
result = build_taxonomy_candidates(similar_documents, user=None)
result = build_taxonomy_candidates(nodes, user=None)
assert len(result["storage_paths"]) == 1
assert result["storage_paths"][0]["id"] == storage_path.pk
@@ -352,12 +350,12 @@ class TestBuildTaxonomyCandidates:
- Only 5 storage_paths are returned
"""
storage_paths = StoragePathFactory.create_batch(7)
similar_documents = [
make_similar(DocumentFactory.create(storage_path=sp).pk, 0.5)
nodes = [
make_node(DocumentFactory.create(storage_path=sp).pk, 0.5)
for sp in storage_paths
]
result = build_taxonomy_candidates(similar_documents, user=None)
result = build_taxonomy_candidates(nodes, user=None)
assert len(result["storage_paths"]) == 5
@@ -377,14 +375,14 @@ class TestBuildTaxonomyCandidates:
tag = TagFactory.create(name="Restricted")
document = DocumentFactory.create()
document.tags.add(tag)
similar_documents = [make_similar(document.pk, 0.5)]
nodes = [make_node(document.pk, 0.5)]
user = UserFactory.create()
mocker.patch(
"documents.permissions.permitted_object_ids",
return_value=[], # user cannot see this tag
)
result = build_taxonomy_candidates(similar_documents, user=user)
result = build_taxonomy_candidates(nodes, user=user)
assert result["tags"] == []
@@ -414,10 +412,10 @@ class TestBuildTaxonomyCandidates:
tag.save()
document = DocumentFactory.create()
document.tags.add(tag)
similar_documents = [make_similar(document.pk, 0.5)]
nodes = [make_node(document.pk, 0.5)]
spy = mocker.patch("documents.permissions.permitted_object_ids")
result = build_taxonomy_candidates(similar_documents, user=None)
result = build_taxonomy_candidates(nodes, user=None)
assert result["tags"][0]["name"] == "Owned"
spy.assert_not_called()