mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-09-08 18:57:58 +00:00
Compare commits
10
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c20f357697 | ||
|
|
8331923fb2 | ||
|
|
5d87bc5a70 | ||
|
|
2c04fe1382 | ||
|
|
6b25d18fc8 | ||
|
|
714885d7a5 | ||
|
|
73e777a48c | ||
|
|
e9141366bb | ||
|
|
7813375123 | ||
|
|
0132c7bd6e |
@@ -72,7 +72,7 @@ jobs:
|
|||||||
'You are welcome to open a new issue that describes the problem you observed in your own words.'
|
'You are welcome to open a new issue that describes the problem you observed in your own words.'
|
||||||
: 'This issue was automatically closed because it was not opened using our bug report form. ' +
|
: 'This issue was automatically closed because it was not opened using our bug report form. ' +
|
||||||
'Issues have to be created through the form so that the details we need to investigate are included.\n\n' +
|
'Issues have to be created through the form so that the details we need to investigate are included.\n\n' +
|
||||||
`If the problem is still there, please [open a new issue](${newIssue}) using the form. No other action is needed here.\n\n' +
|
`If the problem is still there, please [open a new issue](${newIssue}) using the form. No other action is needed here.\n\n` +
|
||||||
'If any part of your report was written by an AI tool or agent, you must say so: undisclosed AI-generated ' +
|
'If any part of your report was written by an AI tool or agent, you must say so: undisclosed AI-generated ' +
|
||||||
`contributions are a violation of our [Code of Conduct](${codeOfConduct}).`;
|
`contributions are a violation of our [Code of Conduct](${codeOfConduct}).`;
|
||||||
|
|
||||||
|
|||||||
@@ -25,6 +25,10 @@ jobs:
|
|||||||
pr-bot:
|
pr-bot:
|
||||||
name: Automated PR Bot
|
name: Automated PR Bot
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
# Runs after Anti-slop so the welcome comment can see whether the PR was closed
|
||||||
|
# instead of racing it. Still runs if that job fails, so labeling is not lost.
|
||||||
|
needs: Anti-slop
|
||||||
|
if: ${{ !cancelled() }}
|
||||||
permissions:
|
permissions:
|
||||||
contents: read
|
contents: read
|
||||||
pull-requests: write
|
pull-requests: write
|
||||||
@@ -99,8 +103,25 @@ jobs:
|
|||||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||||
with:
|
with:
|
||||||
script: |
|
script: |
|
||||||
const pr = context.payload.pull_request;
|
const user = context.payload.pull_request.user.login;
|
||||||
const user = pr.user.login;
|
|
||||||
|
// Re-read the PR: Anti-slop may have closed and labeled it after the webhook
|
||||||
|
const { data: pr } = await github.rest.pulls.get({
|
||||||
|
owner: context.repo.owner,
|
||||||
|
repo: context.repo.repo,
|
||||||
|
pull_number: context.payload.pull_request.number,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (pr.state === 'closed') {
|
||||||
|
core.info('Skipping comment: PR is already closed');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const labels = pr.labels.map((label) => (typeof label === 'string' ? label : label.name));
|
||||||
|
if (labels.includes('ai')) {
|
||||||
|
core.info('Skipping comment: PR is labeled ai');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
const { data: members } = await github.rest.orgs.listMembers({
|
const { data: members } = await github.rest.orgs.listMembers({
|
||||||
org: 'paperless-ngx',
|
org: 'paperless-ngx',
|
||||||
|
|||||||
+39
-11
@@ -501,15 +501,43 @@
|
|||||||
<context context-type="linenumber">30</context>
|
<context context-type="linenumber">30</context>
|
||||||
</context-group>
|
</context-group>
|
||||||
</trans-unit>
|
</trans-unit>
|
||||||
|
<trans-unit id="7057826840809102816" datatype="html">
|
||||||
|
<source>This value overrides <x id="INTERPOLATION" equiv-text="{{option.config_key}}"/>, which is set outside Paperless.</source>
|
||||||
|
<context-group purpose="location">
|
||||||
|
<context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
|
||||||
|
<context context-type="linenumber">35</context>
|
||||||
|
</context-group>
|
||||||
|
</trans-unit>
|
||||||
|
<trans-unit id="7221396516204435584" datatype="html">
|
||||||
|
<source><x id="INTERPOLATION" equiv-text="{{option.config_key}}"/> is set outside Paperless. Enter a value here to override it.</source>
|
||||||
|
<context-group purpose="location">
|
||||||
|
<context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
|
||||||
|
<context context-type="linenumber">37</context>
|
||||||
|
</context-group>
|
||||||
|
</trans-unit>
|
||||||
|
<trans-unit id="8318849619178340389" datatype="html">
|
||||||
|
<source>Use the externally configured value</source>
|
||||||
|
<context-group purpose="location">
|
||||||
|
<context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
|
||||||
|
<context context-type="linenumber">42</context>
|
||||||
|
</context-group>
|
||||||
|
</trans-unit>
|
||||||
|
<trans-unit id="6032629623003430385" datatype="html">
|
||||||
|
<source>Reset to external</source>
|
||||||
|
<context-group purpose="location">
|
||||||
|
<context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
|
||||||
|
<context context-type="linenumber">43</context>
|
||||||
|
</context-group>
|
||||||
|
</trans-unit>
|
||||||
<trans-unit id="7808756054397155068" datatype="html">
|
<trans-unit id="7808756054397155068" datatype="html">
|
||||||
<source>Reset</source>
|
<source>Reset</source>
|
||||||
<context-group purpose="location">
|
<context-group purpose="location">
|
||||||
<context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
|
<context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
|
||||||
<context context-type="linenumber">34</context>
|
<context context-type="linenumber">46</context>
|
||||||
</context-group>
|
</context-group>
|
||||||
<context-group purpose="location">
|
<context-group purpose="location">
|
||||||
<context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
|
<context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
|
||||||
<context context-type="linenumber">35</context>
|
<context context-type="linenumber">47</context>
|
||||||
</context-group>
|
</context-group>
|
||||||
<context-group purpose="location">
|
<context-group purpose="location">
|
||||||
<context context-type="sourcefile">src/app/components/admin/settings/settings.component.html</context>
|
<context context-type="sourcefile">src/app/components/admin/settings/settings.component.html</context>
|
||||||
@@ -520,7 +548,7 @@
|
|||||||
<source>Enable</source>
|
<source>Enable</source>
|
||||||
<context-group purpose="location">
|
<context-group purpose="location">
|
||||||
<context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
|
<context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
|
||||||
<context context-type="linenumber">43</context>
|
<context context-type="linenumber">56</context>
|
||||||
</context-group>
|
</context-group>
|
||||||
<context-group purpose="location">
|
<context-group purpose="location">
|
||||||
<context context-type="sourcefile">src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html</context>
|
<context context-type="sourcefile">src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html</context>
|
||||||
@@ -531,7 +559,7 @@
|
|||||||
<source>Cancel</source>
|
<source>Cancel</source>
|
||||||
<context-group purpose="location">
|
<context-group purpose="location">
|
||||||
<context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
|
<context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
|
||||||
<context context-type="linenumber">67,68</context>
|
<context context-type="linenumber">80,81</context>
|
||||||
</context-group>
|
</context-group>
|
||||||
<context-group purpose="location">
|
<context-group purpose="location">
|
||||||
<context context-type="sourcefile">src/app/components/admin/settings/settings.component.html</context>
|
<context context-type="sourcefile">src/app/components/admin/settings/settings.component.html</context>
|
||||||
@@ -610,7 +638,7 @@
|
|||||||
<source>Save</source>
|
<source>Save</source>
|
||||||
<context-group purpose="location">
|
<context-group purpose="location">
|
||||||
<context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
|
<context context-type="sourcefile">src/app/components/admin/config/config.component.html</context>
|
||||||
<context context-type="linenumber">70,71</context>
|
<context context-type="linenumber">83,84</context>
|
||||||
</context-group>
|
</context-group>
|
||||||
<context-group purpose="location">
|
<context-group purpose="location">
|
||||||
<context context-type="sourcefile">src/app/components/admin/settings/settings.component.html</context>
|
<context context-type="sourcefile">src/app/components/admin/settings/settings.component.html</context>
|
||||||
@@ -681,42 +709,42 @@
|
|||||||
<source>Error retrieving config</source>
|
<source>Error retrieving config</source>
|
||||||
<context-group purpose="location">
|
<context-group purpose="location">
|
||||||
<context context-type="sourcefile">src/app/components/admin/config/config.component.ts</context>
|
<context context-type="sourcefile">src/app/components/admin/config/config.component.ts</context>
|
||||||
<context context-type="linenumber">117</context>
|
<context context-type="linenumber">118</context>
|
||||||
</context-group>
|
</context-group>
|
||||||
</trans-unit>
|
</trans-unit>
|
||||||
<trans-unit id="1172622527269118932" datatype="html">
|
<trans-unit id="1172622527269118932" datatype="html">
|
||||||
<source>Invalid JSON</source>
|
<source>Invalid JSON</source>
|
||||||
<context-group purpose="location">
|
<context-group purpose="location">
|
||||||
<context context-type="sourcefile">src/app/components/admin/config/config.component.ts</context>
|
<context context-type="sourcefile">src/app/components/admin/config/config.component.ts</context>
|
||||||
<context context-type="linenumber">143</context>
|
<context context-type="linenumber">144</context>
|
||||||
</context-group>
|
</context-group>
|
||||||
</trans-unit>
|
</trans-unit>
|
||||||
<trans-unit id="5103146006962696736" datatype="html">
|
<trans-unit id="5103146006962696736" datatype="html">
|
||||||
<source>Configuration updated</source>
|
<source>Configuration updated</source>
|
||||||
<context-group purpose="location">
|
<context-group purpose="location">
|
||||||
<context context-type="sourcefile">src/app/components/admin/config/config.component.ts</context>
|
<context context-type="sourcefile">src/app/components/admin/config/config.component.ts</context>
|
||||||
<context context-type="linenumber">187</context>
|
<context context-type="linenumber">193</context>
|
||||||
</context-group>
|
</context-group>
|
||||||
</trans-unit>
|
</trans-unit>
|
||||||
<trans-unit id="1664963291286452273" datatype="html">
|
<trans-unit id="1664963291286452273" datatype="html">
|
||||||
<source>An error occurred updating configuration</source>
|
<source>An error occurred updating configuration</source>
|
||||||
<context-group purpose="location">
|
<context-group purpose="location">
|
||||||
<context context-type="sourcefile">src/app/components/admin/config/config.component.ts</context>
|
<context context-type="sourcefile">src/app/components/admin/config/config.component.ts</context>
|
||||||
<context context-type="linenumber">192</context>
|
<context context-type="linenumber">198</context>
|
||||||
</context-group>
|
</context-group>
|
||||||
</trans-unit>
|
</trans-unit>
|
||||||
<trans-unit id="2653081282186526824" datatype="html">
|
<trans-unit id="2653081282186526824" datatype="html">
|
||||||
<source>File successfully updated</source>
|
<source>File successfully updated</source>
|
||||||
<context-group purpose="location">
|
<context-group purpose="location">
|
||||||
<context context-type="sourcefile">src/app/components/admin/config/config.component.ts</context>
|
<context context-type="sourcefile">src/app/components/admin/config/config.component.ts</context>
|
||||||
<context context-type="linenumber">214</context>
|
<context context-type="linenumber">220</context>
|
||||||
</context-group>
|
</context-group>
|
||||||
</trans-unit>
|
</trans-unit>
|
||||||
<trans-unit id="5902783625859504265" datatype="html">
|
<trans-unit id="5902783625859504265" datatype="html">
|
||||||
<source>An error occurred uploading file</source>
|
<source>An error occurred uploading file</source>
|
||||||
<context-group purpose="location">
|
<context-group purpose="location">
|
||||||
<context context-type="sourcefile">src/app/components/admin/config/config.component.ts</context>
|
<context context-type="sourcefile">src/app/components/admin/config/config.component.ts</context>
|
||||||
<context context-type="linenumber">219</context>
|
<context context-type="linenumber">225</context>
|
||||||
</context-group>
|
</context-group>
|
||||||
</trans-unit>
|
</trans-unit>
|
||||||
<trans-unit id="4804785061014590286" datatype="html">
|
<trans-unit id="4804785061014590286" datatype="html">
|
||||||
|
|||||||
@@ -23,18 +23,31 @@
|
|||||||
<div class="col">
|
<div class="col">
|
||||||
<div class="card bg-light">
|
<div class="card bg-light">
|
||||||
<div class="card-body">
|
<div class="card-body">
|
||||||
<div class="card-title d-flex align-items-center">
|
<div class="card-title d-flex align-items-center flex-wrap">
|
||||||
<h6 class="mb-0">
|
<h6 class="mb-0">
|
||||||
{{option.title}}
|
{{option.title}}
|
||||||
</h6>
|
</h6>
|
||||||
<a class="btn btn-sm btn-link" title="Read the documentation about this setting" i18n-title [href]="getDocsUrl(option.config_key)" target="_blank" referrerpolicy="no-referrer">
|
<a class="btn btn-sm btn-link" title="Read the documentation about this setting" i18n-title [href]="getDocsUrl(option.config_key)" target="_blank" referrerpolicy="no-referrer">
|
||||||
<i-bs name="info-circle"></i-bs>
|
<i-bs name="info-circle"></i-bs>
|
||||||
</a>
|
</a>
|
||||||
|
@if (isExternallyConfigured(option.config_key)) {
|
||||||
@if (isSet(option.key)) {
|
@if (isSet(option.key)) {
|
||||||
|
<span class="badge rounded-pill bg-body-secondary text-dark fw-normal" title="This value overrides {{option.config_key}}, which is set outside Paperless." i18n-title>Overrides external</span>
|
||||||
|
} @else {
|
||||||
|
<span class="badge rounded-pill bg-body-secondary text-dark fw-normal" title="{{option.config_key}} is set outside Paperless. Enter a value here to override it." i18n-title>Set externally</span>
|
||||||
|
}
|
||||||
|
}
|
||||||
|
@if (isSet(option.key)) {
|
||||||
|
@if (isExternallyConfigured(option.config_key)) {
|
||||||
|
<button type="button" class="btn btn-sm btn-link text-danger ms-auto pe-0" title="Use the externally configured value" i18n-title (click)="resetOption(option.key)">
|
||||||
|
<i-bs class="me-1" name="x"></i-bs><ng-container i18n>Reset to external</ng-container>
|
||||||
|
</button>
|
||||||
|
} @else {
|
||||||
<button type="button" class="btn btn-sm btn-link text-danger ms-auto pe-0" title="Reset" i18n-title (click)="resetOption(option.key)">
|
<button type="button" class="btn btn-sm btn-link text-danger ms-auto pe-0" title="Reset" i18n-title (click)="resetOption(option.key)">
|
||||||
<i-bs class="me-1" name="x"></i-bs><ng-container i18n>Reset</ng-container>
|
<i-bs class="me-1" name="x"></i-bs><ng-container i18n>Reset</ng-container>
|
||||||
</button>
|
</button>
|
||||||
}
|
}
|
||||||
|
}
|
||||||
</div>
|
</div>
|
||||||
<div class="mb-n3">
|
<div class="mb-n3">
|
||||||
@switch (option.type) {
|
@switch (option.type) {
|
||||||
|
|||||||
@@ -163,6 +163,19 @@ describe('ConfigComponent', () => {
|
|||||||
expect(component.configForm.get('barcodes_enabled').value).toBeNull()
|
expect(component.configForm.get('barcodes_enabled').value).toBeNull()
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it('should identify externally configured options', () => {
|
||||||
|
component.externallyConfiguredVariables = new Set([
|
||||||
|
'PAPERLESS_OCR_LANGUAGE',
|
||||||
|
])
|
||||||
|
|
||||||
|
expect(
|
||||||
|
component.isExternallyConfigured('PAPERLESS_OCR_LANGUAGE')
|
||||||
|
).toBeTruthy()
|
||||||
|
expect(
|
||||||
|
component.isExternallyConfigured('PAPERLESS_OCR_OUTPUT_TYPE')
|
||||||
|
).toBeFalsy()
|
||||||
|
})
|
||||||
|
|
||||||
it('should group options into sections within a category, or not', () => {
|
it('should group options into sections within a category, or not', () => {
|
||||||
const sections = component.getCategorySections(ConfigCategory.OCR)
|
const sections = component.getCategorySections(ConfigCategory.OCR)
|
||||||
expect(sections).toEqual([null, ConfigSection.RemoteOCR])
|
expect(sections).toEqual([null, ConfigSection.RemoteOCR])
|
||||||
|
|||||||
@@ -69,6 +69,7 @@ export class ConfigComponent
|
|||||||
public configForm = new FormGroup({})
|
public configForm = new FormGroup({})
|
||||||
|
|
||||||
public errors = {}
|
public errors = {}
|
||||||
|
public externallyConfiguredVariables = new Set<string>()
|
||||||
|
|
||||||
get optionCategories(): string[] {
|
get optionCategories(): string[] {
|
||||||
return Object.values(ConfigCategory)
|
return Object.values(ConfigCategory)
|
||||||
@@ -152,6 +153,9 @@ export class ConfigComponent
|
|||||||
}
|
}
|
||||||
|
|
||||||
private initialize(config: PaperlessConfig) {
|
private initialize(config: PaperlessConfig) {
|
||||||
|
this.externallyConfiguredVariables = new Set(
|
||||||
|
config.externally_configured_variables ?? []
|
||||||
|
)
|
||||||
if (!this.store) {
|
if (!this.store) {
|
||||||
this.store = new BehaviorSubject(config)
|
this.store = new BehaviorSubject(config)
|
||||||
|
|
||||||
@@ -162,7 +166,9 @@ export class ConfigComponent
|
|||||||
this.configForm.patchValue(state, { emitEvent: false })
|
this.configForm.patchValue(state, { emitEvent: false })
|
||||||
})
|
})
|
||||||
|
|
||||||
this.isDirty$ = dirtyCheck(this.configForm, this.store.asObservable())
|
this.isDirty$ = dirtyCheck(this.configForm, this.store.asObservable(), {
|
||||||
|
excludeKeys: ['externally_configured_variables'],
|
||||||
|
})
|
||||||
}
|
}
|
||||||
this.configForm.patchValue(config)
|
this.configForm.patchValue(config)
|
||||||
|
|
||||||
@@ -227,6 +233,10 @@ export class ConfigComponent
|
|||||||
return this.configForm.get(key).value != null
|
return this.configForm.get(key).value != null
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public isExternallyConfigured(configKey: string): boolean {
|
||||||
|
return this.externallyConfiguredVariables.has(configKey)
|
||||||
|
}
|
||||||
|
|
||||||
public resetOption(key: string) {
|
public resetOption(key: string) {
|
||||||
this.configForm.get(key).setValue(null)
|
this.configForm.get(key).setValue(null)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -422,6 +422,7 @@ export const PaperlessConfigOptions: ConfigOption[] = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
export interface PaperlessConfig extends ObjectWithId {
|
export interface PaperlessConfig extends ObjectWithId {
|
||||||
|
externally_configured_variables: string[]
|
||||||
output_type: OutputTypeConfig
|
output_type: OutputTypeConfig
|
||||||
pages: number
|
pages: number
|
||||||
language: string
|
language: string
|
||||||
|
|||||||
@@ -674,6 +674,9 @@ class TagSerializer(MatchingModelSerializer, OwnedObjectSerializer):
|
|||||||
ordering = ordering or (Lower("name"),)
|
ordering = ordering or (Lower("name"),)
|
||||||
children = children.order_by(*ordering)
|
children = children.order_by(*ordering)
|
||||||
|
|
||||||
|
if not children:
|
||||||
|
return []
|
||||||
|
|
||||||
serializer = TagSerializer(
|
serializer = TagSerializer(
|
||||||
children,
|
children,
|
||||||
many=True,
|
many=True,
|
||||||
|
|||||||
@@ -35,6 +35,7 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
|
|||||||
THEN:
|
THEN:
|
||||||
- Existing config
|
- Existing config
|
||||||
"""
|
"""
|
||||||
|
with patch.dict("os.environ", {}, clear=True):
|
||||||
response = self.client.get(self.ENDPOINT, format="json")
|
response = self.client.get(self.ENDPOINT, format="json")
|
||||||
|
|
||||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||||
@@ -45,6 +46,7 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
|
|||||||
response.data[0],
|
response.data[0],
|
||||||
{
|
{
|
||||||
"id": 1,
|
"id": 1,
|
||||||
|
"externally_configured_variables": [],
|
||||||
"output_type": None,
|
"output_type": None,
|
||||||
"pages": None,
|
"pages": None,
|
||||||
"language": None,
|
"language": None,
|
||||||
@@ -91,6 +93,31 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
|
|||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def test_api_get_config_reports_external_configuration_without_values(self) -> None:
|
||||||
|
with patch.dict(
|
||||||
|
"os.environ",
|
||||||
|
{
|
||||||
|
"PAPERLESS_OCR_LANGUAGE": "eng",
|
||||||
|
"PAPERLESS_REMOTE_OCR_API_KEY": "secret-value",
|
||||||
|
"PAPERLESS_FUTURE_SETTING": "future-value",
|
||||||
|
"UNRELATED_SETTING": "unrelated-value",
|
||||||
|
},
|
||||||
|
clear=True,
|
||||||
|
):
|
||||||
|
response = self.client.get(self.ENDPOINT, format="json")
|
||||||
|
|
||||||
|
self.assertCountEqual(
|
||||||
|
response.data[0]["externally_configured_variables"],
|
||||||
|
[
|
||||||
|
"PAPERLESS_FUTURE_SETTING",
|
||||||
|
"PAPERLESS_OCR_LANGUAGE",
|
||||||
|
"PAPERLESS_REMOTE_OCR_API_KEY",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
self.assertNotContains(response, "secret-value")
|
||||||
|
self.assertNotContains(response, "future-value")
|
||||||
|
self.assertNotContains(response, "UNRELATED_SETTING")
|
||||||
|
|
||||||
def test_api_get_ui_settings_with_config(self) -> None:
|
def test_api_get_ui_settings_with_config(self) -> None:
|
||||||
"""
|
"""
|
||||||
GIVEN:
|
GIVEN:
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ msgid ""
|
|||||||
msgstr ""
|
msgstr ""
|
||||||
"Project-Id-Version: paperless-ngx\n"
|
"Project-Id-Version: paperless-ngx\n"
|
||||||
"Report-Msgid-Bugs-To: \n"
|
"Report-Msgid-Bugs-To: \n"
|
||||||
"POT-Creation-Date: 2026-09-07 20:47+0000\n"
|
"POT-Creation-Date: 2026-09-08 15:31+0000\n"
|
||||||
"PO-Revision-Date: 2022-02-17 04:17\n"
|
"PO-Revision-Date: 2022-02-17 04:17\n"
|
||||||
"Last-Translator: \n"
|
"Last-Translator: \n"
|
||||||
"Language-Team: English\n"
|
"Language-Team: English\n"
|
||||||
@@ -1631,49 +1631,49 @@ msgstr ""
|
|||||||
msgid "workflow runs"
|
msgid "workflow runs"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: documents/serialisers.py:524 documents/serialisers.py:878
|
#: documents/serialisers.py:524 documents/serialisers.py:881
|
||||||
#: documents/serialisers.py:2838 documents/views.py:314 documents/views.py:2624
|
#: documents/serialisers.py:2841 documents/views.py:314 documents/views.py:2624
|
||||||
#: paperless_mail/serialisers.py:156
|
#: paperless_mail/serialisers.py:156
|
||||||
msgid "Insufficient permissions."
|
msgid "Insufficient permissions."
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: documents/serialisers.py:714
|
#: documents/serialisers.py:717
|
||||||
msgid "Invalid color."
|
msgid "Invalid color."
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: documents/serialisers.py:2315
|
#: documents/serialisers.py:2318
|
||||||
#, python-format
|
#, python-format
|
||||||
msgid "File type %(type)s not supported"
|
msgid "File type %(type)s not supported"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: documents/serialisers.py:2359
|
#: documents/serialisers.py:2362
|
||||||
#, python-format
|
#, python-format
|
||||||
msgid "Custom field id must be an integer: %(id)s"
|
msgid "Custom field id must be an integer: %(id)s"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: documents/serialisers.py:2366
|
#: documents/serialisers.py:2369
|
||||||
#, python-format
|
#, python-format
|
||||||
msgid "Custom field with id %(id)s does not exist"
|
msgid "Custom field with id %(id)s does not exist"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: documents/serialisers.py:2383 documents/serialisers.py:2393
|
#: documents/serialisers.py:2386 documents/serialisers.py:2396
|
||||||
msgid ""
|
msgid ""
|
||||||
"Custom fields must be a list of integers or an object mapping ids to values."
|
"Custom fields must be a list of integers or an object mapping ids to values."
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: documents/serialisers.py:2388
|
#: documents/serialisers.py:2391
|
||||||
msgid "Some custom fields don't exist or were specified twice."
|
msgid "Some custom fields don't exist or were specified twice."
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: documents/serialisers.py:2535
|
#: documents/serialisers.py:2538
|
||||||
msgid "Invalid variable detected."
|
msgid "Invalid variable detected."
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: documents/serialisers.py:2894
|
#: documents/serialisers.py:2897
|
||||||
msgid "Duplicate document identifiers are not allowed."
|
msgid "Duplicate document identifiers are not allowed."
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: documents/serialisers.py:2924 documents/views.py:4626
|
#: documents/serialisers.py:2927 documents/views.py:4626
|
||||||
#, python-format
|
#, python-format
|
||||||
msgid "Documents not found: %(ids)s"
|
msgid "Documents not found: %(ids)s"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import logging
|
import logging
|
||||||
|
import os
|
||||||
from io import BytesIO
|
from io import BytesIO
|
||||||
|
|
||||||
import magic
|
import magic
|
||||||
@@ -212,6 +213,7 @@ class ProfileSerializer(PasswordValidationMixin, serializers.ModelSerializer[Use
|
|||||||
class ApplicationConfigurationSerializer(
|
class ApplicationConfigurationSerializer(
|
||||||
serializers.ModelSerializer[ApplicationConfiguration],
|
serializers.ModelSerializer[ApplicationConfiguration],
|
||||||
):
|
):
|
||||||
|
externally_configured_variables = serializers.SerializerMethodField()
|
||||||
user_args = serializers.JSONField(binary=True, allow_null=True)
|
user_args = serializers.JSONField(binary=True, allow_null=True)
|
||||||
barcode_tag_mapping = serializers.JSONField(binary=True, allow_null=True)
|
barcode_tag_mapping = serializers.JSONField(binary=True, allow_null=True)
|
||||||
llm_api_key = ObfuscatedPasswordField(
|
llm_api_key = ObfuscatedPasswordField(
|
||||||
@@ -227,6 +229,12 @@ class ApplicationConfigurationSerializer(
|
|||||||
|
|
||||||
OBFUSCATED_FIELDS = ("llm_api_key", "remote_ocr_api_key")
|
OBFUSCATED_FIELDS = ("llm_api_key", "remote_ocr_api_key")
|
||||||
|
|
||||||
|
def get_externally_configured_variables(
|
||||||
|
self,
|
||||||
|
instance: ApplicationConfiguration,
|
||||||
|
) -> list[str]:
|
||||||
|
return sorted(name for name in os.environ if name.startswith("PAPERLESS_"))
|
||||||
|
|
||||||
def run_validation(self, data):
|
def run_validation(self, data):
|
||||||
# Empty strings treated as None to avoid unexpected behavior
|
# Empty strings treated as None to avoid unexpected behavior
|
||||||
if "user_args" in data and data["user_args"] == "":
|
if "user_args" in data and data["user_args"] == "":
|
||||||
|
|||||||
@@ -4,21 +4,24 @@ from django.conf import settings
|
|||||||
from django.contrib.auth.models import User
|
from django.contrib.auth.models import User
|
||||||
|
|
||||||
from documents.models import Document
|
from documents.models import Document
|
||||||
from documents.permissions import get_objects_for_user_owner_aware
|
from documents.permissions import permitted_object_ids
|
||||||
|
from documents.permissions import restrict_queryset_to_visible
|
||||||
|
from documents.permissions import user_is_unrestricted
|
||||||
from paperless.config import AIConfig
|
from paperless.config import AIConfig
|
||||||
from paperless_ai.base_model import ClassificationSuggestions
|
from paperless_ai.base_model import ClassificationSuggestions
|
||||||
from paperless_ai.base_model import TaxonomyChoiceDict
|
from paperless_ai.base_model import TaxonomyChoiceDict
|
||||||
from paperless_ai.base_model import classification_suggestions_to_model
|
from paperless_ai.base_model import classification_suggestions_to_model
|
||||||
from paperless_ai.client import AIClient
|
from paperless_ai.client import AIClient
|
||||||
from paperless_ai.db import db_connection_released
|
from paperless_ai.db import db_connection_released
|
||||||
from paperless_ai.indexing import _node_document_ids
|
|
||||||
from paperless_ai.indexing import retrieve_similar_nodes
|
from paperless_ai.indexing import retrieve_similar_nodes
|
||||||
from paperless_ai.indexing import truncate_content
|
from paperless_ai.indexing import truncate_content
|
||||||
from paperless_ai.prompts.context import ClassificationPromptContext
|
from paperless_ai.prompts.context import ClassificationPromptContext
|
||||||
from paperless_ai.prompts.context import LocalizationPromptContext
|
from paperless_ai.prompts.context import LocalizationPromptContext
|
||||||
from paperless_ai.prompts.context import RagContextPromptContext
|
from paperless_ai.prompts.context import RagContextPromptContext
|
||||||
from paperless_ai.prompts.render import render_prompt
|
from paperless_ai.prompts.render import render_prompt
|
||||||
|
from paperless_ai.taxonomy import SimilarDocument
|
||||||
from paperless_ai.taxonomy import TaxonomyCandidates
|
from paperless_ai.taxonomy import TaxonomyCandidates
|
||||||
|
from paperless_ai.taxonomy import _node_document_weights
|
||||||
from paperless_ai.taxonomy import build_taxonomy_candidates
|
from paperless_ai.taxonomy import build_taxonomy_candidates
|
||||||
from paperless_ai.taxonomy import empty_taxonomy_candidates
|
from paperless_ai.taxonomy import empty_taxonomy_candidates
|
||||||
from paperless_ai.taxonomy import format_taxonomy_for_prompt
|
from paperless_ai.taxonomy import format_taxonomy_for_prompt
|
||||||
@@ -37,6 +40,48 @@ logger = logging.getLogger("paperless_ai.rag_classifier")
|
|||||||
TAXONOMY_CANDIDATE_TOP_K = 15
|
TAXONOMY_CANDIDATE_TOP_K = 15
|
||||||
|
|
||||||
|
|
||||||
|
def _fulltext_similar_documents(
|
||||||
|
document: Document,
|
||||||
|
user: User | None,
|
||||||
|
top_k: int,
|
||||||
|
) -> list[SimilarDocument]:
|
||||||
|
"""Rank-based fallback when no embedding backend is configured. Uses
|
||||||
|
Tantivy's "More Like This" (term-overlap similarity) instead of vector
|
||||||
|
similarity - cruder, but far better than no candidates at all.
|
||||||
|
more_like_this_ids returns only a ranked ID list, no scores, so weight is
|
||||||
|
synthesized from rank (descending from top_k) rather than claiming a
|
||||||
|
similarity magnitude that doesn't exist. An unrestricted user (none, or an
|
||||||
|
active superuser - see user_is_unrestricted) is normalized to ``None``
|
||||||
|
before calling, since the backend's permission filter has no superuser
|
||||||
|
short-circuit of its own. Results are re-checked with
|
||||||
|
restrict_queryset_to_visible() since Tantivy's indexed permission fields
|
||||||
|
lag the DB via async reindexing.
|
||||||
|
"""
|
||||||
|
from documents.search import get_backend
|
||||||
|
|
||||||
|
unrestricted = user_is_unrestricted(user)
|
||||||
|
search_user = None if unrestricted else user
|
||||||
|
backend = get_backend()
|
||||||
|
similar_ids = backend.more_like_this_ids(
|
||||||
|
document.pk,
|
||||||
|
user=search_user,
|
||||||
|
limit=top_k,
|
||||||
|
)
|
||||||
|
if not unrestricted:
|
||||||
|
allowed_ids = set(
|
||||||
|
restrict_queryset_to_visible(
|
||||||
|
Document.objects.filter(pk__in=similar_ids),
|
||||||
|
user,
|
||||||
|
"view_document",
|
||||||
|
).values_list("pk", flat=True),
|
||||||
|
)
|
||||||
|
similar_ids = [doc_id for doc_id in similar_ids if doc_id in allowed_ids]
|
||||||
|
return [
|
||||||
|
SimilarDocument(document_id=doc_id, weight=float(top_k - rank))
|
||||||
|
for rank, doc_id in enumerate(similar_ids)
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
def get_language_name(language_code: str) -> str:
|
def get_language_name(language_code: str) -> str:
|
||||||
normalized_language_code = language_code.lower()
|
normalized_language_code = language_code.lower()
|
||||||
for code, name in settings.LANGUAGES:
|
for code, name in settings.LANGUAGES:
|
||||||
@@ -136,43 +181,52 @@ def get_taxonomy_context(
|
|||||||
user: User | None = None,
|
user: User | None = None,
|
||||||
max_docs: int = 5,
|
max_docs: int = 5,
|
||||||
) -> tuple[TaxonomyCandidates, str]:
|
) -> tuple[TaxonomyCandidates, str]:
|
||||||
"""One retrieval feeds both taxonomy candidates and RAG text context.
|
"""One retrieval feeds both taxonomy candidates and RAG text context. Uses
|
||||||
On any retrieval failure, degrades to empty candidates/context rather than
|
vector similarity when an embedding backend is configured, otherwise
|
||||||
propagating the exception - a vector-store outage should not block
|
falls back to Tantivy full-text "More Like This" similarity - see
|
||||||
classification, only its RAG-assisted enrichment.
|
_fulltext_similar_documents. On any retrieval failure, degrades to empty
|
||||||
|
candidates/context rather than propagating the exception - neither a
|
||||||
|
vector-store outage nor a search-index issue should block classification,
|
||||||
|
only its context-assisted enrichment.
|
||||||
"""
|
"""
|
||||||
|
ai_config = AIConfig()
|
||||||
try:
|
try:
|
||||||
# None means "no restriction" to retrieve_similar_nodes. A superuser
|
if ai_config.llm_embedding_backend:
|
||||||
# (like no user at all) can see every document, so skip materializing
|
# None means "no restriction" to retrieve_similar_nodes. An
|
||||||
# every visible pk into a Python list and passing it through as an IN
|
# unrestricted user (no user at all, or an active superuser -- see
|
||||||
# filter: for a large library that is a wasted quadratic scan in the
|
# user_is_unrestricted) can see every document, so skip
|
||||||
# vector store at best, and past ~32,763 documents a hard
|
# materializing every visible pk into a Python list and passing it
|
||||||
# sqlite3.OperationalError (SQLite's bound-parameter limit) at worst.
|
# through as an IN filter: for a large library that is a wasted
|
||||||
# get_objects_for_user_owner_aware() would return every Document for a
|
# quadratic scan in the vector store at best, and past ~32,763
|
||||||
# superuser anyway (guardian's own with_superuser shortcut), so this
|
# documents a hard sqlite3.OperationalError (SQLite's
|
||||||
# changes nothing about which documents are considered -- only how we
|
# bound-parameter limit) at worst.
|
||||||
# get there.
|
# permitted_object_ids() has its own superuser shortcut that would
|
||||||
|
# return every Document's id anyway, so this changes nothing about
|
||||||
|
# which documents are considered -- only how we get there.
|
||||||
visible_document_ids = (
|
visible_document_ids = (
|
||||||
None
|
None
|
||||||
if user is None or user.is_superuser
|
if user_is_unrestricted(user)
|
||||||
else list(
|
else list(permitted_object_ids(user, Document, "view_document"))
|
||||||
get_objects_for_user_owner_aware(
|
|
||||||
user,
|
|
||||||
"view_document",
|
|
||||||
Document,
|
|
||||||
).values_list("pk", flat=True),
|
|
||||||
)
|
|
||||||
)
|
)
|
||||||
nodes = retrieve_similar_nodes(
|
nodes = retrieve_similar_nodes(
|
||||||
document,
|
document,
|
||||||
top_k=TAXONOMY_CANDIDATE_TOP_K,
|
top_k=TAXONOMY_CANDIDATE_TOP_K,
|
||||||
document_ids=visible_document_ids,
|
document_ids=visible_document_ids,
|
||||||
)
|
)
|
||||||
|
similar_documents = _node_document_weights(nodes)
|
||||||
|
else:
|
||||||
|
# See _fulltext_similar_documents: it applies its own permission
|
||||||
|
# filter via `user`, so no visible-document-id list is needed here.
|
||||||
|
similar_documents = _fulltext_similar_documents(
|
||||||
|
document,
|
||||||
|
user,
|
||||||
|
top_k=TAXONOMY_CANDIDATE_TOP_K,
|
||||||
|
)
|
||||||
|
|
||||||
candidates = build_taxonomy_candidates(nodes, user)
|
candidates = build_taxonomy_candidates(similar_documents, user)
|
||||||
|
|
||||||
# ``nodes`` are already ordered by descending vector similarity; don't lose it.
|
# similar_documents is already ordered by descending weight; don't lose it.
|
||||||
similar_document_ids = list(dict.fromkeys(_node_document_ids(nodes)))
|
similar_document_ids = [s["document_id"] for s in similar_documents]
|
||||||
similar_documents_by_id = Document.objects.in_bulk(similar_document_ids)
|
similar_documents_by_id = Document.objects.in_bulk(similar_document_ids)
|
||||||
similar_docs = [
|
similar_docs = [
|
||||||
similar_documents_by_id[document_id]
|
similar_documents_by_id[document_id]
|
||||||
@@ -186,8 +240,8 @@ def get_taxonomy_context(
|
|||||||
context_blocks.append(f"TITLE: {title}\n{text}")
|
context_blocks.append(f"TITLE: {title}\n{text}")
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.exception(
|
logger.exception(
|
||||||
"Failed to retrieve RAG neighbours for document %s; continuing "
|
"Failed to retrieve similar-document context for document %s; "
|
||||||
"without taxonomy candidates or similar-document context.",
|
"continuing without taxonomy candidates or similar-document context.",
|
||||||
document.pk,
|
document.pk,
|
||||||
)
|
)
|
||||||
return empty_taxonomy_candidates(), ""
|
return empty_taxonomy_candidates(), ""
|
||||||
@@ -241,7 +295,6 @@ def get_ai_document_classification(
|
|||||||
) -> ClassificationSuggestions:
|
) -> ClassificationSuggestions:
|
||||||
ai_config = AIConfig()
|
ai_config = AIConfig()
|
||||||
|
|
||||||
if ai_config.llm_embedding_backend:
|
|
||||||
candidates, context = get_taxonomy_context(document, user)
|
candidates, context = get_taxonomy_context(document, user)
|
||||||
prompt = build_prompt_with_rag(
|
prompt = build_prompt_with_rag(
|
||||||
document,
|
document,
|
||||||
@@ -249,9 +302,6 @@ def get_ai_document_classification(
|
|||||||
candidates=candidates,
|
candidates=candidates,
|
||||||
context=context,
|
context=context,
|
||||||
)
|
)
|
||||||
else:
|
|
||||||
candidates = empty_taxonomy_candidates()
|
|
||||||
prompt = build_prompt_without_rag(document, ai_config, candidates=candidates)
|
|
||||||
|
|
||||||
client = AIClient()
|
client = AIClient()
|
||||||
# Hand the pooled DB connection back while the (slow) LLM query runs so it
|
# Hand the pooled DB connection back while the (slow) LLM query runs so it
|
||||||
|
|||||||
@@ -721,20 +721,3 @@ def retrieve_similar_nodes(
|
|||||||
continue
|
continue
|
||||||
filtered.append(node)
|
filtered.append(node)
|
||||||
return filtered
|
return filtered
|
||||||
|
|
||||||
|
|
||||||
def _node_document_ids(nodes: list["NodeWithScore"]) -> list[int]:
|
|
||||||
document_ids: list[int] = []
|
|
||||||
for node in nodes:
|
|
||||||
document_id = node.metadata.get("document_id")
|
|
||||||
if document_id is None: # pragma: no cover
|
|
||||||
# See the matching guard in retrieve_similar_nodes() above.
|
|
||||||
continue
|
|
||||||
try:
|
|
||||||
document_ids.append(int(document_id))
|
|
||||||
except ValueError: # pragma: no cover
|
|
||||||
logger.warning(
|
|
||||||
"Skipping LLM index result with invalid document_id %r.",
|
|
||||||
document_id,
|
|
||||||
)
|
|
||||||
return document_ids
|
|
||||||
|
|||||||
@@ -31,6 +31,11 @@ class TaxonomyCandidate(TypedDict):
|
|||||||
weight: float
|
weight: float
|
||||||
|
|
||||||
|
|
||||||
|
class SimilarDocument(TypedDict):
|
||||||
|
document_id: int
|
||||||
|
weight: float
|
||||||
|
|
||||||
|
|
||||||
class TaxonomyCandidates(TypedDict):
|
class TaxonomyCandidates(TypedDict):
|
||||||
tags: list[TaxonomyCandidate]
|
tags: list[TaxonomyCandidate]
|
||||||
document_types: list[TaxonomyCandidate]
|
document_types: list[TaxonomyCandidate]
|
||||||
@@ -49,10 +54,10 @@ def empty_taxonomy_candidates() -> TaxonomyCandidates:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _node_document_weights(nodes: list["NodeWithScore"]) -> dict[int, float]:
|
def _node_document_weights(nodes: list["NodeWithScore"]) -> list[SimilarDocument]:
|
||||||
"""document_id -> that node's similarity score, summed if a document_id
|
"""Sum each node's similarity score into its document_id (a document can
|
||||||
appears more than once across the retrieved nodes (e.g. multiple chunks
|
appear via multiple chunks/nodes) and return one SimilarDocument per
|
||||||
of the same source document)."""
|
distinct document_id."""
|
||||||
weights: dict[int, float] = defaultdict(float)
|
weights: dict[int, float] = defaultdict(float)
|
||||||
for node in nodes:
|
for node in nodes:
|
||||||
document_id = node.metadata.get("document_id")
|
document_id = node.metadata.get("document_id")
|
||||||
@@ -65,7 +70,14 @@ def _node_document_weights(nodes: list["NodeWithScore"]) -> dict[int, float]:
|
|||||||
weights[int(document_id)] += float(node.score or 0.0)
|
weights[int(document_id)] += float(node.score or 0.0)
|
||||||
except (TypeError, ValueError): # pragma: no cover
|
except (TypeError, ValueError): # pragma: no cover
|
||||||
continue
|
continue
|
||||||
return weights
|
return sorted(
|
||||||
|
(
|
||||||
|
SimilarDocument(document_id=document_id, weight=weight)
|
||||||
|
for document_id, weight in weights.items()
|
||||||
|
),
|
||||||
|
key=lambda similar: similar["weight"],
|
||||||
|
reverse=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _visible_ranked_candidates(
|
def _visible_ranked_candidates(
|
||||||
@@ -101,21 +113,26 @@ def _visible_ranked_candidates(
|
|||||||
|
|
||||||
|
|
||||||
def build_taxonomy_candidates(
|
def build_taxonomy_candidates(
|
||||||
nodes: list["NodeWithScore"],
|
similar_documents: list[SimilarDocument],
|
||||||
user: User | None,
|
user: User | None,
|
||||||
) -> TaxonomyCandidates:
|
) -> TaxonomyCandidates:
|
||||||
"""Resolve each neighbour node's document_id to a live Document, read its
|
"""Resolve each similar document's id to a live Document, read its
|
||||||
*current* tags/type/correspondent/storage_path via the ORM (never the
|
*current* tags/type/correspondent/storage_path via the ORM (never any
|
||||||
possibly-stale names cached in vector-index node metadata), weight each
|
possibly-stale names an adapter's source might have cached), weight each
|
||||||
distinct taxonomy object by aggregate neighbour similarity, permission-filter
|
distinct taxonomy object by aggregate similarity weight, permission-filter
|
||||||
against what ``user`` can see, and return each category ranked by weight
|
against what ``user`` can see, and return each category ranked by weight
|
||||||
and capped.
|
and capped. ``similar_documents`` may come from either the vector-RAG
|
||||||
|
adapter or the full-text fallback adapter - both produce this same shape.
|
||||||
"""
|
"""
|
||||||
|
if not similar_documents:
|
||||||
document_weights = _node_document_weights(nodes)
|
|
||||||
if not document_weights:
|
|
||||||
return empty_taxonomy_candidates()
|
return empty_taxonomy_candidates()
|
||||||
|
|
||||||
|
# Both adapters guarantee at most one SimilarDocument per document_id, so
|
||||||
|
# this never silently drops a duplicate's weight.
|
||||||
|
document_weights: dict[int, float] = {
|
||||||
|
s["document_id"]: s["weight"] for s in similar_documents
|
||||||
|
}
|
||||||
|
|
||||||
# Only .tags.all() needs prefetching (a reverse M2M, one extra query for
|
# Only .tags.all() needs prefetching (a reverse M2M, one extra query for
|
||||||
# the whole batch). document_type/correspondent/storage_path are read
|
# the whole batch). document_type/correspondent/storage_path are read
|
||||||
# below via their *_id columns (neighbour.document_type_id, etc.), which
|
# below via their *_id columns (neighbour.document_type_id, etc.), which
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import datetime
|
import datetime
|
||||||
|
from collections.abc import Generator
|
||||||
from types import SimpleNamespace
|
from types import SimpleNamespace
|
||||||
from unittest.mock import MagicMock
|
from unittest.mock import MagicMock
|
||||||
from unittest.mock import patch
|
from unittest.mock import patch
|
||||||
@@ -6,18 +7,24 @@ from unittest.mock import patch
|
|||||||
import pytest
|
import pytest
|
||||||
import pytest_mock
|
import pytest_mock
|
||||||
from django.test import override_settings
|
from django.test import override_settings
|
||||||
|
from guardian.shortcuts import assign_perm
|
||||||
|
from guardian.shortcuts import remove_perm
|
||||||
|
|
||||||
from documents.models import Document
|
from documents.models import Document
|
||||||
|
from documents.search import TantivyBackend
|
||||||
from documents.tests.factories import DocumentFactory
|
from documents.tests.factories import DocumentFactory
|
||||||
from documents.tests.factories import TagFactory
|
from documents.tests.factories import TagFactory
|
||||||
from documents.tests.factories import UserFactory
|
from documents.tests.factories import UserFactory
|
||||||
from paperless.config import AIConfig
|
from paperless.config import AIConfig
|
||||||
|
from paperless_ai.ai_classifier import TAXONOMY_CANDIDATE_TOP_K
|
||||||
|
from paperless_ai.ai_classifier import _fulltext_similar_documents
|
||||||
from paperless_ai.ai_classifier import build_localization_prompt
|
from paperless_ai.ai_classifier import build_localization_prompt
|
||||||
from paperless_ai.ai_classifier import build_prompt_with_rag
|
from paperless_ai.ai_classifier import build_prompt_with_rag
|
||||||
from paperless_ai.ai_classifier import build_prompt_without_rag
|
from paperless_ai.ai_classifier import build_prompt_without_rag
|
||||||
from paperless_ai.ai_classifier import get_ai_document_classification
|
from paperless_ai.ai_classifier import get_ai_document_classification
|
||||||
from paperless_ai.ai_classifier import get_language_name
|
from paperless_ai.ai_classifier import get_language_name
|
||||||
from paperless_ai.ai_classifier import get_taxonomy_context
|
from paperless_ai.ai_classifier import get_taxonomy_context
|
||||||
|
from paperless_ai.taxonomy import SimilarDocument
|
||||||
from paperless_ai.taxonomy import TaxonomyCandidate
|
from paperless_ai.taxonomy import TaxonomyCandidate
|
||||||
from paperless_ai.taxonomy import TaxonomyCandidates
|
from paperless_ai.taxonomy import TaxonomyCandidates
|
||||||
|
|
||||||
@@ -220,12 +227,10 @@ def test_use_rag_if_configured(
|
|||||||
|
|
||||||
@pytest.mark.django_db
|
@pytest.mark.django_db
|
||||||
@patch("paperless_ai.client.AIClient.run_llm_query")
|
@patch("paperless_ai.client.AIClient.run_llm_query")
|
||||||
@patch("paperless_ai.ai_classifier.build_prompt_without_rag")
|
@patch("paperless_ai.ai_classifier.build_prompt_with_rag")
|
||||||
@patch("paperless_ai.ai_classifier.AIConfig")
|
|
||||||
@override_settings(LLM_BACKEND="ollama", LLM_MODEL="some_model")
|
@override_settings(LLM_BACKEND="ollama", LLM_MODEL="some_model")
|
||||||
def test_use_without_rag_if_not_configured(
|
def test_use_rag_prompt_even_without_embedding_backend(
|
||||||
mock_ai_config,
|
mock_build_prompt_with_rag,
|
||||||
mock_build_prompt_without_rag,
|
|
||||||
mock_run_llm_query,
|
mock_run_llm_query,
|
||||||
mock_document,
|
mock_document,
|
||||||
):
|
):
|
||||||
@@ -235,13 +240,13 @@ def test_use_without_rag_if_not_configured(
|
|||||||
WHEN:
|
WHEN:
|
||||||
- get_ai_document_classification() is called
|
- get_ai_document_classification() is called
|
||||||
THEN:
|
THEN:
|
||||||
- The non-RAG prompt builder is used
|
- The RAG-context prompt builder is still used (fed by the full-text
|
||||||
|
fallback's context/candidates instead of the vector store's)
|
||||||
"""
|
"""
|
||||||
mock_ai_config.return_value.llm_embedding_backend = None
|
mock_build_prompt_with_rag.return_value = "Prompt with RAG"
|
||||||
mock_build_prompt_without_rag.return_value = "Prompt without RAG"
|
|
||||||
mock_run_llm_query.return_value = NESTED_SUGGESTIONS
|
mock_run_llm_query.return_value = NESTED_SUGGESTIONS
|
||||||
get_ai_document_classification(mock_document)
|
get_ai_document_classification(mock_document)
|
||||||
mock_build_prompt_without_rag.assert_called_once()
|
mock_build_prompt_with_rag.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.django_db
|
@pytest.mark.django_db
|
||||||
@@ -320,6 +325,7 @@ def test_build_localization_prompt_preserves_unicode_characters():
|
|||||||
|
|
||||||
|
|
||||||
@pytest.mark.django_db
|
@pytest.mark.django_db
|
||||||
|
@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
|
||||||
def test_get_taxonomy_context_assembles_rag_text_and_candidates():
|
def test_get_taxonomy_context_assembles_rag_text_and_candidates():
|
||||||
"""
|
"""
|
||||||
GIVEN:
|
GIVEN:
|
||||||
@@ -354,6 +360,7 @@ def test_get_taxonomy_context_assembles_rag_text_and_candidates():
|
|||||||
|
|
||||||
|
|
||||||
@pytest.mark.django_db
|
@pytest.mark.django_db
|
||||||
|
@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
|
||||||
def test_get_taxonomy_context_preserves_similarity_order_and_distinct_documents():
|
def test_get_taxonomy_context_preserves_similarity_order_and_distinct_documents():
|
||||||
"""
|
"""
|
||||||
GIVEN:
|
GIVEN:
|
||||||
@@ -424,6 +431,7 @@ def test_get_taxonomy_context_preserves_similarity_order_and_distinct_documents(
|
|||||||
|
|
||||||
|
|
||||||
@pytest.mark.django_db
|
@pytest.mark.django_db
|
||||||
|
@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
|
||||||
def test_get_taxonomy_context_no_similar_docs():
|
def test_get_taxonomy_context_no_similar_docs():
|
||||||
"""
|
"""
|
||||||
GIVEN:
|
GIVEN:
|
||||||
@@ -447,6 +455,67 @@ def test_get_taxonomy_context_no_similar_docs():
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
def test_get_taxonomy_context_uses_fulltext_fallback_when_no_embedding_backend(
|
||||||
|
mocker: pytest_mock.MockerFixture,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- No LLM embedding backend is configured (the default test settings)
|
||||||
|
WHEN:
|
||||||
|
- get_taxonomy_context() is called
|
||||||
|
THEN:
|
||||||
|
- _fulltext_similar_documents() is called with the document, the user
|
||||||
|
and TAXONOMY_CANDIDATE_TOP_K
|
||||||
|
- retrieve_similar_nodes() (the vector path) is never called
|
||||||
|
"""
|
||||||
|
document = DocumentFactory.create(content="Some content")
|
||||||
|
mock_fulltext = mocker.patch(
|
||||||
|
"paperless_ai.ai_classifier._fulltext_similar_documents",
|
||||||
|
return_value=[],
|
||||||
|
)
|
||||||
|
mock_retrieve = mocker.patch("paperless_ai.ai_classifier.retrieve_similar_nodes")
|
||||||
|
|
||||||
|
get_taxonomy_context(document, user=None)
|
||||||
|
|
||||||
|
mock_fulltext.assert_called_once_with(
|
||||||
|
document,
|
||||||
|
None,
|
||||||
|
top_k=TAXONOMY_CANDIDATE_TOP_K,
|
||||||
|
)
|
||||||
|
mock_retrieve.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
|
||||||
|
def test_get_taxonomy_context_uses_vector_path_when_embedding_backend_configured(
|
||||||
|
mocker: pytest_mock.MockerFixture,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- An LLM embedding backend is configured
|
||||||
|
WHEN:
|
||||||
|
- get_taxonomy_context() is called
|
||||||
|
THEN:
|
||||||
|
- retrieve_similar_nodes() (the vector path) is called
|
||||||
|
- _fulltext_similar_documents() (the no-embedding-backend fallback)
|
||||||
|
is never called
|
||||||
|
"""
|
||||||
|
document = DocumentFactory.create(content="Some content")
|
||||||
|
mock_retrieve = mocker.patch(
|
||||||
|
"paperless_ai.ai_classifier.retrieve_similar_nodes",
|
||||||
|
return_value=[],
|
||||||
|
)
|
||||||
|
mock_fulltext = mocker.patch(
|
||||||
|
"paperless_ai.ai_classifier._fulltext_similar_documents",
|
||||||
|
)
|
||||||
|
|
||||||
|
get_taxonomy_context(document, user=None)
|
||||||
|
|
||||||
|
mock_retrieve.assert_called_once()
|
||||||
|
mock_fulltext.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
class TestGetTaxonomyContextVisibility:
|
class TestGetTaxonomyContextVisibility:
|
||||||
"""get_taxonomy_context must not materialize every visible document id
|
"""get_taxonomy_context must not materialize every visible document id
|
||||||
for a user who can already see the whole library: a superuser (like no
|
for a user who can already see the whole library: a superuser (like no
|
||||||
@@ -459,6 +528,7 @@ class TestGetTaxonomyContextVisibility:
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
@pytest.mark.django_db
|
@pytest.mark.django_db
|
||||||
|
@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
|
||||||
def test_skips_permission_lookup_for_superuser(
|
def test_skips_permission_lookup_for_superuser(
|
||||||
self,
|
self,
|
||||||
mocker: pytest_mock.MockerFixture,
|
mocker: pytest_mock.MockerFixture,
|
||||||
@@ -477,17 +547,18 @@ class TestGetTaxonomyContextVisibility:
|
|||||||
"paperless_ai.ai_classifier.retrieve_similar_nodes",
|
"paperless_ai.ai_classifier.retrieve_similar_nodes",
|
||||||
return_value=[],
|
return_value=[],
|
||||||
)
|
)
|
||||||
mock_get_objects = mocker.patch(
|
mock_permitted = mocker.patch(
|
||||||
"paperless_ai.ai_classifier.get_objects_for_user_owner_aware",
|
"paperless_ai.ai_classifier.permitted_object_ids",
|
||||||
)
|
)
|
||||||
user = UserFactory.create(is_superuser=True)
|
user = UserFactory.create(is_superuser=True)
|
||||||
|
|
||||||
get_taxonomy_context(document, user)
|
get_taxonomy_context(document, user)
|
||||||
|
|
||||||
mock_get_objects.assert_not_called()
|
mock_permitted.assert_not_called()
|
||||||
assert mock_retrieve.call_args.kwargs["document_ids"] is None
|
assert mock_retrieve.call_args.kwargs["document_ids"] is None
|
||||||
|
|
||||||
@pytest.mark.django_db
|
@pytest.mark.django_db
|
||||||
|
@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
|
||||||
def test_skips_permission_lookup_when_no_user(
|
def test_skips_permission_lookup_when_no_user(
|
||||||
self,
|
self,
|
||||||
mocker: pytest_mock.MockerFixture,
|
mocker: pytest_mock.MockerFixture,
|
||||||
@@ -506,16 +577,17 @@ class TestGetTaxonomyContextVisibility:
|
|||||||
"paperless_ai.ai_classifier.retrieve_similar_nodes",
|
"paperless_ai.ai_classifier.retrieve_similar_nodes",
|
||||||
return_value=[],
|
return_value=[],
|
||||||
)
|
)
|
||||||
mock_get_objects = mocker.patch(
|
mock_permitted = mocker.patch(
|
||||||
"paperless_ai.ai_classifier.get_objects_for_user_owner_aware",
|
"paperless_ai.ai_classifier.permitted_object_ids",
|
||||||
)
|
)
|
||||||
|
|
||||||
get_taxonomy_context(document, None)
|
get_taxonomy_context(document, None)
|
||||||
|
|
||||||
mock_get_objects.assert_not_called()
|
mock_permitted.assert_not_called()
|
||||||
assert mock_retrieve.call_args.kwargs["document_ids"] is None
|
assert mock_retrieve.call_args.kwargs["document_ids"] is None
|
||||||
|
|
||||||
@pytest.mark.django_db
|
@pytest.mark.django_db
|
||||||
|
@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
|
||||||
def test_restricts_to_visible_documents_for_non_superuser(
|
def test_restricts_to_visible_documents_for_non_superuser(
|
||||||
self,
|
self,
|
||||||
mocker: pytest_mock.MockerFixture,
|
mocker: pytest_mock.MockerFixture,
|
||||||
@@ -526,7 +598,7 @@ class TestGetTaxonomyContextVisibility:
|
|||||||
WHEN:
|
WHEN:
|
||||||
- get_taxonomy_context() is called
|
- get_taxonomy_context() is called
|
||||||
THEN:
|
THEN:
|
||||||
- The user's visible document ids are looked up and passed to
|
- The user's permitted document ids are looked up and passed to
|
||||||
retrieve_similar_nodes() as a restriction
|
retrieve_similar_nodes() as a restriction
|
||||||
"""
|
"""
|
||||||
document = DocumentFactory.create(content="Some content")
|
document = DocumentFactory.create(content="Some content")
|
||||||
@@ -534,21 +606,232 @@ class TestGetTaxonomyContextVisibility:
|
|||||||
"paperless_ai.ai_classifier.retrieve_similar_nodes",
|
"paperless_ai.ai_classifier.retrieve_similar_nodes",
|
||||||
return_value=[],
|
return_value=[],
|
||||||
)
|
)
|
||||||
mock_queryset = mocker.MagicMock()
|
mock_permitted = mocker.patch(
|
||||||
mock_queryset.values_list.return_value = [1, 2, 3]
|
"paperless_ai.ai_classifier.permitted_object_ids",
|
||||||
mock_get_objects = mocker.patch(
|
return_value=[1, 2, 3],
|
||||||
"paperless_ai.ai_classifier.get_objects_for_user_owner_aware",
|
|
||||||
return_value=mock_queryset,
|
|
||||||
)
|
)
|
||||||
user = UserFactory.create(is_superuser=False)
|
user = UserFactory.create(is_superuser=False)
|
||||||
|
|
||||||
get_taxonomy_context(document, user)
|
get_taxonomy_context(document, user)
|
||||||
|
|
||||||
mock_get_objects.assert_called_once_with(user, "view_document", Document)
|
mock_permitted.assert_called_once_with(user, Document, "view_document")
|
||||||
assert mock_retrieve.call_args.kwargs["document_ids"] == [1, 2, 3]
|
assert mock_retrieve.call_args.kwargs["document_ids"] == [1, 2, 3]
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.django_db
|
@pytest.mark.django_db
|
||||||
|
class TestFulltextSimilarDocuments:
|
||||||
|
"""_fulltext_similar_documents is the no-embedding-backend fallback: it
|
||||||
|
asks the Tantivy full-text index for "More Like This" neighbours instead
|
||||||
|
of the vector store, and synthesizes a rank-based weight since Tantivy's
|
||||||
|
more_like_this_ids returns only an ordered id list, no scores.
|
||||||
|
"""
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def fulltext_backend(
|
||||||
|
self,
|
||||||
|
mocker: pytest_mock.MockerFixture,
|
||||||
|
) -> Generator[TantivyBackend, None, None]:
|
||||||
|
"""An in-memory Tantivy backend, wired up as the module-level
|
||||||
|
singleton _fulltext_similar_documents resolves via get_backend()."""
|
||||||
|
backend = TantivyBackend(path=None)
|
||||||
|
backend.open()
|
||||||
|
mocker.patch("documents.search.get_backend", return_value=backend)
|
||||||
|
try:
|
||||||
|
yield backend
|
||||||
|
finally:
|
||||||
|
backend.close()
|
||||||
|
|
||||||
|
def test_ranks_by_rank_based_weight_descending(
|
||||||
|
self,
|
||||||
|
fulltext_backend: TantivyBackend,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A source document and two similar documents indexed in Tantivy
|
||||||
|
WHEN:
|
||||||
|
- _fulltext_similar_documents() is called
|
||||||
|
THEN:
|
||||||
|
- Each result's weight reflects its rank (first result weighted
|
||||||
|
higher than the second), not a raw similarity score
|
||||||
|
"""
|
||||||
|
source = DocumentFactory.create(content="quarterly financial report details")
|
||||||
|
first = DocumentFactory.create(content="quarterly financial report details")
|
||||||
|
second = DocumentFactory.create(content="financial report")
|
||||||
|
for doc in (source, first, second):
|
||||||
|
fulltext_backend.add_or_update(doc)
|
||||||
|
|
||||||
|
result = _fulltext_similar_documents(source, user=None, top_k=5)
|
||||||
|
|
||||||
|
assert len(result) == 2
|
||||||
|
weight_by_id = {s["document_id"]: s["weight"] for s in result}
|
||||||
|
assert weight_by_id[first.pk] > weight_by_id[second.pk]
|
||||||
|
|
||||||
|
def test_excludes_source_document(
|
||||||
|
self,
|
||||||
|
fulltext_backend: TantivyBackend,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A source document indexed in Tantivy with no other documents
|
||||||
|
WHEN:
|
||||||
|
- _fulltext_similar_documents() is called
|
||||||
|
THEN:
|
||||||
|
- An empty list is returned - the source document is never its
|
||||||
|
own similar document
|
||||||
|
"""
|
||||||
|
source = DocumentFactory.create(content="unique unrelated content")
|
||||||
|
fulltext_backend.add_or_update(source)
|
||||||
|
|
||||||
|
result = _fulltext_similar_documents(source, user=None, top_k=5)
|
||||||
|
|
||||||
|
assert result == []
|
||||||
|
|
||||||
|
def test_empty_index_returns_empty_list(
|
||||||
|
self,
|
||||||
|
fulltext_backend: TantivyBackend,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A document that has never been indexed (fresh/empty Tantivy index)
|
||||||
|
WHEN:
|
||||||
|
- _fulltext_similar_documents() is called
|
||||||
|
THEN:
|
||||||
|
- An empty list is returned rather than raising
|
||||||
|
"""
|
||||||
|
source = DocumentFactory.create(content="never indexed")
|
||||||
|
|
||||||
|
result = _fulltext_similar_documents(source, user=None, top_k=5)
|
||||||
|
|
||||||
|
assert result == []
|
||||||
|
|
||||||
|
def test_respects_top_k_limit(
|
||||||
|
self,
|
||||||
|
fulltext_backend: TantivyBackend,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A source document and four similar documents indexed
|
||||||
|
WHEN:
|
||||||
|
- _fulltext_similar_documents() is called with top_k=2
|
||||||
|
THEN:
|
||||||
|
- At most 2 results are returned
|
||||||
|
"""
|
||||||
|
source = DocumentFactory.create(content="shared overlapping keyword text")
|
||||||
|
fulltext_backend.add_or_update(source)
|
||||||
|
for _ in range(4):
|
||||||
|
fulltext_backend.add_or_update(
|
||||||
|
DocumentFactory.create(content="shared overlapping keyword text"),
|
||||||
|
)
|
||||||
|
|
||||||
|
result = _fulltext_similar_documents(source, user=None, top_k=2)
|
||||||
|
|
||||||
|
assert len(result) == 2
|
||||||
|
|
||||||
|
def test_result_shape_is_similar_document(
|
||||||
|
self,
|
||||||
|
fulltext_backend: TantivyBackend,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A source document and one similar document indexed
|
||||||
|
WHEN:
|
||||||
|
- _fulltext_similar_documents() is called
|
||||||
|
THEN:
|
||||||
|
- Each result is a SimilarDocument (document_id + weight only)
|
||||||
|
"""
|
||||||
|
source = DocumentFactory.create(content="shared content phrase")
|
||||||
|
other = DocumentFactory.create(content="shared content phrase")
|
||||||
|
fulltext_backend.add_or_update(source)
|
||||||
|
fulltext_backend.add_or_update(other)
|
||||||
|
|
||||||
|
result = _fulltext_similar_documents(source, user=None, top_k=5)
|
||||||
|
|
||||||
|
# rank 0 (the only/best result) with top_k=5 -> weight = top_k - rank = 5.0,
|
||||||
|
# per the "first result gets top_k, the last gets 1" formula.
|
||||||
|
assert result == [SimilarDocument(document_id=other.pk, weight=5.0)]
|
||||||
|
|
||||||
|
def test_superuser_sees_other_users_documents(
|
||||||
|
self,
|
||||||
|
fulltext_backend: TantivyBackend,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A source document owned by one user and a similar document
|
||||||
|
owned by a different user, with no sharing between them
|
||||||
|
WHEN:
|
||||||
|
- _fulltext_similar_documents() is called with a superuser
|
||||||
|
THEN:
|
||||||
|
- The other user's document is still returned as a similar
|
||||||
|
document - a superuser must not be narrowed by the backend's
|
||||||
|
owner-based permission filter
|
||||||
|
"""
|
||||||
|
owner = UserFactory.create()
|
||||||
|
other_owner = UserFactory.create()
|
||||||
|
superuser = UserFactory.create(is_superuser=True)
|
||||||
|
source = DocumentFactory.create(
|
||||||
|
content="shared content phrase",
|
||||||
|
owner=owner,
|
||||||
|
)
|
||||||
|
other = DocumentFactory.create(
|
||||||
|
content="shared content phrase",
|
||||||
|
owner=other_owner,
|
||||||
|
)
|
||||||
|
fulltext_backend.add_or_update(source)
|
||||||
|
fulltext_backend.add_or_update(other)
|
||||||
|
|
||||||
|
result = _fulltext_similar_documents(source, user=superuser, top_k=5)
|
||||||
|
|
||||||
|
assert [s["document_id"] for s in result] == [other.pk]
|
||||||
|
|
||||||
|
def test_excludes_stale_permitted_document_for_regular_user(
|
||||||
|
self,
|
||||||
|
fulltext_backend: TantivyBackend,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A regular (non-superuser) user
|
||||||
|
- A similar document the user is permitted to view, and another
|
||||||
|
similar document indexed while the user still had view
|
||||||
|
permission but which has since had that permission revoked in
|
||||||
|
the database, i.e. the Tantivy index has stale permission data
|
||||||
|
WHEN:
|
||||||
|
- _fulltext_similar_documents() is called with that user
|
||||||
|
THEN:
|
||||||
|
- Only the still-permitted document is returned - the DB
|
||||||
|
re-check via restrict_queryset_to_visible() must catch the
|
||||||
|
document Tantivy's stale index still thinks is visible
|
||||||
|
"""
|
||||||
|
owner = UserFactory.create()
|
||||||
|
viewer = UserFactory.create(is_superuser=False)
|
||||||
|
source = DocumentFactory.create(
|
||||||
|
content="shared content phrase",
|
||||||
|
owner=owner,
|
||||||
|
)
|
||||||
|
permitted = DocumentFactory.create(
|
||||||
|
content="shared content phrase",
|
||||||
|
owner=owner,
|
||||||
|
)
|
||||||
|
now_private = DocumentFactory.create(
|
||||||
|
content="shared content phrase",
|
||||||
|
owner=owner,
|
||||||
|
)
|
||||||
|
assign_perm("view_document", viewer, permitted)
|
||||||
|
assign_perm("view_document", viewer, now_private)
|
||||||
|
fulltext_backend.add_or_update(source)
|
||||||
|
fulltext_backend.add_or_update(permitted)
|
||||||
|
fulltext_backend.add_or_update(now_private)
|
||||||
|
|
||||||
|
# Revoke access after indexing, without reindexing: the index still
|
||||||
|
# carries viewer as a permitted viewer for `now_private`.
|
||||||
|
remove_perm("view_document", viewer, now_private)
|
||||||
|
|
||||||
|
result = _fulltext_similar_documents(source, user=viewer, top_k=5)
|
||||||
|
|
||||||
|
assert [s["document_id"] for s in result] == [permitted.pk]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
|
||||||
@patch("paperless_ai.ai_classifier.retrieve_similar_nodes")
|
@patch("paperless_ai.ai_classifier.retrieve_similar_nodes")
|
||||||
def test_get_taxonomy_context_retrieval_failure_degrades_to_no_hints(mock_retrieve):
|
def test_get_taxonomy_context_retrieval_failure_degrades_to_no_hints(mock_retrieve):
|
||||||
"""
|
"""
|
||||||
@@ -575,6 +858,7 @@ def test_get_taxonomy_context_retrieval_failure_degrades_to_no_hints(mock_retrie
|
|||||||
|
|
||||||
|
|
||||||
@pytest.mark.django_db
|
@pytest.mark.django_db
|
||||||
|
@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
|
||||||
@patch("paperless_ai.ai_classifier.build_taxonomy_candidates")
|
@patch("paperless_ai.ai_classifier.build_taxonomy_candidates")
|
||||||
@patch("paperless_ai.ai_classifier.retrieve_similar_nodes")
|
@patch("paperless_ai.ai_classifier.retrieve_similar_nodes")
|
||||||
def test_get_taxonomy_context_candidate_building_failure_degrades_to_no_hints(
|
def test_get_taxonomy_context_candidate_building_failure_degrades_to_no_hints(
|
||||||
|
|||||||
@@ -1188,9 +1188,7 @@ class TestRetrieveSimilarNodesAgainstRealIndex:
|
|||||||
|
|
||||||
nodes = indexing.retrieve_similar_nodes(a, document_ids=[b.id])
|
nodes = indexing.retrieve_similar_nodes(a, document_ids=[b.id])
|
||||||
|
|
||||||
assert all(
|
assert all(int(node.metadata["document_id"]) == b.id for node in nodes)
|
||||||
document_id == b.id for document_id in indexing._node_document_ids(nodes)
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_excludes_self(
|
def test_excludes_self(
|
||||||
self,
|
self,
|
||||||
@@ -1212,7 +1210,7 @@ class TestRetrieveSimilarNodesAgainstRealIndex:
|
|||||||
|
|
||||||
nodes = indexing.retrieve_similar_nodes(a, top_k=5)
|
nodes = indexing.retrieve_similar_nodes(a, top_k=5)
|
||||||
|
|
||||||
assert set(indexing._node_document_ids(nodes)) == {b.id}
|
assert {int(node.metadata["document_id"]) for node in nodes} == {b.id}
|
||||||
|
|
||||||
def test_excludes_self_with_multiple_chunks(
|
def test_excludes_self_with_multiple_chunks(
|
||||||
self,
|
self,
|
||||||
@@ -1235,4 +1233,4 @@ class TestRetrieveSimilarNodesAgainstRealIndex:
|
|||||||
|
|
||||||
nodes = indexing.retrieve_similar_nodes(a, top_k=3)
|
nodes = indexing.retrieve_similar_nodes(a, top_k=3)
|
||||||
|
|
||||||
assert set(indexing._node_document_ids(nodes)) == {b.id}
|
assert {int(node.metadata["document_id"]) for node in nodes} == {b.id}
|
||||||
|
|||||||
@@ -1,5 +1,4 @@
|
|||||||
import json
|
import json
|
||||||
from types import SimpleNamespace
|
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
import pytest_mock
|
import pytest_mock
|
||||||
@@ -10,14 +9,14 @@ from documents.tests.factories import DocumentTypeFactory
|
|||||||
from documents.tests.factories import StoragePathFactory
|
from documents.tests.factories import StoragePathFactory
|
||||||
from documents.tests.factories import TagFactory
|
from documents.tests.factories import TagFactory
|
||||||
from documents.tests.factories import UserFactory
|
from documents.tests.factories import UserFactory
|
||||||
|
from paperless_ai.taxonomy import SimilarDocument
|
||||||
from paperless_ai.taxonomy import TaxonomyCandidates
|
from paperless_ai.taxonomy import TaxonomyCandidates
|
||||||
from paperless_ai.taxonomy import build_taxonomy_candidates
|
from paperless_ai.taxonomy import build_taxonomy_candidates
|
||||||
from paperless_ai.taxonomy import format_taxonomy_for_prompt
|
from paperless_ai.taxonomy import format_taxonomy_for_prompt
|
||||||
|
|
||||||
|
|
||||||
def make_node(document_id: int, score: float) -> SimpleNamespace:
|
def make_similar(document_id: int, weight: float) -> SimilarDocument:
|
||||||
"""A stand-in for NodeWithScore: only ``.metadata``/``.score`` are read."""
|
return SimilarDocument(document_id=document_id, weight=weight)
|
||||||
return SimpleNamespace(metadata={"document_id": str(document_id)}, score=score)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.django_db
|
@pytest.mark.django_db
|
||||||
@@ -53,9 +52,9 @@ class TestBuildTaxonomyCandidates:
|
|||||||
doc_a.tags.add(tag)
|
doc_a.tags.add(tag)
|
||||||
doc_b = DocumentFactory.create()
|
doc_b = DocumentFactory.create()
|
||||||
doc_b.tags.add(tag)
|
doc_b.tags.add(tag)
|
||||||
nodes = [make_node(doc_a.pk, 0.9), make_node(doc_b.pk, 0.4)]
|
similar_documents = [make_similar(doc_a.pk, 0.9), make_similar(doc_b.pk, 0.4)]
|
||||||
|
|
||||||
result = build_taxonomy_candidates(nodes, user=None)
|
result = build_taxonomy_candidates(similar_documents, user=None)
|
||||||
|
|
||||||
assert len(result["tags"]) == 1
|
assert len(result["tags"]) == 1
|
||||||
assert result["tags"][0]["id"] == tag.pk
|
assert result["tags"][0]["id"] == tag.pk
|
||||||
@@ -80,9 +79,9 @@ class TestBuildTaxonomyCandidates:
|
|||||||
document.tags.add(tag)
|
document.tags.add(tag)
|
||||||
tag.name = "New Name"
|
tag.name = "New Name"
|
||||||
tag.save()
|
tag.save()
|
||||||
nodes = [make_node(document.pk, 0.5)]
|
similar_documents = [make_similar(document.pk, 0.5)]
|
||||||
|
|
||||||
result = build_taxonomy_candidates(nodes, user=None)
|
result = build_taxonomy_candidates(similar_documents, user=None)
|
||||||
|
|
||||||
assert result["tags"][0]["name"] == "New Name"
|
assert result["tags"][0]["name"] == "New Name"
|
||||||
|
|
||||||
@@ -102,9 +101,9 @@ class TestBuildTaxonomyCandidates:
|
|||||||
document = DocumentFactory.create()
|
document = DocumentFactory.create()
|
||||||
document.tags.add(tag)
|
document.tags.add(tag)
|
||||||
tag.delete()
|
tag.delete()
|
||||||
nodes = [make_node(document.pk, 0.5)]
|
similar_documents = [make_similar(document.pk, 0.5)]
|
||||||
|
|
||||||
result = build_taxonomy_candidates(nodes, user=None)
|
result = build_taxonomy_candidates(similar_documents, user=None)
|
||||||
|
|
||||||
assert result["tags"] == []
|
assert result["tags"] == []
|
||||||
|
|
||||||
@@ -123,9 +122,12 @@ class TestBuildTaxonomyCandidates:
|
|||||||
strong_doc.tags.add(strong_tag)
|
strong_doc.tags.add(strong_tag)
|
||||||
weak_doc = DocumentFactory.create()
|
weak_doc = DocumentFactory.create()
|
||||||
weak_doc.tags.add(weak_tag)
|
weak_doc.tags.add(weak_tag)
|
||||||
nodes = [make_node(strong_doc.pk, 0.9), make_node(weak_doc.pk, 0.1)]
|
similar_documents = [
|
||||||
|
make_similar(strong_doc.pk, 0.9),
|
||||||
|
make_similar(weak_doc.pk, 0.1),
|
||||||
|
]
|
||||||
|
|
||||||
result = build_taxonomy_candidates(nodes, user=None)
|
result = build_taxonomy_candidates(similar_documents, user=None)
|
||||||
|
|
||||||
assert [c["name"] for c in result["tags"]] == ["Strong", "Weak"]
|
assert [c["name"] for c in result["tags"]] == ["Strong", "Weak"]
|
||||||
|
|
||||||
@@ -141,9 +143,9 @@ class TestBuildTaxonomyCandidates:
|
|||||||
document = DocumentFactory.create()
|
document = DocumentFactory.create()
|
||||||
for i in range(15):
|
for i in range(15):
|
||||||
document.tags.add(TagFactory.create(name=f"Tag{i}"))
|
document.tags.add(TagFactory.create(name=f"Tag{i}"))
|
||||||
nodes = [make_node(document.pk, 0.5)]
|
similar_documents = [make_similar(document.pk, 0.5)]
|
||||||
|
|
||||||
result = build_taxonomy_candidates(nodes, user=None)
|
result = build_taxonomy_candidates(similar_documents, user=None)
|
||||||
|
|
||||||
assert len(result["tags"]) == 10
|
assert len(result["tags"]) == 10
|
||||||
|
|
||||||
@@ -157,12 +159,12 @@ class TestBuildTaxonomyCandidates:
|
|||||||
- Only 5 correspondents are returned
|
- Only 5 correspondents are returned
|
||||||
"""
|
"""
|
||||||
correspondents = CorrespondentFactory.create_batch(7)
|
correspondents = CorrespondentFactory.create_batch(7)
|
||||||
nodes = [
|
similar_documents = [
|
||||||
make_node(DocumentFactory.create(correspondent=c).pk, 0.5)
|
make_similar(DocumentFactory.create(correspondent=c).pk, 0.5)
|
||||||
for c in correspondents
|
for c in correspondents
|
||||||
]
|
]
|
||||||
|
|
||||||
result = build_taxonomy_candidates(nodes, user=None)
|
result = build_taxonomy_candidates(similar_documents, user=None)
|
||||||
|
|
||||||
assert len(result["correspondents"]) == 5
|
assert len(result["correspondents"]) == 5
|
||||||
|
|
||||||
@@ -177,9 +179,9 @@ class TestBuildTaxonomyCandidates:
|
|||||||
"""
|
"""
|
||||||
document_type = DocumentTypeFactory.create(name="Invoice")
|
document_type = DocumentTypeFactory.create(name="Invoice")
|
||||||
document = DocumentFactory.create(document_type=document_type)
|
document = DocumentFactory.create(document_type=document_type)
|
||||||
nodes = [make_node(document.pk, 0.5)]
|
similar_documents = [make_similar(document.pk, 0.5)]
|
||||||
|
|
||||||
result = build_taxonomy_candidates(nodes, user=None)
|
result = build_taxonomy_candidates(similar_documents, user=None)
|
||||||
|
|
||||||
assert len(result["document_types"]) == 1
|
assert len(result["document_types"]) == 1
|
||||||
assert result["document_types"][0]["id"] == document_type.pk
|
assert result["document_types"][0]["id"] == document_type.pk
|
||||||
@@ -195,12 +197,12 @@ class TestBuildTaxonomyCandidates:
|
|||||||
- Only 5 document_types are returned
|
- Only 5 document_types are returned
|
||||||
"""
|
"""
|
||||||
document_types = DocumentTypeFactory.create_batch(7)
|
document_types = DocumentTypeFactory.create_batch(7)
|
||||||
nodes = [
|
similar_documents = [
|
||||||
make_node(DocumentFactory.create(document_type=dt).pk, 0.5)
|
make_similar(DocumentFactory.create(document_type=dt).pk, 0.5)
|
||||||
for dt in document_types
|
for dt in document_types
|
||||||
]
|
]
|
||||||
|
|
||||||
result = build_taxonomy_candidates(nodes, user=None)
|
result = build_taxonomy_candidates(similar_documents, user=None)
|
||||||
|
|
||||||
assert len(result["document_types"]) == 5
|
assert len(result["document_types"]) == 5
|
||||||
|
|
||||||
@@ -215,9 +217,9 @@ class TestBuildTaxonomyCandidates:
|
|||||||
"""
|
"""
|
||||||
storage_path = StoragePathFactory.create(name="Invoices")
|
storage_path = StoragePathFactory.create(name="Invoices")
|
||||||
document = DocumentFactory.create(storage_path=storage_path)
|
document = DocumentFactory.create(storage_path=storage_path)
|
||||||
nodes = [make_node(document.pk, 0.5)]
|
similar_documents = [make_similar(document.pk, 0.5)]
|
||||||
|
|
||||||
result = build_taxonomy_candidates(nodes, user=None)
|
result = build_taxonomy_candidates(similar_documents, user=None)
|
||||||
|
|
||||||
assert len(result["storage_paths"]) == 1
|
assert len(result["storage_paths"]) == 1
|
||||||
assert result["storage_paths"][0]["id"] == storage_path.pk
|
assert result["storage_paths"][0]["id"] == storage_path.pk
|
||||||
@@ -233,12 +235,12 @@ class TestBuildTaxonomyCandidates:
|
|||||||
- Only 5 storage_paths are returned
|
- Only 5 storage_paths are returned
|
||||||
"""
|
"""
|
||||||
storage_paths = StoragePathFactory.create_batch(7)
|
storage_paths = StoragePathFactory.create_batch(7)
|
||||||
nodes = [
|
similar_documents = [
|
||||||
make_node(DocumentFactory.create(storage_path=sp).pk, 0.5)
|
make_similar(DocumentFactory.create(storage_path=sp).pk, 0.5)
|
||||||
for sp in storage_paths
|
for sp in storage_paths
|
||||||
]
|
]
|
||||||
|
|
||||||
result = build_taxonomy_candidates(nodes, user=None)
|
result = build_taxonomy_candidates(similar_documents, user=None)
|
||||||
|
|
||||||
assert len(result["storage_paths"]) == 5
|
assert len(result["storage_paths"]) == 5
|
||||||
|
|
||||||
@@ -258,14 +260,14 @@ class TestBuildTaxonomyCandidates:
|
|||||||
tag = TagFactory.create(name="Restricted")
|
tag = TagFactory.create(name="Restricted")
|
||||||
document = DocumentFactory.create()
|
document = DocumentFactory.create()
|
||||||
document.tags.add(tag)
|
document.tags.add(tag)
|
||||||
nodes = [make_node(document.pk, 0.5)]
|
similar_documents = [make_similar(document.pk, 0.5)]
|
||||||
user = UserFactory.create()
|
user = UserFactory.create()
|
||||||
mocker.patch(
|
mocker.patch(
|
||||||
"documents.permissions.permitted_object_ids",
|
"documents.permissions.permitted_object_ids",
|
||||||
return_value=[], # user cannot see this tag
|
return_value=[], # user cannot see this tag
|
||||||
)
|
)
|
||||||
|
|
||||||
result = build_taxonomy_candidates(nodes, user=user)
|
result = build_taxonomy_candidates(similar_documents, user=user)
|
||||||
|
|
||||||
assert result["tags"] == []
|
assert result["tags"] == []
|
||||||
|
|
||||||
@@ -295,10 +297,10 @@ class TestBuildTaxonomyCandidates:
|
|||||||
tag.save()
|
tag.save()
|
||||||
document = DocumentFactory.create()
|
document = DocumentFactory.create()
|
||||||
document.tags.add(tag)
|
document.tags.add(tag)
|
||||||
nodes = [make_node(document.pk, 0.5)]
|
similar_documents = [make_similar(document.pk, 0.5)]
|
||||||
spy = mocker.patch("documents.permissions.permitted_object_ids")
|
spy = mocker.patch("documents.permissions.permitted_object_ids")
|
||||||
|
|
||||||
result = build_taxonomy_candidates(nodes, user=None)
|
result = build_taxonomy_candidates(similar_documents, user=None)
|
||||||
|
|
||||||
assert result["tags"][0]["name"] == "Owned"
|
assert result["tags"][0]["name"] == "Owned"
|
||||||
spy.assert_not_called()
|
spy.assert_not_called()
|
||||||
|
|||||||
Reference in New Issue
Block a user