mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-07-27 14:24:54 +00:00
Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
af7a1fcdba | ||
|
|
3e6579a4ec | ||
|
|
f4e7a2e9e4 |
@@ -129,8 +129,8 @@ jobs:
|
||||
~/.pnpm-store
|
||||
~/.cache
|
||||
key: ${{ runner.os }}-frontend-${{ hashFiles('src-ui/pnpm-lock.yaml') }}
|
||||
- name: Install dependencies
|
||||
run: cd src-ui && pnpm install --frozen-lockfile
|
||||
- name: Re-link Angular CLI
|
||||
run: cd src-ui && pnpm link @angular/cli
|
||||
- name: Run lint
|
||||
run: cd src-ui && pnpm run lint
|
||||
unit-tests:
|
||||
@@ -168,8 +168,8 @@ jobs:
|
||||
~/.pnpm-store
|
||||
~/.cache
|
||||
key: ${{ runner.os }}-frontend-${{ hashFiles('src-ui/pnpm-lock.yaml') }}
|
||||
- name: Install dependencies
|
||||
run: cd src-ui && pnpm install --frozen-lockfile
|
||||
- name: Re-link Angular CLI
|
||||
run: cd src-ui && pnpm link @angular/cli
|
||||
- name: Run Jest unit tests
|
||||
run: cd src-ui && pnpm run test --max-workers=2 --shard=${{ matrix.shard-index }}/${{ matrix.shard-count }}
|
||||
- name: Upload test results to Codecov
|
||||
@@ -223,15 +223,18 @@ jobs:
|
||||
~/.pnpm-store
|
||||
~/.cache
|
||||
key: ${{ runner.os }}-frontend-${{ hashFiles('src-ui/pnpm-lock.yaml') }}
|
||||
- name: Re-link Angular CLI
|
||||
run: cd src-ui && pnpm link @angular/cli
|
||||
- name: Install dependencies
|
||||
run: cd src-ui && pnpm install --frozen-lockfile
|
||||
run: cd src-ui && pnpm install --no-frozen-lockfile
|
||||
- name: Run Playwright E2E tests
|
||||
run: cd src-ui && pnpm exec playwright test --shard ${{ matrix.shard-index }}/${{ matrix.shard-count }}
|
||||
frontend-build:
|
||||
name: Frontend Build
|
||||
bundle-analysis:
|
||||
name: Bundle Analysis
|
||||
needs: [changes, unit-tests, e2e-tests]
|
||||
if: needs.changes.outputs.frontend_changed == 'true'
|
||||
runs-on: ubuntu-24.04
|
||||
environment: bundle-analysis
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
@@ -257,19 +260,21 @@ jobs:
|
||||
~/.pnpm-store
|
||||
~/.cache
|
||||
key: ${{ runner.os }}-frontend-${{ hashFiles('src-ui/pnpm-lock.yaml') }}
|
||||
- name: Install dependencies
|
||||
run: cd src-ui && pnpm install --frozen-lockfile
|
||||
- name: Build
|
||||
- name: Re-link Angular CLI
|
||||
run: cd src-ui && pnpm link @angular/cli
|
||||
- name: Build and analyze
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
run: cd src-ui && pnpm run build --configuration=production
|
||||
gate:
|
||||
name: Frontend CI Gate
|
||||
needs: [changes, install-dependencies, lint, unit-tests, e2e-tests, frontend-build]
|
||||
needs: [changes, install-dependencies, lint, unit-tests, e2e-tests, bundle-analysis]
|
||||
if: always()
|
||||
runs-on: ubuntu-slim
|
||||
steps:
|
||||
- name: Check gate
|
||||
env:
|
||||
BUILD_RESULT: ${{ needs['frontend-build'].result }}
|
||||
BUNDLE_ANALYSIS_RESULT: ${{ needs['bundle-analysis'].result }}
|
||||
E2E_RESULT: ${{ needs['e2e-tests'].result }}
|
||||
FRONTEND_CHANGED: ${{ needs.changes.outputs.frontend_changed }}
|
||||
INSTALL_RESULT: ${{ needs['install-dependencies'].result }}
|
||||
@@ -301,8 +306,8 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "${BUILD_RESULT}" != "success" ]]; then
|
||||
echo "::error::Frontend build job result: ${BUILD_RESULT}"
|
||||
if [[ "${BUNDLE_ANALYSIS_RESULT}" != "success" ]]; then
|
||||
echo "::error::Frontend bundle-analysis job result: ${BUNDLE_ANALYSIS_RESULT}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
@@ -61,7 +61,10 @@ jobs:
|
||||
~/.cache
|
||||
key: ${{ runner.os }}-frontenddeps-${{ hashFiles('src-ui/pnpm-lock.yaml') }}
|
||||
- name: Install frontend dependencies
|
||||
run: cd src-ui && pnpm install --frozen-lockfile
|
||||
if: steps.cache-frontend-deps.outputs.cache-hit != 'true'
|
||||
run: cd src-ui && pnpm install
|
||||
- name: Re-link Angular cli
|
||||
run: cd src-ui && pnpm link @angular/cli
|
||||
- name: Generate frontend translation strings
|
||||
run: |
|
||||
cd src-ui
|
||||
|
||||
+9
-12
@@ -56,13 +56,13 @@
|
||||
},
|
||||
"architect": {
|
||||
"build": {
|
||||
"builder": "@angular/build:application",
|
||||
"builder": "@angular-builders/custom-webpack:browser",
|
||||
"options": {
|
||||
"outputPath": {
|
||||
"base": "dist/paperless-ui",
|
||||
"browser": ""
|
||||
"customWebpackConfig": {
|
||||
"path": "./extra-webpack.config.ts"
|
||||
},
|
||||
"browser": "src/main.ts",
|
||||
"outputPath": "dist/paperless-ui",
|
||||
"main": "src/main.ts",
|
||||
"outputHashing": "none",
|
||||
"index": "src/index.html",
|
||||
"polyfills": [
|
||||
@@ -97,7 +97,6 @@
|
||||
"scripts": [],
|
||||
"allowedCommonJsDependencies": [
|
||||
"file-saver",
|
||||
"mime-names",
|
||||
"utif"
|
||||
],
|
||||
"extractLicenses": false,
|
||||
@@ -118,13 +117,11 @@
|
||||
"with": "src/environments/environment.prod.ts"
|
||||
}
|
||||
],
|
||||
"outputPath": {
|
||||
"base": "../src/documents/static/frontend/",
|
||||
"browser": ""
|
||||
},
|
||||
"outputPath": "../src/documents/static/frontend/",
|
||||
"optimization": true,
|
||||
"outputHashing": "none",
|
||||
"sourceMap": false,
|
||||
"namedChunks": false,
|
||||
"extractLicenses": true,
|
||||
"budgets": [
|
||||
{
|
||||
@@ -148,7 +145,7 @@
|
||||
"defaultConfiguration": ""
|
||||
},
|
||||
"serve": {
|
||||
"builder": "@angular/build:dev-server",
|
||||
"builder": "@angular-builders/custom-webpack:dev-server",
|
||||
"options": {
|
||||
"buildTarget": "paperless-ui:build:en-US"
|
||||
},
|
||||
@@ -159,7 +156,7 @@
|
||||
}
|
||||
},
|
||||
"extract-i18n": {
|
||||
"builder": "@angular/build:extract-i18n",
|
||||
"builder": "@angular-builders/custom-webpack:extract-i18n",
|
||||
"options": {
|
||||
"buildTarget": "paperless-ui:build"
|
||||
}
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
import {
|
||||
CustomWebpackBrowserSchema,
|
||||
TargetOptions,
|
||||
} from '@angular-builders/custom-webpack'
|
||||
import * as webpack from 'webpack'
|
||||
const { codecovWebpackPlugin } = require('@codecov/webpack-plugin')
|
||||
|
||||
export default (
|
||||
config: webpack.Configuration,
|
||||
options: CustomWebpackBrowserSchema,
|
||||
targetOptions: TargetOptions
|
||||
) => {
|
||||
if (config.plugins) {
|
||||
config.plugins.push(
|
||||
codecovWebpackPlugin({
|
||||
enableBundleAnalysis: process.env.CODECOV_TOKEN !== undefined,
|
||||
bundleName: 'paperless-ngx',
|
||||
uploadToken: process.env.CODECOV_TOKEN,
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
return config
|
||||
}
|
||||
+49
-49
@@ -1277,7 +1277,7 @@
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
<context context-type="linenumber">1787</context>
|
||||
<context context-type="linenumber">1791</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="1577733187050997705" datatype="html">
|
||||
@@ -2184,7 +2184,7 @@
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
<context context-type="linenumber">653</context>
|
||||
<context context-type="linenumber">657</context>
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.html</context>
|
||||
@@ -3087,11 +3087,11 @@
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
<context context-type="linenumber">1405</context>
|
||||
<context context-type="linenumber">1409</context>
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
<context context-type="linenumber">1788</context>
|
||||
<context context-type="linenumber">1792</context>
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-list/bulk-editor/bulk-editor.component.ts</context>
|
||||
@@ -3688,7 +3688,7 @@
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
<context context-type="linenumber">1358</context>
|
||||
<context context-type="linenumber">1362</context>
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-list/bulk-editor/bulk-editor.component.ts</context>
|
||||
@@ -3800,7 +3800,7 @@
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
<context context-type="linenumber">1841</context>
|
||||
<context context-type="linenumber">1845</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="6661109599266152398" datatype="html">
|
||||
@@ -3811,7 +3811,7 @@
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
<context context-type="linenumber">1842</context>
|
||||
<context context-type="linenumber">1846</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="5162686434580248853" datatype="html">
|
||||
@@ -3822,7 +3822,7 @@
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
<context context-type="linenumber">1843</context>
|
||||
<context context-type="linenumber">1847</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="8157388568390631653" datatype="html">
|
||||
@@ -5724,7 +5724,7 @@
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
<context context-type="linenumber">1362</context>
|
||||
<context context-type="linenumber">1366</context>
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-list/bulk-editor/bulk-editor.component.ts</context>
|
||||
@@ -7966,88 +7966,88 @@
|
||||
<source>Enter Password</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.html</context>
|
||||
<context context-type="linenumber">512</context>
|
||||
<context context-type="linenumber">513</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="5758784066858623886" datatype="html">
|
||||
<source>Error retrieving metadata</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
<context context-type="linenumber">423</context>
|
||||
<context context-type="linenumber">424</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="2218903673684131427" datatype="html">
|
||||
<source>An error occurred loading content: <x id="PH" equiv-text="err.message ?? err.toString()"/></source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
<context context-type="linenumber">525,527</context>
|
||||
<context context-type="linenumber">526,528</context>
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
<context context-type="linenumber">982,984</context>
|
||||
<context context-type="linenumber">986,988</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="6357361810318120957" datatype="html">
|
||||
<source>Document was updated</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
<context context-type="linenumber">648</context>
|
||||
<context context-type="linenumber">652</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="5154064822428631306" datatype="html">
|
||||
<source>Document was updated at <x id="PH" equiv-text="formattedModified"/>.</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
<context context-type="linenumber">649</context>
|
||||
<context context-type="linenumber">653</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="8462497568316256794" datatype="html">
|
||||
<source>Reload to discard your local unsaved edits and load the latest remote version.</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
<context context-type="linenumber">650</context>
|
||||
<context context-type="linenumber">654</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="7967484035994732534" datatype="html">
|
||||
<source>Reload</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
<context context-type="linenumber">652</context>
|
||||
<context context-type="linenumber">656</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="2907037627372942104" datatype="html">
|
||||
<source>Document reloaded with latest changes.</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
<context context-type="linenumber">708</context>
|
||||
<context context-type="linenumber">712</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="6435639868943916539" datatype="html">
|
||||
<source>Document reloaded.</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
<context context-type="linenumber">719</context>
|
||||
<context context-type="linenumber">723</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="6142395741265832184" datatype="html">
|
||||
<source>Next document</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
<context context-type="linenumber">821</context>
|
||||
<context context-type="linenumber">825</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="651985345816518480" datatype="html">
|
||||
<source>Previous document</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
<context context-type="linenumber">831</context>
|
||||
<context context-type="linenumber">835</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="2885986061416655600" datatype="html">
|
||||
<source>Close document</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
<context context-type="linenumber">839</context>
|
||||
<context context-type="linenumber">843</context>
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/services/open-documents.service.ts</context>
|
||||
@@ -8058,67 +8058,67 @@
|
||||
<source>Save document</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
<context context-type="linenumber">846</context>
|
||||
<context context-type="linenumber">850</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="1784543155727940353" datatype="html">
|
||||
<source>Save and close / next</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
<context context-type="linenumber">855</context>
|
||||
<context context-type="linenumber">859</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="7427704425579737895" datatype="html">
|
||||
<source>Error retrieving version content</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
<context context-type="linenumber">965</context>
|
||||
<context context-type="linenumber">969</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="3456881259945295697" datatype="html">
|
||||
<source>Error retrieving suggestions.</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
<context context-type="linenumber">1026</context>
|
||||
<context context-type="linenumber">1030</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="2194092841814123758" datatype="html">
|
||||
<source>Document "<x id="PH" equiv-text="newValues.title"/>" saved successfully.</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
<context context-type="linenumber">1238</context>
|
||||
<context context-type="linenumber">1242</context>
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
<context context-type="linenumber">1265</context>
|
||||
<context context-type="linenumber">1269</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="6626387786259219838" datatype="html">
|
||||
<source>Error saving document "<x id="PH" equiv-text="this.document().title"/>"</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
<context context-type="linenumber">1271</context>
|
||||
<context context-type="linenumber">1275</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="448882439049417053" datatype="html">
|
||||
<source>Error saving document</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
<context context-type="linenumber">1326</context>
|
||||
<context context-type="linenumber">1330</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="8410796510716511826" datatype="html">
|
||||
<source>Do you really want to move the document "<x id="PH" equiv-text="this.document().title"/>" to the trash?</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
<context context-type="linenumber">1359</context>
|
||||
<context context-type="linenumber">1363</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="282586936710748252" datatype="html">
|
||||
<source>Documents can be restored prior to permanent deletion.</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
<context context-type="linenumber">1360</context>
|
||||
<context context-type="linenumber">1364</context>
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-list/bulk-editor/bulk-editor.component.ts</context>
|
||||
@@ -8129,14 +8129,14 @@
|
||||
<source>Error deleting document</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
<context context-type="linenumber">1381</context>
|
||||
<context context-type="linenumber">1385</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="619486176823357521" datatype="html">
|
||||
<source>Reprocess confirm</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
<context context-type="linenumber">1401</context>
|
||||
<context context-type="linenumber">1405</context>
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-list/bulk-editor/bulk-editor.component.ts</context>
|
||||
@@ -8147,102 +8147,102 @@
|
||||
<source>This operation will permanently recreate the archive file for this document.</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
<context context-type="linenumber">1402</context>
|
||||
<context context-type="linenumber">1406</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="302054111564709516" datatype="html">
|
||||
<source>The archive file will be re-generated with the current settings.</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
<context context-type="linenumber">1403</context>
|
||||
<context context-type="linenumber">1407</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="4700389117298802932" datatype="html">
|
||||
<source>Reprocess operation for "<x id="PH" equiv-text="this.document().title"/>" will begin in the background.</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
<context context-type="linenumber">1413</context>
|
||||
<context context-type="linenumber">1417</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="4409560272830824468" datatype="html">
|
||||
<source>Error executing operation</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
<context context-type="linenumber">1424</context>
|
||||
<context context-type="linenumber">1428</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="6030453331794586802" datatype="html">
|
||||
<source>Error downloading document</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
<context context-type="linenumber">1487</context>
|
||||
<context context-type="linenumber">1491</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="4458954481601077369" datatype="html">
|
||||
<source>Page Fit</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
<context context-type="linenumber">1565</context>
|
||||
<context context-type="linenumber">1569</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="4663705961777238777" datatype="html">
|
||||
<source>PDF edit operation for "<x id="PH" equiv-text="this.document().title"/>" will begin in the background.</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
<context context-type="linenumber">1808</context>
|
||||
<context context-type="linenumber">1812</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="9043972994040261999" datatype="html">
|
||||
<source>Error executing PDF edit operation</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
<context context-type="linenumber">1820</context>
|
||||
<context context-type="linenumber">1824</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="6172690334763056188" datatype="html">
|
||||
<source>Please enter the current password before attempting to remove it.</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
<context context-type="linenumber">1831</context>
|
||||
<context context-type="linenumber">1835</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="968660764814228922" datatype="html">
|
||||
<source>Password removal operation for "<x id="PH" equiv-text="this.document().title"/>" will begin in the background.</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
<context context-type="linenumber">1865</context>
|
||||
<context context-type="linenumber">1869</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="2282118435712883014" datatype="html">
|
||||
<source>Error executing password removal operation</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
<context context-type="linenumber">1879</context>
|
||||
<context context-type="linenumber">1883</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="3740891324955700797" datatype="html">
|
||||
<source>Print failed.</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
<context context-type="linenumber">1925</context>
|
||||
<context context-type="linenumber">1929</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="6457245677384603573" datatype="html">
|
||||
<source>Error loading document for printing.</source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
<context context-type="linenumber">1938</context>
|
||||
<context context-type="linenumber">1942</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="6085793215710522488" datatype="html">
|
||||
<source>An error occurred loading tiff: <x id="PH" equiv-text="err.toString()"/></source>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
<context context-type="linenumber">2008</context>
|
||||
<context context-type="linenumber">2012</context>
|
||||
</context-group>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
|
||||
<context context-type="linenumber">2014</context>
|
||||
<context context-type="linenumber">2018</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="4958946940233632319" datatype="html">
|
||||
|
||||
+6
-3
@@ -39,6 +39,7 @@
|
||||
"uuid": "^14.0.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@angular-builders/custom-webpack": "^22.0.1",
|
||||
"@angular-builders/jest": "^22.0.1",
|
||||
"@angular-devkit/core": "^22.0.5",
|
||||
"@angular-devkit/schematics": "^22.0.5",
|
||||
@@ -47,9 +48,10 @@
|
||||
"@angular-eslint/eslint-plugin-template": "22.0.0",
|
||||
"@angular-eslint/schematics": "22.0.0",
|
||||
"@angular-eslint/template-parser": "22.0.0",
|
||||
"@angular/build": "22.1.0-rc.0",
|
||||
"@angular/cli": "22.1.0-rc.0",
|
||||
"@angular/build": "^22.0.5",
|
||||
"@angular/cli": "~22.0.5",
|
||||
"@angular/compiler-cli": "~22.0.5",
|
||||
"@codecov/webpack-plugin": "^2.0.1",
|
||||
"@playwright/test": "^1.61.1",
|
||||
"@types/jest": "^30.0.0",
|
||||
"@types/node": "^26.0.0",
|
||||
@@ -64,7 +66,8 @@
|
||||
"jest-websocket-mock": "^2.5.0",
|
||||
"prettier-plugin-organize-imports": "^4.3.0",
|
||||
"ts-node": "~10.9.1",
|
||||
"typescript": "^6.0.3"
|
||||
"typescript": "^6.0.3",
|
||||
"webpack": "^5.107.2"
|
||||
},
|
||||
"packageManager": "pnpm@10.26.0"
|
||||
}
|
||||
|
||||
Generated
+1365
-1563
File diff suppressed because it is too large
Load Diff
@@ -1,5 +1,7 @@
|
||||
import sqlite3
|
||||
from collections.abc import Generator
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
@@ -9,6 +11,7 @@ from llama_index.core.vector_stores.types import MetadataFilter
|
||||
from llama_index.core.vector_stores.types import MetadataFilters
|
||||
from llama_index.core.vector_stores.types import VectorStoreQuery
|
||||
|
||||
from paperless_ai import vector_store as vs_mod
|
||||
from paperless_ai.vector_store import DB_FILENAME
|
||||
from paperless_ai.vector_store import DEFAULT_TABLE_NAME
|
||||
from paperless_ai.vector_store import MIGRATIONS
|
||||
@@ -89,8 +92,87 @@ def _ne_filter(document_id: str):
|
||||
)
|
||||
|
||||
|
||||
def _chunk_index_rows(
|
||||
store: PaperlessSqliteVecVectorStore,
|
||||
document_id: str | None = None,
|
||||
) -> list[tuple[str, str]]:
|
||||
"""The document_chunks side table, as (chunk_id, document_id) pairs."""
|
||||
sql = "SELECT chunk_id, document_id FROM document_chunks"
|
||||
params: list[str] = []
|
||||
if document_id is not None:
|
||||
sql += " WHERE document_id = ?"
|
||||
params.append(document_id)
|
||||
rows = store.client.execute(sql + " ORDER BY chunk_id", params).fetchall()
|
||||
return [(r["chunk_id"], r["document_id"]) for r in rows]
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _pending_migrations(*migrations: Migration, schema_version: int) -> Iterator[None]:
|
||||
"""Register ``migrations`` and raise the module's SCHEMA_VERSION for the
|
||||
duration of the block, so check_and_run_migrations() sees them as pending.
|
||||
"""
|
||||
original = vs_mod.SCHEMA_VERSION
|
||||
MIGRATIONS.extend(migrations)
|
||||
vs_mod.SCHEMA_VERSION = schema_version
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
for migration in migrations:
|
||||
MIGRATIONS.remove(migration)
|
||||
vs_mod.SCHEMA_VERSION = original
|
||||
|
||||
|
||||
def _copying_apply(src: sqlite3.Connection, dst: sqlite3.Connection, dim: int) -> None:
|
||||
"""A structural migration apply() that just copies every row across.
|
||||
|
||||
Deliberately spells its SQL out rather than reusing the production
|
||||
helpers, so these tests exercise the migration machinery against a
|
||||
realistic third-party apply() instead of co-drifting with it.
|
||||
"""
|
||||
dst.execute( # nosemgrep
|
||||
f"CREATE VIRTUAL TABLE {DEFAULT_TABLE_NAME} USING vec0("
|
||||
"id TEXT PRIMARY KEY, document_id TEXT, modified TEXT,"
|
||||
f" +node_content TEXT, embedding float[{dim}] distance_metric=cosine"
|
||||
")",
|
||||
)
|
||||
dst.execute(
|
||||
"INSERT INTO index_meta (key, value) VALUES ('dim', ?) "
|
||||
"ON CONFLICT(key) DO UPDATE SET value = excluded.value",
|
||||
(str(dim),),
|
||||
)
|
||||
rows = src.execute(
|
||||
"SELECT id, document_id, modified, node_content, embedding "
|
||||
f"FROM {DEFAULT_TABLE_NAME}",
|
||||
).fetchall()
|
||||
dst.execute("BEGIN IMMEDIATE")
|
||||
dst.executemany(
|
||||
f"INSERT INTO {DEFAULT_TABLE_NAME} "
|
||||
"(id, document_id, modified, node_content, embedding) "
|
||||
"VALUES (?, ?, ?, ?, ?)",
|
||||
[
|
||||
(
|
||||
r["id"],
|
||||
r["document_id"],
|
||||
r["modified"],
|
||||
r["node_content"],
|
||||
bytes(r["embedding"]),
|
||||
)
|
||||
for r in rows
|
||||
],
|
||||
)
|
||||
dst.execute(
|
||||
"INSERT INTO index_meta (key, value) VALUES ('total_inserts', ?) "
|
||||
"ON CONFLICT(key) DO UPDATE SET value = excluded.value",
|
||||
(str(len(rows)),),
|
||||
)
|
||||
dst.execute("COMMIT")
|
||||
|
||||
|
||||
class TestCrud:
|
||||
def test_add_then_query_returns_node(self, store) -> None:
|
||||
def test_add_then_query_returns_node(
|
||||
self,
|
||||
store: PaperlessSqliteVecVectorStore,
|
||||
) -> None:
|
||||
node = make_node("n1", "1")
|
||||
assert store.add([node]) == ["n1"]
|
||||
result = _query(store, node.embedding, top_k=1)
|
||||
@@ -99,21 +181,30 @@ class TestCrud:
|
||||
# cosine distance of the identical vector is 0 -> similarity 1
|
||||
assert result.similarities[0] == pytest.approx(1.0)
|
||||
|
||||
def test_query_empty_store_returns_empty_no_raise(self, store) -> None:
|
||||
def test_query_empty_store_returns_empty_no_raise(
|
||||
self,
|
||||
store: PaperlessSqliteVecVectorStore,
|
||||
) -> None:
|
||||
result = _query(store, [0.0] * DIM)
|
||||
assert result.ids == [] and result.nodes == [] and result.similarities == []
|
||||
|
||||
def test_add_empty_list_is_noop(self, store) -> None:
|
||||
def test_add_empty_list_is_noop(self, store: PaperlessSqliteVecVectorStore) -> None:
|
||||
assert store.add([]) == []
|
||||
assert not store.table_exists()
|
||||
|
||||
def test_delete_removes_all_chunks_of_document(self, store) -> None:
|
||||
def test_delete_removes_all_chunks_of_document(
|
||||
self,
|
||||
store: PaperlessSqliteVecVectorStore,
|
||||
) -> None:
|
||||
store.add([make_node("a1", "1"), make_node("a2", "1"), make_node("b1", "2")])
|
||||
store.delete("1")
|
||||
result = _query(store, [0.0] * DIM, top_k=10)
|
||||
assert result.ids == ["b1"]
|
||||
|
||||
def test_query_with_in_filter_scopes_results(self, store) -> None:
|
||||
def test_query_with_in_filter_scopes_results(
|
||||
self,
|
||||
store: PaperlessSqliteVecVectorStore,
|
||||
) -> None:
|
||||
store.add(
|
||||
[
|
||||
make_node("a1", "1", seed=0.0),
|
||||
@@ -124,7 +215,10 @@ class TestCrud:
|
||||
result = _query(store, [0.0] * DIM, top_k=10, filters=_in_filter(["2", "3"]))
|
||||
assert sorted(result.ids) == ["b1", "c1"]
|
||||
|
||||
def test_query_respects_top_k_with_filter(self, store) -> None:
|
||||
def test_query_respects_top_k_with_filter(
|
||||
self,
|
||||
store: PaperlessSqliteVecVectorStore,
|
||||
) -> None:
|
||||
# k semantics: global top-k even with IN filters (document_id is a
|
||||
# metadata column, not a partition key -- see design doc).
|
||||
store.add(
|
||||
@@ -139,7 +233,10 @@ class TestCrud:
|
||||
assert len(result.ids) == 3
|
||||
assert result.similarities == sorted(result.similarities, reverse=True)
|
||||
|
||||
def test_get_nodes_filter_and_empty_paths(self, store) -> None:
|
||||
def test_get_nodes_filter_and_empty_paths(
|
||||
self,
|
||||
store: PaperlessSqliteVecVectorStore,
|
||||
) -> None:
|
||||
assert store.get_nodes(filters=_in_filter(["1"])) == [] # no table yet
|
||||
store.add([make_node("a1", "1"), make_node("b1", "2")])
|
||||
nodes = store.get_nodes(filters=_in_filter(["1"]))
|
||||
@@ -147,7 +244,10 @@ class TestCrud:
|
||||
assert nodes[0].embedding is not None
|
||||
assert store.get_nodes(filters=_in_filter(["999"])) == []
|
||||
|
||||
def test_query_with_eq_filter_scopes_results(self, store) -> None:
|
||||
def test_query_with_eq_filter_scopes_results(
|
||||
self,
|
||||
store: PaperlessSqliteVecVectorStore,
|
||||
) -> None:
|
||||
store.add(
|
||||
[
|
||||
make_node("a1", "1", seed=0.0),
|
||||
@@ -163,18 +263,25 @@ class TestCrud:
|
||||
)
|
||||
assert result.ids == ["b1"]
|
||||
|
||||
def test_get_nodes_node_ids_not_implemented(self, store) -> None:
|
||||
def test_get_nodes_node_ids_not_implemented(
|
||||
self,
|
||||
store: PaperlessSqliteVecVectorStore,
|
||||
) -> None:
|
||||
with pytest.raises(NotImplementedError):
|
||||
store.get_nodes(node_ids=["x"])
|
||||
|
||||
def test_fresh_instance_sees_existing_table(self, store, tmp_path: Path) -> None:
|
||||
def test_fresh_instance_sees_existing_table(
|
||||
self,
|
||||
store: PaperlessSqliteVecVectorStore,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
store.add([make_node("a1", "1")])
|
||||
with PaperlessSqliteVecVectorStore(uri=str(tmp_path)) as reopened:
|
||||
assert reopened.table_exists()
|
||||
assert reopened.vector_dim() == DIM
|
||||
assert _query(reopened, [0.0] * DIM, top_k=1).ids == ["a1"]
|
||||
|
||||
def test_table_exists_and_drop(self, store) -> None:
|
||||
def test_table_exists_and_drop(self, store: PaperlessSqliteVecVectorStore) -> None:
|
||||
assert not store.table_exists()
|
||||
store.add([make_node("a1", "1")])
|
||||
assert store.table_exists()
|
||||
@@ -189,7 +296,10 @@ class TestBuildWhere:
|
||||
assert where == "(document_id != ?)"
|
||||
assert params == ["1"]
|
||||
|
||||
def test_query_with_ne_filter_excludes_matching_document(self, store) -> None:
|
||||
def test_query_with_ne_filter_excludes_matching_document(
|
||||
self,
|
||||
store: PaperlessSqliteVecVectorStore,
|
||||
) -> None:
|
||||
store.add([make_node("a1", "1"), make_node("b1", "2")])
|
||||
assert sorted(
|
||||
_query(store, [0.0] * DIM, top_k=5, filters=_ne_filter("1")).ids,
|
||||
@@ -214,7 +324,10 @@ class TestBuildWhere:
|
||||
assert where == "1 = 0"
|
||||
assert params == []
|
||||
|
||||
def test_query_with_untranslatable_filter_returns_no_rows(self, store) -> None:
|
||||
def test_query_with_untranslatable_filter_returns_no_rows(
|
||||
self,
|
||||
store: PaperlessSqliteVecVectorStore,
|
||||
) -> None:
|
||||
store.add([make_node("a1", "1"), make_node("b1", "2")])
|
||||
nested = MetadataFilters(
|
||||
filters=[
|
||||
@@ -232,7 +345,10 @@ class TestBuildWhere:
|
||||
|
||||
|
||||
class TestUpsert:
|
||||
def test_upsert_replaces_and_prunes_stale_chunks(self, store) -> None:
|
||||
def test_upsert_replaces_and_prunes_stale_chunks(
|
||||
self,
|
||||
store: PaperlessSqliteVecVectorStore,
|
||||
) -> None:
|
||||
store.add(
|
||||
[make_node("d1c1", "1"), make_node("d1c2", "1"), make_node("d2c1", "2")],
|
||||
)
|
||||
@@ -240,18 +356,24 @@ class TestUpsert:
|
||||
result = _query(store, [0.0] * DIM, top_k=10)
|
||||
assert sorted(result.ids) == ["d1new", "d2c1"]
|
||||
|
||||
def test_upsert_creates_table_when_missing(self, store) -> None:
|
||||
def test_upsert_creates_table_when_missing(
|
||||
self,
|
||||
store: PaperlessSqliteVecVectorStore,
|
||||
) -> None:
|
||||
store.upsert_document("1", [make_node("a1", "1")])
|
||||
assert _query(store, [0.0] * DIM, top_k=1).ids == ["a1"]
|
||||
|
||||
def test_upsert_empty_nodes_removes_document(self, store) -> None:
|
||||
def test_upsert_empty_nodes_removes_document(
|
||||
self,
|
||||
store: PaperlessSqliteVecVectorStore,
|
||||
) -> None:
|
||||
store.add([make_node("a1", "1"), make_node("b1", "2")])
|
||||
store.upsert_document("1", [])
|
||||
assert _query(store, [0.0] * DIM, top_k=10).ids == ["b1"]
|
||||
|
||||
def test_upsert_is_atomic_for_concurrent_readers(
|
||||
self,
|
||||
store,
|
||||
store: PaperlessSqliteVecVectorStore,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""A second connection must never observe document 1 half-replaced."""
|
||||
@@ -262,8 +384,52 @@ class TestUpsert:
|
||||
assert ids == ["a3"]
|
||||
|
||||
|
||||
class TestDocumentChunksIndex:
|
||||
"""document_chunks lets delete()/upsert_document() find a document's chunk
|
||||
ids without a vec0 full table scan on document_id -- see
|
||||
PaperlessSqliteVecVectorStore._delete_chunks_by_document_id."""
|
||||
|
||||
def test_add_populates_document_chunks(
|
||||
self,
|
||||
store: PaperlessSqliteVecVectorStore,
|
||||
) -> None:
|
||||
store.add([make_node("a1", "1"), make_node("a2", "1"), make_node("b1", "2")])
|
||||
assert _chunk_index_rows(store) == [
|
||||
("a1", "1"),
|
||||
("a2", "1"),
|
||||
("b1", "2"),
|
||||
]
|
||||
|
||||
def test_delete_clears_document_chunks(
|
||||
self,
|
||||
store: PaperlessSqliteVecVectorStore,
|
||||
) -> None:
|
||||
store.add([make_node("a1", "1"), make_node("a2", "1"), make_node("b1", "2")])
|
||||
store.delete("1")
|
||||
assert _chunk_index_rows(store) == [("b1", "2")]
|
||||
|
||||
def test_upsert_replaces_document_chunks(
|
||||
self,
|
||||
store: PaperlessSqliteVecVectorStore,
|
||||
) -> None:
|
||||
store.add([make_node("a1", "1"), make_node("b1", "2")])
|
||||
store.upsert_document("1", [make_node("a2", "1")])
|
||||
assert _chunk_index_rows(store) == [("a2", "1"), ("b1", "2")]
|
||||
|
||||
def test_drop_table_clears_document_chunks(
|
||||
self,
|
||||
store: PaperlessSqliteVecVectorStore,
|
||||
) -> None:
|
||||
store.add([make_node("a1", "1")])
|
||||
store.drop_table()
|
||||
assert _chunk_index_rows(store) == []
|
||||
|
||||
|
||||
class TestMetadataCoercion:
|
||||
def test_none_metadata_values_become_empty_strings(self, store) -> None:
|
||||
def test_none_metadata_values_become_empty_strings(
|
||||
self,
|
||||
store: PaperlessSqliteVecVectorStore,
|
||||
) -> None:
|
||||
node = make_node("a1", "1")
|
||||
node.metadata["modified"] = None
|
||||
store.add([node]) # must not raise (vec0 rejects NULL metadata)
|
||||
@@ -308,10 +474,16 @@ class TestModelNameTracking:
|
||||
|
||||
|
||||
class TestGetModifiedTimes:
|
||||
def test_empty_store_returns_empty_dict(self, store) -> None:
|
||||
def test_empty_store_returns_empty_dict(
|
||||
self,
|
||||
store: PaperlessSqliteVecVectorStore,
|
||||
) -> None:
|
||||
assert store.get_modified_times() == {}
|
||||
|
||||
def test_returns_one_entry_per_document(self, store) -> None:
|
||||
def test_returns_one_entry_per_document(
|
||||
self,
|
||||
store: PaperlessSqliteVecVectorStore,
|
||||
) -> None:
|
||||
store.add(
|
||||
[
|
||||
make_node("a1", "1", modified="2026-01-01T00:00:00"),
|
||||
@@ -326,7 +498,7 @@ class TestGetModifiedTimes:
|
||||
|
||||
|
||||
class TestCompact:
|
||||
def _bloat_ratio(self, store) -> float:
|
||||
def _bloat_ratio(self, store: PaperlessSqliteVecVectorStore) -> float:
|
||||
live = store.client.execute(
|
||||
"SELECT count(*) FROM documents",
|
||||
).fetchone()[0]
|
||||
@@ -338,19 +510,25 @@ class TestCompact:
|
||||
total = int(row["value"]) if row else live
|
||||
return total / max(live, 1)
|
||||
|
||||
def _churn(self, store, cycles: int) -> None:
|
||||
def _churn(self, store: PaperlessSqliteVecVectorStore, cycles: int) -> None:
|
||||
for i in range(cycles):
|
||||
store.upsert_document(
|
||||
"1",
|
||||
[make_node(f"gen{i}-{j}", "1", seed=float(j)) for j in range(20)],
|
||||
)
|
||||
|
||||
def test_compact_noop_below_threshold(self, store) -> None:
|
||||
def test_compact_noop_below_threshold(
|
||||
self,
|
||||
store: PaperlessSqliteVecVectorStore,
|
||||
) -> None:
|
||||
store.add([make_node("a1", "1")])
|
||||
store.compact()
|
||||
assert _query(store, [0.0] * DIM, top_k=1).ids == ["a1"]
|
||||
|
||||
def test_force_compact_preserves_rows_and_metadata(self, store) -> None:
|
||||
def test_force_compact_preserves_rows_and_metadata(
|
||||
self,
|
||||
store: PaperlessSqliteVecVectorStore,
|
||||
) -> None:
|
||||
store.add([make_node("a1", "1"), make_node("b1", "2", seed=3.0)])
|
||||
self._churn(store, 5)
|
||||
before = {
|
||||
@@ -370,22 +548,44 @@ class TestCompact:
|
||||
store.upsert_document("3", [make_node("c1", "3", seed=100.0)])
|
||||
assert "c1" in _query(store, [100.0] * DIM, top_k=1).ids
|
||||
|
||||
def test_auto_compact_triggers_on_churn(self, store) -> None:
|
||||
def test_auto_compact_triggers_on_churn(
|
||||
self,
|
||||
store: PaperlessSqliteVecVectorStore,
|
||||
) -> None:
|
||||
store.add([make_node(f"s{j}", "1", seed=float(j)) for j in range(20)])
|
||||
self._churn(store, 5)
|
||||
assert self._bloat_ratio(store) > 2
|
||||
store.compact()
|
||||
assert self._bloat_ratio(store) == pytest.approx(1.0)
|
||||
|
||||
def test_compact_on_missing_table_is_noop(self, store) -> None:
|
||||
def test_compact_on_missing_table_is_noop(
|
||||
self,
|
||||
store: PaperlessSqliteVecVectorStore,
|
||||
) -> None:
|
||||
store.compact()
|
||||
store.compact(force=True)
|
||||
|
||||
def test_compact_preserves_document_chunks(
|
||||
self,
|
||||
store: PaperlessSqliteVecVectorStore,
|
||||
) -> None:
|
||||
"""document_chunks must survive the file-swap rebuild, or delete()
|
||||
would silently stop finding chunk ids for anything indexed before a
|
||||
compaction ran."""
|
||||
store.add([make_node("a1", "1"), make_node("b1", "2")])
|
||||
self._churn(store, 5)
|
||||
store.compact(force=True)
|
||||
|
||||
assert _chunk_index_rows(store, "2") == [("b1", "2")]
|
||||
|
||||
store.delete("2")
|
||||
assert "b1" not in _query(store, [0.0] * DIM, top_k=10).ids
|
||||
|
||||
def test_failed_compact_removes_temp_wal_and_shm(
|
||||
self,
|
||||
store,
|
||||
store: PaperlessSqliteVecVectorStore,
|
||||
tmp_path: Path,
|
||||
monkeypatch,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""A compact() that raises mid-rebuild must leave no .compact* files.
|
||||
|
||||
@@ -420,8 +620,8 @@ class TestCompact:
|
||||
|
||||
def test_force_compact_streams_rows_across_batches(
|
||||
self,
|
||||
store,
|
||||
monkeypatch,
|
||||
store: PaperlessSqliteVecVectorStore,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Rebuild must preserve every row when rows span multiple batches.
|
||||
|
||||
@@ -437,11 +637,15 @@ class TestCompact:
|
||||
|
||||
|
||||
class TestDbFile:
|
||||
def test_single_db_file_in_index_dir(self, store, tmp_path: Path) -> None:
|
||||
def test_single_db_file_in_index_dir(
|
||||
self,
|
||||
store: PaperlessSqliteVecVectorStore,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
store.add([make_node("a1", "1")])
|
||||
assert (tmp_path / DB_FILENAME).exists()
|
||||
|
||||
def test_wal_mode_enabled(self, store) -> None:
|
||||
def test_wal_mode_enabled(self, store: PaperlessSqliteVecVectorStore) -> None:
|
||||
assert (
|
||||
store.client.execute("PRAGMA journal_mode").fetchone()[0].lower() == "wal"
|
||||
)
|
||||
@@ -456,193 +660,136 @@ class TestMigrations:
|
||||
).fetchone()
|
||||
return int(row[0]) if row else None
|
||||
|
||||
def test_new_table_records_schema_version(self, store) -> None:
|
||||
def test_new_table_records_schema_version(
|
||||
self,
|
||||
store: PaperlessSqliteVecVectorStore,
|
||||
) -> None:
|
||||
store.add([make_node("a1", "1")])
|
||||
assert self._schema_version(store) == SCHEMA_VERSION
|
||||
|
||||
def test_check_migrations_no_table_returns_false(self, store) -> None:
|
||||
def test_check_migrations_no_table_returns_false(
|
||||
self,
|
||||
store: PaperlessSqliteVecVectorStore,
|
||||
) -> None:
|
||||
assert store.check_and_run_migrations() is False
|
||||
|
||||
def test_check_migrations_current_version_returns_false(self, store) -> None:
|
||||
def test_check_migrations_current_version_returns_false(
|
||||
self,
|
||||
store: PaperlessSqliteVecVectorStore,
|
||||
) -> None:
|
||||
store.add([make_node("a1", "1")])
|
||||
assert store.check_and_run_migrations() is False
|
||||
|
||||
def test_reembed_migration_returns_true(self, store, tmp_path: Path) -> None:
|
||||
def test_reembed_migration_returns_true(
|
||||
self,
|
||||
store: PaperlessSqliteVecVectorStore,
|
||||
) -> None:
|
||||
store.add([make_node("a1", "1")])
|
||||
# store was just created at the real (current) SCHEMA_VERSION, so the
|
||||
# simulated pending migration must target one version past that --
|
||||
# not a hardcoded 1 -> 2, which would collide with the real
|
||||
# document_chunks migration already registered in MIGRATIONS.
|
||||
migration = Migration(
|
||||
from_version=1,
|
||||
to_version=2,
|
||||
from_version=SCHEMA_VERSION,
|
||||
to_version=SCHEMA_VERSION + 1,
|
||||
kind="re-embed",
|
||||
description="test re-embed",
|
||||
)
|
||||
MIGRATIONS.append(migration)
|
||||
try:
|
||||
from paperless_ai import vector_store as vs_mod
|
||||
|
||||
original = vs_mod.SCHEMA_VERSION
|
||||
vs_mod.SCHEMA_VERSION = 2
|
||||
result = store.check_and_run_migrations()
|
||||
finally:
|
||||
MIGRATIONS.remove(migration)
|
||||
vs_mod.SCHEMA_VERSION = original
|
||||
assert result is True
|
||||
with _pending_migrations(migration, schema_version=SCHEMA_VERSION + 1):
|
||||
assert store.check_and_run_migrations() is True
|
||||
|
||||
def test_structural_migration_copies_rows_and_updates_version(
|
||||
self,
|
||||
store,
|
||||
tmp_path: Path,
|
||||
store: PaperlessSqliteVecVectorStore,
|
||||
) -> None:
|
||||
store.add([make_node("a1", "1"), make_node("b1", "2")])
|
||||
|
||||
def apply(
|
||||
src: sqlite3.Connection,
|
||||
dst: sqlite3.Connection,
|
||||
dim: int,
|
||||
) -> None:
|
||||
dst.execute( # nosemgrep
|
||||
f"CREATE VIRTUAL TABLE {DEFAULT_TABLE_NAME} USING vec0("
|
||||
"id TEXT PRIMARY KEY, document_id TEXT, modified TEXT,"
|
||||
f" +node_content TEXT, embedding float[{dim}] distance_metric=cosine"
|
||||
")",
|
||||
)
|
||||
dst.execute(
|
||||
"INSERT INTO index_meta (key, value) VALUES ('dim', ?) "
|
||||
"ON CONFLICT(key) DO UPDATE SET value = excluded.value",
|
||||
(str(dim),),
|
||||
)
|
||||
rows = src.execute(
|
||||
"SELECT id, document_id, modified, node_content, embedding "
|
||||
f"FROM {DEFAULT_TABLE_NAME}",
|
||||
).fetchall()
|
||||
dst.execute("BEGIN IMMEDIATE")
|
||||
dst.executemany(
|
||||
f"INSERT INTO {DEFAULT_TABLE_NAME} "
|
||||
"(id, document_id, modified, node_content, embedding) "
|
||||
"VALUES (?, ?, ?, ?, ?)",
|
||||
[
|
||||
(
|
||||
r["id"],
|
||||
r["document_id"],
|
||||
r["modified"],
|
||||
r["node_content"],
|
||||
bytes(r["embedding"]),
|
||||
)
|
||||
for r in rows
|
||||
],
|
||||
)
|
||||
dst.execute(
|
||||
"INSERT INTO index_meta (key, value) VALUES ('total_inserts', ?) "
|
||||
"ON CONFLICT(key) DO UPDATE SET value = excluded.value",
|
||||
(str(len(rows)),),
|
||||
)
|
||||
dst.execute("COMMIT")
|
||||
|
||||
migration = Migration(
|
||||
from_version=1,
|
||||
to_version=2,
|
||||
from_version=SCHEMA_VERSION,
|
||||
to_version=SCHEMA_VERSION + 1,
|
||||
kind="structural",
|
||||
description="test structural",
|
||||
apply=apply,
|
||||
apply=_copying_apply,
|
||||
)
|
||||
MIGRATIONS.append(migration)
|
||||
try:
|
||||
from paperless_ai import vector_store as vs_mod
|
||||
with _pending_migrations(migration, schema_version=SCHEMA_VERSION + 1):
|
||||
assert store.check_and_run_migrations() is False
|
||||
|
||||
original = vs_mod.SCHEMA_VERSION
|
||||
vs_mod.SCHEMA_VERSION = 2
|
||||
result = store.check_and_run_migrations()
|
||||
finally:
|
||||
MIGRATIONS.remove(migration)
|
||||
vs_mod.SCHEMA_VERSION = original
|
||||
|
||||
assert result is False
|
||||
assert self._schema_version(store) == 2
|
||||
assert self._schema_version(store) == SCHEMA_VERSION + 1
|
||||
ids = {n.node_id for n in store.get_nodes()}
|
||||
assert ids == {"a1", "b1"}
|
||||
|
||||
def test_compact_preserves_schema_version(self, store) -> None:
|
||||
def test_compact_preserves_schema_version(
|
||||
self,
|
||||
store: PaperlessSqliteVecVectorStore,
|
||||
) -> None:
|
||||
store.add([make_node("a1", "1")])
|
||||
assert self._schema_version(store) == SCHEMA_VERSION
|
||||
store.compact(force=True)
|
||||
assert self._schema_version(store) == SCHEMA_VERSION
|
||||
|
||||
def test_stop_at_reembed_boundary(self, store) -> None:
|
||||
# Registry: structural v2, re-embed v3, structural v4.
|
||||
# Only v2 should apply; the re-embed boundary must stop execution
|
||||
# before v4 runs, and the stored version must stay at 2.
|
||||
def test_v1_to_v2_migration_backfills_document_chunks(
|
||||
self,
|
||||
store: PaperlessSqliteVecVectorStore,
|
||||
) -> None:
|
||||
"""The real v1 -> v2 migration (registered in MIGRATIONS) must
|
||||
backfill document_chunks for rows written before it existed, or
|
||||
delete()/upsert_document() would find zero chunk ids for them and
|
||||
leave their vec0 rows orphaned forever."""
|
||||
store.add([make_node("a1", "1"), make_node("a2", "1"), make_node("b1", "2")])
|
||||
# Simulate a pre-migration (schema v1) store: rows exist in the vec0
|
||||
# table, but document_chunks (added by the v1 -> v2 migration) has no
|
||||
# entries for them, mirroring a real on-disk index created before
|
||||
# this migration existed.
|
||||
store.client.execute("DELETE FROM document_chunks")
|
||||
store.client.execute(
|
||||
"UPDATE index_meta SET value = '1' WHERE key = 'schema_version'",
|
||||
)
|
||||
|
||||
assert store.check_and_run_migrations() is False # structural, not re-embed
|
||||
assert self._schema_version(store) == SCHEMA_VERSION
|
||||
|
||||
assert _chunk_index_rows(store) == [
|
||||
("a1", "1"),
|
||||
("a2", "1"),
|
||||
("b1", "2"),
|
||||
]
|
||||
|
||||
# And delete() now actually removes rows for a document that
|
||||
# predates the migration, instead of silently no-op'ing.
|
||||
store.delete("1")
|
||||
assert sorted(_query(store, [0.0] * DIM, top_k=10).ids) == ["b1"]
|
||||
|
||||
def test_stop_at_reembed_boundary(
|
||||
self,
|
||||
store: PaperlessSqliteVecVectorStore,
|
||||
) -> None:
|
||||
# Registry, relative to the store's current (real) SCHEMA_VERSION N:
|
||||
# structural N+1, re-embed N+2, structural N+3. Only N+1 should apply;
|
||||
# the re-embed boundary must stop execution before N+3 runs, and the
|
||||
# stored version must stay at N+1.
|
||||
store.add([make_node("a1", "1"), make_node("b1", "2")])
|
||||
|
||||
def copy_apply(
|
||||
src: sqlite3.Connection,
|
||||
dst: sqlite3.Connection,
|
||||
dim: int,
|
||||
) -> None:
|
||||
dst.execute( # nosemgrep
|
||||
f"CREATE VIRTUAL TABLE {DEFAULT_TABLE_NAME} USING vec0("
|
||||
"id TEXT PRIMARY KEY, document_id TEXT, modified TEXT,"
|
||||
f" +node_content TEXT, embedding float[{dim}] distance_metric=cosine"
|
||||
")",
|
||||
)
|
||||
dst.execute(
|
||||
"INSERT INTO index_meta (key, value) VALUES ('dim', ?) "
|
||||
"ON CONFLICT(key) DO UPDATE SET value = excluded.value",
|
||||
(str(dim),),
|
||||
)
|
||||
rows = src.execute(
|
||||
"SELECT id, document_id, modified, node_content, embedding "
|
||||
f"FROM {DEFAULT_TABLE_NAME}",
|
||||
).fetchall()
|
||||
dst.execute("BEGIN IMMEDIATE")
|
||||
dst.executemany(
|
||||
f"INSERT INTO {DEFAULT_TABLE_NAME} "
|
||||
"(id, document_id, modified, node_content, embedding) "
|
||||
"VALUES (?, ?, ?, ?, ?)",
|
||||
[
|
||||
(
|
||||
r["id"],
|
||||
r["document_id"],
|
||||
r["modified"],
|
||||
r["node_content"],
|
||||
bytes(r["embedding"]),
|
||||
)
|
||||
for r in rows
|
||||
],
|
||||
)
|
||||
dst.execute("COMMIT")
|
||||
|
||||
migrations = [
|
||||
Migration(
|
||||
from_version=1,
|
||||
to_version=2,
|
||||
from_version=SCHEMA_VERSION,
|
||||
to_version=SCHEMA_VERSION + 1,
|
||||
kind="structural",
|
||||
description="v2 structural",
|
||||
apply=copy_apply,
|
||||
description="structural",
|
||||
apply=_copying_apply,
|
||||
),
|
||||
Migration(
|
||||
from_version=2,
|
||||
to_version=3,
|
||||
from_version=SCHEMA_VERSION + 1,
|
||||
to_version=SCHEMA_VERSION + 2,
|
||||
kind="re-embed",
|
||||
description="v3 re-embed boundary",
|
||||
description="re-embed boundary",
|
||||
),
|
||||
Migration(
|
||||
from_version=3,
|
||||
to_version=4,
|
||||
from_version=SCHEMA_VERSION + 2,
|
||||
to_version=SCHEMA_VERSION + 3,
|
||||
kind="structural",
|
||||
description="v4 structural - must not run",
|
||||
apply=copy_apply,
|
||||
description="structural - must not run",
|
||||
apply=_copying_apply,
|
||||
),
|
||||
]
|
||||
MIGRATIONS.extend(migrations)
|
||||
try:
|
||||
from paperless_ai import vector_store as vs_mod
|
||||
with _pending_migrations(*migrations, schema_version=SCHEMA_VERSION + 3):
|
||||
assert store.check_and_run_migrations() is True
|
||||
|
||||
original = vs_mod.SCHEMA_VERSION
|
||||
vs_mod.SCHEMA_VERSION = 4
|
||||
result = store.check_and_run_migrations()
|
||||
finally:
|
||||
for m in migrations:
|
||||
MIGRATIONS.remove(m)
|
||||
vs_mod.SCHEMA_VERSION = original
|
||||
|
||||
assert result is True
|
||||
assert self._schema_version(store) == 2
|
||||
assert self._schema_version(store) == SCHEMA_VERSION + 1
|
||||
|
||||
@@ -31,10 +31,20 @@ logger = logging.getLogger("paperless_ai.vector_store")
|
||||
DB_FILENAME = "llmindex.db"
|
||||
DEFAULT_TABLE_NAME = "documents"
|
||||
|
||||
_INSERT = (
|
||||
"INSERT INTO "
|
||||
+ DEFAULT_TABLE_NAME
|
||||
+ " (id, document_id, modified, node_content, embedding) VALUES (?, ?, ?, ?, ?)"
|
||||
)
|
||||
|
||||
_INSERT_CHUNK_INDEX = (
|
||||
"INSERT INTO document_chunks (chunk_id, document_id) VALUES (?, ?)"
|
||||
)
|
||||
|
||||
# Current schema version. Written to index_meta at table creation and bumped
|
||||
# whenever a Migration is added to MIGRATIONS. check_and_run_migrations() uses
|
||||
# this to decide which migrations to run on an existing store.
|
||||
SCHEMA_VERSION = 1
|
||||
SCHEMA_VERSION = 2
|
||||
|
||||
# compact(): rebuild when the cumulative rowid count exceeds this multiple of
|
||||
# the live row count. DELETEs on vec0 tables never reclaim space (upstream
|
||||
@@ -80,8 +90,10 @@ class Migration:
|
||||
)
|
||||
|
||||
|
||||
# Registry of all schema migrations in order. Empty at v1 -- this is the
|
||||
# baseline. Add entries here (and bump SCHEMA_VERSION) when the schema changes.
|
||||
# Registry of all schema migrations in order. Populated after the class body
|
||||
# below, since v1 -> v2's apply() needs PaperlessSqliteVecVectorStore's own
|
||||
# static methods. Add entries here (and bump SCHEMA_VERSION) when the schema
|
||||
# changes.
|
||||
MIGRATIONS: list[Migration] = []
|
||||
|
||||
|
||||
@@ -93,6 +105,42 @@ def _unpack(blob: bytes) -> list[float]:
|
||||
return list(struct.unpack(f"{len(blob) // 4}f", blob))
|
||||
|
||||
|
||||
def _copy_rows(src_conn: sqlite3.Connection, dst_conn: sqlite3.Connection) -> int:
|
||||
"""Copy every live vec0 row from ``src_conn`` into ``dst_conn``, recording
|
||||
each one in ``dst_conn``'s document_chunks side table. Returns the number
|
||||
of rows copied. The caller owns ``dst_conn``'s transaction.
|
||||
|
||||
Rows are streamed from the source cursor in batches instead of being
|
||||
materialized all at once, so a large index does not cause an OOM during a
|
||||
routine compaction or migration.
|
||||
"""
|
||||
src_cursor = src_conn.execute(
|
||||
"SELECT id, document_id, modified, node_content, embedding FROM "
|
||||
+ DEFAULT_TABLE_NAME,
|
||||
)
|
||||
copied = 0
|
||||
while batch := src_cursor.fetchmany(COMPACT_BATCH_SIZE):
|
||||
dst_conn.executemany(
|
||||
_INSERT,
|
||||
[
|
||||
(
|
||||
r["id"],
|
||||
r["document_id"],
|
||||
r["modified"],
|
||||
r["node_content"],
|
||||
bytes(r["embedding"]),
|
||||
)
|
||||
for r in batch
|
||||
],
|
||||
)
|
||||
dst_conn.executemany(
|
||||
_INSERT_CHUNK_INDEX,
|
||||
[(r["id"], r["document_id"]) for r in batch],
|
||||
)
|
||||
copied += len(batch)
|
||||
return copied
|
||||
|
||||
|
||||
def _build_where(filters: MetadataFilters | None) -> tuple[str, list[str]]:
|
||||
"""Translate the EQ / IN / NE filters we use into a parameterized SQL clause
|
||||
on vec0 metadata columns. Returns ("", []) when there is nothing to filter.
|
||||
@@ -189,6 +237,18 @@ class PaperlessSqliteVecVectorStore(BasePydanticVectorStore):
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS index_meta (key TEXT PRIMARY KEY, value TEXT)",
|
||||
)
|
||||
# vec0 metadata columns only get an efficient lookup path inside a KNN
|
||||
# (MATCH) query; a plain `WHERE document_id = ?` is a full table scan
|
||||
# regardless of index size. This plain, indexed table is how delete()/
|
||||
# upsert_document() find a document's chunk ids without that scan.
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS document_chunks "
|
||||
"(chunk_id TEXT PRIMARY KEY, document_id TEXT NOT NULL)",
|
||||
)
|
||||
conn.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_document_chunks_document_id "
|
||||
"ON document_chunks (document_id)",
|
||||
)
|
||||
return conn
|
||||
|
||||
@property
|
||||
@@ -223,13 +283,17 @@ class PaperlessSqliteVecVectorStore(BasePydanticVectorStore):
|
||||
else:
|
||||
self._conn.execute("COMMIT")
|
||||
|
||||
def _meta_get(self, key: str) -> str | None:
|
||||
row = self._conn.execute(
|
||||
@staticmethod
|
||||
def _meta_get_on(conn: sqlite3.Connection, key: str) -> str | None:
|
||||
row = conn.execute(
|
||||
"SELECT value FROM index_meta WHERE key = ?",
|
||||
(key,),
|
||||
).fetchone()
|
||||
return row["value"] if row else None
|
||||
|
||||
def _meta_get(self, key: str) -> str | None:
|
||||
return self._meta_get_on(self._conn, key)
|
||||
|
||||
@staticmethod
|
||||
def _meta_set_on(conn: sqlite3.Connection, key: str, value: str) -> None:
|
||||
conn.execute(
|
||||
@@ -259,6 +323,7 @@ class PaperlessSqliteVecVectorStore(BasePydanticVectorStore):
|
||||
def drop_table(self) -> None:
|
||||
self._conn.execute("DROP TABLE IF EXISTS " + DEFAULT_TABLE_NAME)
|
||||
self._conn.execute("DELETE FROM index_meta")
|
||||
self._conn.execute("DELETE FROM document_chunks")
|
||||
|
||||
def stored_model_name(self) -> str | None:
|
||||
"""Return the embedding model name recorded at table creation, or None."""
|
||||
@@ -325,11 +390,37 @@ class PaperlessSqliteVecVectorStore(BasePydanticVectorStore):
|
||||
_pack(node.get_embedding()),
|
||||
)
|
||||
|
||||
_INSERT = (
|
||||
"INSERT INTO "
|
||||
+ DEFAULT_TABLE_NAME
|
||||
+ " (id, document_id, modified, node_content, embedding) VALUES (?, ?, ?, ?, ?)"
|
||||
)
|
||||
def _index_chunks(self, rows: list[tuple[str, str, str, str, bytes]]) -> None:
|
||||
"""Record each row's (chunk_id, document_id) in the document_chunks
|
||||
side table, kept in lockstep with every insert into the vec0 table."""
|
||||
self._conn.executemany(
|
||||
_INSERT_CHUNK_INDEX,
|
||||
[(chunk_id, document_id) for chunk_id, document_id, *_ in rows],
|
||||
)
|
||||
|
||||
def _delete_chunks_by_document_id(self, document_id: str) -> None:
|
||||
"""Delete all of a document's chunks via point-deletes on `id`.
|
||||
|
||||
vec0 has no efficient lookup on the document_id metadata column
|
||||
outside a KNN query (see _open_connection), so a plain
|
||||
`DELETE ... WHERE document_id = ?` is a full table scan regardless of
|
||||
index size. Looking the chunk ids up in document_chunks first (a real
|
||||
indexed lookup) and deleting each by its `id` primary key instead
|
||||
turns that scan into a handful of O(1) point deletes.
|
||||
"""
|
||||
doc_id = str(document_id)
|
||||
chunk_rows = self._conn.execute(
|
||||
"SELECT chunk_id FROM document_chunks WHERE document_id = ?",
|
||||
(doc_id,),
|
||||
).fetchall()
|
||||
self._conn.executemany(
|
||||
"DELETE FROM " + DEFAULT_TABLE_NAME + " WHERE id = ?",
|
||||
[(row["chunk_id"],) for row in chunk_rows],
|
||||
)
|
||||
self._conn.execute(
|
||||
"DELETE FROM document_chunks WHERE document_id = ?",
|
||||
(doc_id,),
|
||||
)
|
||||
|
||||
def _increment_total_inserts(self, count: int) -> None:
|
||||
"""Increment the cumulative insert counter stored in index_meta.
|
||||
@@ -348,7 +439,8 @@ class PaperlessSqliteVecVectorStore(BasePydanticVectorStore):
|
||||
rows = [self._row(node) for node in nodes]
|
||||
with self._transaction():
|
||||
self._ensure_table(len(nodes[0].get_embedding()))
|
||||
self._conn.executemany(self._INSERT, rows)
|
||||
self._conn.executemany(_INSERT, rows)
|
||||
self._index_chunks(rows)
|
||||
self._increment_total_inserts(len(rows))
|
||||
return [node.node_id for node in nodes]
|
||||
|
||||
@@ -365,22 +457,17 @@ class PaperlessSqliteVecVectorStore(BasePydanticVectorStore):
|
||||
if nodes:
|
||||
self._ensure_table(len(nodes[0].get_embedding()))
|
||||
if self.table_exists():
|
||||
self._conn.execute(
|
||||
"DELETE FROM " + DEFAULT_TABLE_NAME + " WHERE document_id = ?",
|
||||
(str(document_id),),
|
||||
)
|
||||
self._delete_chunks_by_document_id(document_id)
|
||||
if rows:
|
||||
self._conn.executemany(self._INSERT, rows)
|
||||
self._conn.executemany(_INSERT, rows)
|
||||
self._index_chunks(rows)
|
||||
self._increment_total_inserts(len(rows))
|
||||
return [node.node_id for node in nodes]
|
||||
|
||||
def delete(self, ref_doc_id: str, **delete_kwargs: Any) -> None:
|
||||
if self.table_exists():
|
||||
with self._transaction():
|
||||
self._conn.execute(
|
||||
"DELETE FROM " + DEFAULT_TABLE_NAME + " WHERE document_id = ?",
|
||||
(str(ref_doc_id),),
|
||||
)
|
||||
self._delete_chunks_by_document_id(ref_doc_id)
|
||||
|
||||
def _rows_to_nodes(self, rows: list[sqlite3.Row]) -> list[BaseNode]:
|
||||
nodes: list[BaseNode] = []
|
||||
@@ -508,28 +595,8 @@ class PaperlessSqliteVecVectorStore(BasePydanticVectorStore):
|
||||
value = self._meta_get(key)
|
||||
if value is not None:
|
||||
self._meta_set_on(new_conn, key, value)
|
||||
src_cursor = self._conn.execute(
|
||||
"SELECT id, document_id, modified, node_content, embedding "
|
||||
"FROM " + DEFAULT_TABLE_NAME,
|
||||
)
|
||||
new_conn.execute("BEGIN IMMEDIATE")
|
||||
# Stream rows from the source cursor in batches instead of
|
||||
# materializing the whole table in memory, so a large index does
|
||||
# not cause an OOM during routine maintenance compactions.
|
||||
while batch := src_cursor.fetchmany(COMPACT_BATCH_SIZE):
|
||||
new_conn.executemany(
|
||||
self._INSERT,
|
||||
[
|
||||
(
|
||||
r["id"],
|
||||
r["document_id"],
|
||||
r["modified"],
|
||||
r["node_content"],
|
||||
bytes(r["embedding"]),
|
||||
)
|
||||
for r in batch
|
||||
],
|
||||
)
|
||||
_copy_rows(self._conn, new_conn)
|
||||
# Reset the cumulative counter: after compact, total_inserts == live.
|
||||
self._meta_set_on(new_conn, "total_inserts", str(live))
|
||||
new_conn.execute("COMMIT")
|
||||
@@ -614,3 +681,44 @@ class PaperlessSqliteVecVectorStore(BasePydanticVectorStore):
|
||||
raise
|
||||
new_conn.close()
|
||||
self._swap_in_compact(compact_path, db_path)
|
||||
|
||||
|
||||
def _migrate_v1_to_v2_add_document_chunks(
|
||||
src_conn: sqlite3.Connection,
|
||||
dst_conn: sqlite3.Connection,
|
||||
dim: int,
|
||||
) -> None:
|
||||
"""v1 -> v2: backfill the document_chunks side table.
|
||||
|
||||
document_chunks (see PaperlessSqliteVecVectorStore._open_connection) lets
|
||||
delete()/upsert_document() find a document's chunk ids without a vec0
|
||||
full table scan on the document_id metadata column. Every row written
|
||||
before this migration predates that table, so without backfilling,
|
||||
deleting a pre-migration document would find zero chunk ids and leave its
|
||||
vec0 rows permanently orphaned. Backfilling is a plain row copy into the
|
||||
new-schema file (``_copy_rows``, the same helper compact() uses), which
|
||||
records every copied row in document_chunks as it goes.
|
||||
"""
|
||||
PaperlessSqliteVecVectorStore._create_vec_table(dst_conn, dim)
|
||||
PaperlessSqliteVecVectorStore._meta_set_on(dst_conn, "dim", str(dim))
|
||||
embed_model = PaperlessSqliteVecVectorStore._meta_get_on(src_conn, "embed_model")
|
||||
if embed_model is not None:
|
||||
PaperlessSqliteVecVectorStore._meta_set_on(dst_conn, "embed_model", embed_model)
|
||||
|
||||
dst_conn.execute("BEGIN IMMEDIATE")
|
||||
live = _copy_rows(src_conn, dst_conn)
|
||||
# This migration only ever copies live rows (like compact()), so the
|
||||
# cumulative counter resets to match -- the new file has no bloat yet.
|
||||
PaperlessSqliteVecVectorStore._meta_set_on(dst_conn, "total_inserts", str(live))
|
||||
dst_conn.execute("COMMIT")
|
||||
|
||||
|
||||
MIGRATIONS.append(
|
||||
Migration(
|
||||
from_version=1,
|
||||
to_version=2,
|
||||
kind="structural",
|
||||
description="add document_chunks side table for O(1) per-document deletes",
|
||||
apply=_migrate_v1_to_v2_add_document_chunks,
|
||||
),
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user