mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-06-28 16:24:19 +00:00
Compare commits
35 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 5b8fbdcec7 | |||
| 67972a0740 | |||
| beb048b94a | |||
| a8bfa25efd | |||
| 00baacb26c | |||
| b33d11778a | |||
| 63c19e7f75 | |||
| bf70e597ee | |||
| 78824665aa | |||
| e75946847e | |||
| 7bf2a9ff82 | |||
| bb5d7438b1 | |||
| a009ea1f04 | |||
| 0cdf718d9f | |||
| 262183e848 | |||
| b8f10269a7 | |||
| bcf5d2cffc | |||
| 8bd620d8ab | |||
| ad1b54ce88 | |||
| 20a220ef6c | |||
| e83996135a | |||
| f4fa916579 | |||
| 75f0c4c92e | |||
| a020f64d08 | |||
| b0227dd080 | |||
| 40d927a9ff | |||
| 11fb09e4f4 | |||
| 8ed4bf2011 | |||
| 92c016ce47 | |||
| 82aefe5870 | |||
| 1bef142fd6 | |||
| f56f29111c | |||
| e40e9eb0f9 | |||
| 0ec6610475 | |||
| 05bf334d37 |
@@ -141,13 +141,13 @@ jobs:
|
||||
pytest
|
||||
- name: Upload test results to Codecov
|
||||
if: always()
|
||||
uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0
|
||||
uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1
|
||||
with:
|
||||
flags: backend-python-${{ matrix.python-version }}
|
||||
files: junit.xml
|
||||
report_type: test_results
|
||||
- name: Upload coverage to Codecov
|
||||
uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0
|
||||
uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1
|
||||
with:
|
||||
flags: backend-python-${{ matrix.python-version }}
|
||||
files: coverage.xml
|
||||
|
||||
@@ -106,9 +106,9 @@ jobs:
|
||||
echo "repository=${repo_name}"
|
||||
echo "name=${repo_name}" >> $GITHUB_OUTPUT
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
@@ -121,7 +121,7 @@ jobs:
|
||||
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
|
||||
- name: Docker metadata
|
||||
id: docker-meta
|
||||
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0
|
||||
uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6.1.0
|
||||
with:
|
||||
images: |
|
||||
${{ env.REGISTRY }}/${{ steps.repo.outputs.name }}
|
||||
@@ -132,7 +132,7 @@ jobs:
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
- name: Build and push by digest
|
||||
id: build
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile
|
||||
@@ -182,29 +182,29 @@ jobs:
|
||||
echo "Downloaded digests:"
|
||||
ls -la /tmp/digests/
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Login to Docker Hub
|
||||
if: needs.build-arch.outputs.push-external == 'true'
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
- name: Login to Quay.io
|
||||
if: needs.build-arch.outputs.push-external == 'true'
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
|
||||
with:
|
||||
registry: quay.io
|
||||
username: ${{ secrets.QUAY_USERNAME }}
|
||||
password: ${{ secrets.QUAY_ROBOT_TOKEN }}
|
||||
- name: Docker metadata
|
||||
id: docker-meta
|
||||
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0
|
||||
uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6.1.0
|
||||
with:
|
||||
images: |
|
||||
${{ env.REGISTRY }}/${{ needs.build-arch.outputs.repository }}
|
||||
|
||||
@@ -81,7 +81,7 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Install pnpm
|
||||
uses: pnpm/action-setup@903f9c1a6ebcba6cf41d87230be49611ac97822e # v6.0.3
|
||||
uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8
|
||||
with:
|
||||
version: 10
|
||||
- name: Use Node.js 24
|
||||
@@ -113,7 +113,7 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Install pnpm
|
||||
uses: pnpm/action-setup@903f9c1a6ebcba6cf41d87230be49611ac97822e # v6.0.3
|
||||
uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8
|
||||
with:
|
||||
version: 10
|
||||
- name: Use Node.js 24
|
||||
@@ -152,7 +152,7 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Install pnpm
|
||||
uses: pnpm/action-setup@903f9c1a6ebcba6cf41d87230be49611ac97822e # v6.0.3
|
||||
uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8
|
||||
with:
|
||||
version: 10
|
||||
- name: Use Node.js 24
|
||||
@@ -174,13 +174,13 @@ jobs:
|
||||
run: cd src-ui && pnpm run test --max-workers=2 --shard=${{ matrix.shard-index }}/${{ matrix.shard-count }}
|
||||
- name: Upload test results to Codecov
|
||||
if: always()
|
||||
uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0
|
||||
uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1
|
||||
with:
|
||||
flags: frontend-node-${{ matrix.node-version }}
|
||||
directory: src-ui/
|
||||
report_type: test_results
|
||||
- name: Upload coverage to Codecov
|
||||
uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0
|
||||
uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1
|
||||
with:
|
||||
flags: frontend-node-${{ matrix.node-version }}
|
||||
directory: src-ui/coverage/
|
||||
@@ -207,7 +207,7 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Install pnpm
|
||||
uses: pnpm/action-setup@903f9c1a6ebcba6cf41d87230be49611ac97822e # v6.0.3
|
||||
uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8
|
||||
with:
|
||||
version: 10
|
||||
- name: Use Node.js 24
|
||||
@@ -244,7 +244,7 @@ jobs:
|
||||
fetch-depth: 2
|
||||
persist-credentials: false
|
||||
- name: Install pnpm
|
||||
uses: pnpm/action-setup@903f9c1a6ebcba6cf41d87230be49611ac97822e # v6.0.3
|
||||
uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8
|
||||
with:
|
||||
version: 10
|
||||
- name: Use Node.js 24
|
||||
|
||||
@@ -25,4 +25,4 @@ jobs:
|
||||
with:
|
||||
python-version: "3.14"
|
||||
- name: Run prek
|
||||
uses: j178/prek-action@cbc2f23eb5539cf20d82d1aabd0d0ecbcc56f4e3 # v2.0.2
|
||||
uses: j178/prek-action@bdca6f102f98e2b4c7029491a53dfd366469e33d # v2.0.4
|
||||
|
||||
@@ -39,7 +39,7 @@ jobs:
|
||||
persist-credentials: false
|
||||
# ---- Frontend Build ----
|
||||
- name: Install pnpm
|
||||
uses: pnpm/action-setup@903f9c1a6ebcba6cf41d87230be49611ac97822e # v6.0.3
|
||||
uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8
|
||||
with:
|
||||
version: 10
|
||||
- name: Use Node.js 24
|
||||
@@ -170,7 +170,7 @@ jobs:
|
||||
fi
|
||||
- name: Create release and changelog
|
||||
id: create-release
|
||||
uses: release-drafter/release-drafter@5de93583980a40bd78603b6dfdcda5b4df377b32 # v7.2.0
|
||||
uses: release-drafter/release-drafter@693d20e7c1ce1a81d3a41962f85914253b518449 # v7.3.1
|
||||
with:
|
||||
name: Paperless-ngx ${{ steps.get-version.outputs.version }}
|
||||
tag: ${{ steps.get-version.outputs.version }}
|
||||
|
||||
@@ -26,7 +26,7 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Run zizmor
|
||||
uses: zizmorcore/zizmor-action@b1d7e1fb5de872772f31590499237e7cce841e8e # v0.5.3
|
||||
uses: zizmorcore/zizmor-action@5f14fd08f7cf1cb1609c1e344975f152c7ee938d # v0.5.6
|
||||
semgrep:
|
||||
name: Semgrep CE
|
||||
runs-on: ubuntu-24.04
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
- name: Run Semgrep
|
||||
run: semgrep scan --config auto --sarif-output results.sarif
|
||||
- name: Upload results to GitHub code scanning
|
||||
uses: github/codeql-action/upload-sarif@95e58e9a2cdfd71adc6e0353d5c52f41a045d225 # v4.35.2
|
||||
uses: github/codeql-action/upload-sarif@7211b7c8077ea37d8641b6271f6a365a22a5fbfa # v4.36.0
|
||||
if: always()
|
||||
with:
|
||||
sarif_file: results.sarif
|
||||
|
||||
@@ -39,7 +39,7 @@ jobs:
|
||||
persist-credentials: false
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@95e58e9a2cdfd71adc6e0353d5c52f41a045d225 # v4.35.2
|
||||
uses: github/codeql-action/init@7211b7c8077ea37d8641b6271f6a365a22a5fbfa # v4.36.0
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
# If you wish to specify custom queries, you can do so here or in a config file.
|
||||
@@ -47,4 +47,4 @@ jobs:
|
||||
# Prefix the list here with "+" to use these queries and those in the config file.
|
||||
# queries: ./path/to/local/query, your-org/your-repo/queries@main
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@95e58e9a2cdfd71adc6e0353d5c52f41a045d225 # v4.35.2
|
||||
uses: github/codeql-action/analyze@7211b7c8077ea37d8641b6271f6a365a22a5fbfa # v4.36.0
|
||||
|
||||
@@ -31,7 +31,7 @@ jobs:
|
||||
steps:
|
||||
- name: Label PR by file path or branch name
|
||||
# see .github/labeler.yml for the labeler config
|
||||
uses: actions/labeler@634933edcd8ababfe52f92936142cc22ac488b1b # v6.0.1
|
||||
uses: actions/labeler@f27b608878404679385c85cfa523b85ccb86e213 # v6.1.0
|
||||
with:
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Label by size
|
||||
|
||||
@@ -19,6 +19,6 @@ jobs:
|
||||
if: github.event_name == 'pull_request_target' && (github.event.action == 'opened' || github.event.action == 'reopened') && github.event.pull_request.user.login != 'dependabot'
|
||||
steps:
|
||||
- name: Label PR with release-drafter
|
||||
uses: release-drafter/release-drafter@5de93583980a40bd78603b6dfdcda5b4df377b32 # v7.2.0
|
||||
uses: release-drafter/release-drafter@693d20e7c1ce1a81d3a41962f85914253b518449 # v7.3.1
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
@@ -14,7 +14,7 @@ jobs:
|
||||
issues: write
|
||||
pull-requests: write
|
||||
steps:
|
||||
- uses: actions/stale@b5d41d4e1d5dceea10e7104786b73624c18a190f # v10.2.0
|
||||
- uses: actions/stale@eb5cf3af3ac0a1aa4c9c45633dd1ae542a27a899 # v10.3.0
|
||||
with:
|
||||
days-before-stale: 7
|
||||
days-before-close: 14
|
||||
|
||||
@@ -43,7 +43,7 @@ jobs:
|
||||
PAPERLESS_SECRET_KEY: "ci-translate-not-a-real-secret"
|
||||
run: cd src/ && uv run manage.py makemessages -l en_US -i "samples*"
|
||||
- name: Install pnpm
|
||||
uses: pnpm/action-setup@903f9c1a6ebcba6cf41d87230be49611ac97822e # v6.0.3
|
||||
uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8
|
||||
with:
|
||||
version: 10
|
||||
- name: Use Node.js 24
|
||||
|
||||
@@ -38,7 +38,7 @@ repos:
|
||||
- json
|
||||
# See https://github.com/prettier/prettier/issues/15742 for the fork reason
|
||||
- repo: https://github.com/rbubley/mirrors-prettier
|
||||
rev: 'v3.8.3'
|
||||
rev: 'v3.8.4'
|
||||
hooks:
|
||||
- id: prettier
|
||||
types_or:
|
||||
@@ -50,14 +50,15 @@ repos:
|
||||
- 'prettier-plugin-organize-imports@4.3.0'
|
||||
# Python hooks
|
||||
- repo: https://github.com/astral-sh/ruff-pre-commit
|
||||
rev: v0.15.12
|
||||
rev: v0.15.17
|
||||
hooks:
|
||||
- id: ruff-check
|
||||
- id: ruff-format
|
||||
- repo: https://github.com/tox-dev/pyproject-fmt
|
||||
rev: "v2.21.1"
|
||||
rev: "v2.24.1"
|
||||
hooks:
|
||||
- id: pyproject-fmt
|
||||
additional_dependencies: [tomli]
|
||||
# Dockerfile hooks
|
||||
- repo: https://github.com/AleksaC/hadolint-py
|
||||
rev: v2.14.0
|
||||
|
||||
+1
-1
@@ -30,7 +30,7 @@ RUN set -eux \
|
||||
# Purpose: Installs s6-overlay and rootfs
|
||||
# Comments:
|
||||
# - Don't leave anything extra in here either
|
||||
FROM ghcr.io/astral-sh/uv:0.11.6-python3.12-trixie-slim AS s6-overlay-base
|
||||
FROM ghcr.io/astral-sh/uv:0.11.19-python3.12-trixie-slim AS s6-overlay-base
|
||||
|
||||
WORKDIR /usr/src/s6
|
||||
|
||||
|
||||
@@ -63,6 +63,7 @@ The following are not generally considered vulnerabilities unless accompanied by
|
||||
- optional webhook, mail, AI, OCR, or integration behavior described without a product-level vulnerability
|
||||
- missing limits or hardening settings presented without concrete impact
|
||||
- generic AI or static-analysis output that is not confirmed against the current codebase and a real deployment scenario
|
||||
- the ability to attach objects that a user cannot access to a document by ID is an intentional design choice, and not considered a vulnerability
|
||||
|
||||
## Transparency
|
||||
|
||||
|
||||
@@ -30,7 +30,7 @@
|
||||
# documentation.
|
||||
services:
|
||||
broker:
|
||||
image: docker.io/library/redis:8
|
||||
image: docker.io/valkey/valkey:9-alpine
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- redisdata:/data
|
||||
|
||||
@@ -26,7 +26,7 @@
|
||||
# documentation.
|
||||
services:
|
||||
broker:
|
||||
image: docker.io/library/redis:8
|
||||
image: docker.io/valkey/valkey:9-alpine
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- redisdata:/data
|
||||
|
||||
@@ -27,7 +27,7 @@
|
||||
# documentation.
|
||||
services:
|
||||
broker:
|
||||
image: docker.io/library/redis:8
|
||||
image: docker.io/valkey/valkey:9-alpine
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- redisdata:/data
|
||||
|
||||
@@ -30,7 +30,7 @@
|
||||
# documentation.
|
||||
services:
|
||||
broker:
|
||||
image: docker.io/library/redis:8
|
||||
image: docker.io/valkey/valkey:9-alpine
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- redisdata:/data
|
||||
|
||||
@@ -26,7 +26,7 @@
|
||||
# documentation.
|
||||
services:
|
||||
broker:
|
||||
image: docker.io/library/redis:8
|
||||
image: docker.io/valkey/valkey:9-alpine
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- redisdata:/data
|
||||
|
||||
@@ -30,7 +30,7 @@
|
||||
# documentation.
|
||||
services:
|
||||
broker:
|
||||
image: docker.io/library/redis:8
|
||||
image: docker.io/valkey/valkey:9-alpine
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- redisdata:/data
|
||||
|
||||
@@ -23,7 +23,7 @@
|
||||
# documentation.
|
||||
services:
|
||||
broker:
|
||||
image: docker.io/library/redis:8
|
||||
image: docker.io/valkey/valkey:9-alpine
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- redisdata:/data
|
||||
|
||||
@@ -65,6 +65,11 @@ copies you created in the steps above.
|
||||
|
||||
Please review the [migration instructions](migration-v3.md) before upgrading Paperless-ngx to v3.0, it includes some breaking changes that require manual intervention before upgrading.
|
||||
|
||||
!!! note
|
||||
|
||||
Upgrading to v3 clears the existing task history; previously completed, failed, or
|
||||
acknowledged tasks will no longer appear in the task list afterward. No action is required.
|
||||
|
||||
### Docker Route {#docker-updating}
|
||||
|
||||
If a new release of paperless-ngx is available, upgrading depends on how
|
||||
@@ -500,6 +505,33 @@ task scheduler.
|
||||
python3 manage.py document_index reindex --if-needed
|
||||
```
|
||||
|
||||
### Managing the LLM (AI) index {#llm-index}
|
||||
|
||||
When the [AI features](advanced_usage.md#ai-features) are enabled with an embedding
|
||||
backend, Paperless-ngx maintains a vector index of your documents used for
|
||||
Retrieval-Augmented Generation (RAG), similar-document retrieval, and document chat. The
|
||||
index is updated automatically on the schedule set by
|
||||
[`PAPERLESS_LLM_INDEX_TASK_CRON`](configuration.md#PAPERLESS_LLM_INDEX_TASK_CRON), but you
|
||||
can manage it manually:
|
||||
|
||||
```
|
||||
document_llmindex {rebuild,update,compact}
|
||||
```
|
||||
|
||||
Specify `rebuild` to build the index from scratch from all documents in the database. Use
|
||||
this the first time you enable the feature, or after changing the embedding backend or
|
||||
model.
|
||||
|
||||
Specify `update` to incrementally index new and changed documents. This is what the
|
||||
scheduled task runs.
|
||||
|
||||
Specify `compact` to reclaim space and optimize the on-disk vector store.
|
||||
|
||||
!!! note
|
||||
|
||||
These commands have no effect unless AI is enabled and an embedding backend is
|
||||
configured.
|
||||
|
||||
### Clearing the database read cache
|
||||
|
||||
If the database read cache is enabled, **you must run this command** after making any changes to the database outside the application context.
|
||||
|
||||
+83
-2
@@ -97,6 +97,85 @@ when using this feature:
|
||||
of these correspondents to ANY new document, if both are set to
|
||||
automatic matching.
|
||||
|
||||
## AI features {#ai-features}
|
||||
|
||||
Paperless-ngx includes a set of optional features backed by a large language model
|
||||
(LLM): AI-assisted suggestions, similar-document retrieval, and a document chat. They
|
||||
are **off by default** and never replace the built-in, non-LLM
|
||||
[matching and suggestions](#matching).
|
||||
|
||||
!!! warning
|
||||
|
||||
Enabling these features sends document content (and metadata) to the LLM backend you
|
||||
configure. If that backend is a remote/hosted provider, your documents leave your
|
||||
server and may incur usage charges. Consider the privacy implications before enabling,
|
||||
and prefer a local backend (Ollama, or a self-hosted OpenAI-compatible gateway) if that
|
||||
matters to you.
|
||||
|
||||
All AI settings can be supplied as `PAPERLESS_AI_*` environment variables (see
|
||||
[configuration](configuration.md#ai)) or set in the admin under
|
||||
**Settings → Application Configuration**; the database value takes precedence over the
|
||||
environment.
|
||||
|
||||
### Enabling the AI features
|
||||
|
||||
At a minimum you need to enable AI and choose an LLM backend:
|
||||
|
||||
- [`PAPERLESS_AI_ENABLED`](configuration.md#PAPERLESS_AI_ENABLED) — master switch.
|
||||
- [`PAPERLESS_AI_LLM_BACKEND`](configuration.md#PAPERLESS_AI_LLM_BACKEND) — `ollama`
|
||||
(runs locally) or `openai-like` (OpenAI itself or any OpenAI-compatible API).
|
||||
- [`PAPERLESS_AI_LLM_MODEL`](configuration.md#PAPERLESS_AI_LLM_MODEL), and for
|
||||
`openai-like` usually [`PAPERLESS_AI_LLM_API_KEY`](configuration.md#PAPERLESS_AI_LLM_API_KEY)
|
||||
and/or [`PAPERLESS_AI_LLM_ENDPOINT`](configuration.md#PAPERLESS_AI_LLM_ENDPOINT). Ollama
|
||||
requires `PAPERLESS_AI_LLM_ENDPOINT` pointing at your Ollama server.
|
||||
|
||||
### AI-assisted suggestions
|
||||
|
||||
With AI enabled, Paperless-ngx can suggest a title, tags, correspondent, document type,
|
||||
storage path and dates by sending the document to the LLM. This is **opt-in per request**
|
||||
and surfaces through the "Suggest" control on the document detail page, alongside the
|
||||
classic classifier-based suggestions — it does not disable them. Suggestion output
|
||||
language can be steered with
|
||||
[`PAPERLESS_AI_LLM_OUTPUT_LANGUAGE`](configuration.md#PAPERLESS_AI_LLM_OUTPUT_LANGUAGE)
|
||||
(otherwise it follows the user's UI language).
|
||||
|
||||
### The LLM index (RAG) and similar documents
|
||||
|
||||
Setting an embedding backend turns on the **LLM index**, a vector index of your documents
|
||||
that enables Retrieval-Augmented Generation (RAG). When enabled, suggestions are grounded
|
||||
in similar existing documents, and the document chat can retrieve relevant context.
|
||||
|
||||
Enable it by setting
|
||||
[`PAPERLESS_AI_LLM_EMBEDDING_BACKEND`](configuration.md#PAPERLESS_AI_LLM_EMBEDDING_BACKEND)
|
||||
(`huggingface` for fully-local embeddings, or `ollama` / `openai-like`). The index is only
|
||||
built when AI is enabled **and** an embedding backend is set.
|
||||
|
||||
The index is updated automatically on a schedule controlled by
|
||||
[`PAPERLESS_LLM_INDEX_TASK_CRON`](configuration.md#PAPERLESS_LLM_INDEX_TASK_CRON) (daily by
|
||||
default), and can be rebuilt or compacted manually — see
|
||||
[Managing the LLM index](administration.md#llm-index).
|
||||
|
||||
!!! note
|
||||
|
||||
Local embeddings via `huggingface` download the embedding model on first use into the
|
||||
Paperless data directory. The first run therefore needs network access and some disk
|
||||
space.
|
||||
|
||||
### Document chat
|
||||
|
||||
When the LLM index is enabled, the chat control in the top app toolbar answers questions
|
||||
about your documents. It operates over a single document or across multiple documents
|
||||
depending on the current view, and its answers include links to the source documents it
|
||||
drew from.
|
||||
|
||||
### AI Security notes
|
||||
|
||||
- Document content is passed to the LLM as **untrusted data**.
|
||||
- By default Paperless-ngx allows AI endpoints that resolve to private/loopback addresses
|
||||
(for local backends). Set
|
||||
[`PAPERLESS_AI_LLM_ALLOW_INTERNAL_ENDPOINTS`](configuration.md#PAPERLESS_AI_LLM_ALLOW_INTERNAL_ENDPOINTS)
|
||||
to `false` to block them.
|
||||
|
||||
## Hooking into the consumption process {#consume-hooks}
|
||||
|
||||
Sometimes you may want to do something arbitrary whenever a document is
|
||||
@@ -846,7 +925,7 @@ Paperless is able to utilize barcodes for automatically performing some tasks. B
|
||||
|
||||
At this time, the library utilized for detection of barcodes supports the following types:
|
||||
|
||||
- AN-13/UPC-A
|
||||
- EAN-13/UPC-A
|
||||
- UPC-E
|
||||
- EAN-8
|
||||
- Code 128
|
||||
@@ -855,7 +934,9 @@ At this time, the library utilized for detection of barcodes supports the follow
|
||||
- Codabar
|
||||
- Interleaved 2 of 5
|
||||
- QR Code
|
||||
- SQ Code
|
||||
- Data Matrix
|
||||
- Aztec
|
||||
- PDF417
|
||||
|
||||
For usage in Paperless, the type of barcode does not matter, only the contents of it.
|
||||
|
||||
|
||||
@@ -227,6 +227,7 @@ Version-aware endpoints:
|
||||
- `PATCH /api/documents/{id}/`: content updates target the selected version (`?version={version_id}`) or latest version by default; non-content metadata updates target the root document.
|
||||
- `GET /api/documents/{id}/download/`, `GET /api/documents/{id}/preview/`, `GET /api/documents/{id}/thumb/`, `GET /api/documents/{id}/metadata/`: accept `?version={version_id}`.
|
||||
- `POST /api/documents/{id}/update_version/`: uploads a new version using multipart form field `document` and optional `version_label`.
|
||||
- `PATCH /api/documents/{id}/versions/{version_id}/`: updates the `version_label` of a specific version.
|
||||
- `DELETE /api/documents/{root_id}/versions/{version_id}/`: deletes a non-root version.
|
||||
|
||||
## Permissions
|
||||
@@ -445,3 +446,9 @@ Initial API version.
|
||||
large lists of object IDs for operations affecting many objects.
|
||||
- The legacy `title_content` document search parameter is deprecated and will be removed in a future version.
|
||||
Clients should use `text` for simple title-and-content search and `title_search` for title-only search.
|
||||
- The task tracking system was redesigned. The tasks list (`/api/tasks/`) is now paginated, and the
|
||||
task object exposes `task_type` (formerly `task_name`) and `trigger_source` (formerly `type`). New
|
||||
read-only endpoints `/api/tasks/summary/`, `/api/tasks/status_counts/`, and `/api/tasks/active/`
|
||||
provide aggregate views, and `POST /api/tasks/run/` lets privileged users dispatch supported tasks.
|
||||
API v9 continues to serve the unpaginated list with the legacy field names until support for v9 is
|
||||
dropped.
|
||||
|
||||
+28
-17
@@ -22,7 +22,11 @@ or applicable default will be utilized instead.
|
||||
|
||||
## Required services
|
||||
|
||||
### Redis Broker
|
||||
### Message Broker
|
||||
|
||||
Paperless-ngx uses a Redis-compatible message broker. Any broker that
|
||||
speaks the Redis protocol works here, including [Valkey](https://valkey.io/)
|
||||
(the default in the bundled Docker Compose files) and Redis itself.
|
||||
|
||||
#### [`PAPERLESS_REDIS=<url>`](#PAPERLESS_REDIS) {#PAPERLESS_REDIS}
|
||||
|
||||
@@ -30,21 +34,21 @@ or applicable default will be utilized instead.
|
||||
fetching, index optimization and for training the automatic document
|
||||
matcher.
|
||||
|
||||
- If your Redis server needs login credentials PAPERLESS_REDIS =
|
||||
- If your broker needs login credentials PAPERLESS_REDIS =
|
||||
`redis://<username>:<password>@<host>:<port>`
|
||||
- With the requirepass option PAPERLESS_REDIS =
|
||||
`redis://:<password>@<host>:<port>`
|
||||
- To include the redis database index PAPERLESS_REDIS =
|
||||
- To include the database index PAPERLESS_REDIS =
|
||||
`redis://<username>:<password>@<host>:<port>/<DBIndex>`
|
||||
|
||||
[More information on securing your Redis
|
||||
Instance](https://redis.io/docs/latest/operate/oss_and_stack/management/security).
|
||||
[More information on securing your broker
|
||||
instance](https://valkey.io/topics/security/).
|
||||
|
||||
Defaults to `redis://localhost:6379`.
|
||||
|
||||
#### [`PAPERLESS_REDIS_PREFIX=<prefix>`](#PAPERLESS_REDIS_PREFIX) {#PAPERLESS_REDIS_PREFIX}
|
||||
|
||||
: Prefix to be used in Redis for keys and channels. Useful for sharing one Redis server among multiple Paperless instances.
|
||||
: Prefix to be used in the broker for keys and channels. Useful for sharing one broker among multiple Paperless instances.
|
||||
|
||||
Defaults to no prefix.
|
||||
|
||||
@@ -58,14 +62,14 @@ and the relevant connection variables.
|
||||
#### [`PAPERLESS_DBENGINE=<engine>`](#PAPERLESS_DBENGINE) {#PAPERLESS_DBENGINE}
|
||||
|
||||
: Specifies the database engine to use. Accepted values are `sqlite`, `postgresql`,
|
||||
and `mariadb`.
|
||||
|
||||
Defaults to `sqlite` if not set.
|
||||
and `mariadb`. PostgreSQL and MariaDB users must set this explicitly.
|
||||
|
||||
PostgreSQL and MariaDB both require [`PAPERLESS_DBHOST`](#PAPERLESS_DBHOST) to be
|
||||
set. SQLite does not use any other connection variables; the database file is always
|
||||
located at `<PAPERLESS_DATA_DIR>/db.sqlite3`.
|
||||
|
||||
Defaults to `sqlite`.
|
||||
|
||||
!!! warning
|
||||
Using MariaDB comes with some caveats.
|
||||
See [MySQL Caveats](advanced_usage.md#mysql-caveats).
|
||||
@@ -238,7 +242,7 @@ dictionaries; for example, `pool.max_size=20` sets
|
||||
|
||||
#### [`PAPERLESS_DB_READ_CACHE_ENABLED=<bool>`](#PAPERLESS_DB_READ_CACHE_ENABLED) {#PAPERLESS_DB_READ_CACHE_ENABLED}
|
||||
|
||||
: Caches the database read query results into Redis. This can significantly improve application response times by caching database queries, at the cost of slightly increased memory usage.
|
||||
: Caches the database read query results into the broker. This can significantly improve application response times by caching database queries, at the cost of slightly increased memory usage.
|
||||
|
||||
Defaults to `false`.
|
||||
|
||||
@@ -258,18 +262,18 @@ dictionaries; for example, `pool.max_size=20` sets
|
||||
|
||||
A high TTL increases memory usage over time. Memory may be used until end of TTL, even if the cache is invalidated with the `invalidate_cachalot` command.
|
||||
|
||||
In case of an out-of-memory (OOM) situation, Redis may stop accepting new data — including cache entries, scheduled tasks, and documents to consume.
|
||||
If your system has limited RAM, consider configuring a dedicated Redis instance for the read cache, with a memory limit and the eviction policy set to `allkeys-lru`.
|
||||
For more details, refer to the [Redis eviction policy documentation](https://redis.io/docs/latest/develop/reference/eviction/), and see the `PAPERLESS_READ_CACHE_REDIS_URL` setting to specify a separate Redis broker.
|
||||
In case of an out-of-memory (OOM) situation, the broker may stop accepting new data — including cache entries, scheduled tasks, and documents to consume.
|
||||
If your system has limited RAM, consider configuring a dedicated broker instance for the read cache, with a memory limit and the eviction policy set to `allkeys-lru`.
|
||||
For more details, refer to the [Redis eviction policy documentation](https://redis.io/docs/latest/develop/reference/eviction/), and see the `PAPERLESS_READ_CACHE_REDIS_URL` setting to specify a separate broker.
|
||||
|
||||
#### [`PAPERLESS_READ_CACHE_REDIS_URL=<url>`](#PAPERLESS_READ_CACHE_REDIS_URL) {#PAPERLESS_READ_CACHE_REDIS_URL}
|
||||
|
||||
: Defines the Redis instance used for the read cache.
|
||||
: Defines the broker instance used for the read cache.
|
||||
|
||||
Defaults to `None`.
|
||||
|
||||
!!! Note
|
||||
If this value is not set, the same Redis instance used for scheduled tasks will be used for caching as well.
|
||||
If this value is not set, the same broker instance used for scheduled tasks will be used for caching as well.
|
||||
|
||||
## Optional Services
|
||||
|
||||
@@ -888,7 +892,7 @@ modes are available:
|
||||
|
||||
The default is `auto`.
|
||||
|
||||
For the `skip`, `redo`, and `force` modes, read more about OCR
|
||||
For the `redo` and `force` modes, read more about OCR
|
||||
behaviour in the [OCRmyPDF
|
||||
documentation](https://ocrmypdf.readthedocs.io/en/latest/advanced.html#when-ocr-is-skipped).
|
||||
|
||||
@@ -2068,6 +2072,13 @@ context by default.
|
||||
|
||||
Defaults to 8192.
|
||||
|
||||
#### [`PAPERLESS_AI_LLM_REQUEST_TIMEOUT=<int>`](#PAPERLESS_AI_LLM_REQUEST_TIMEOUT) {#PAPERLESS_AI_LLM_REQUEST_TIMEOUT}
|
||||
|
||||
: The timeout, in seconds, for requests to the configured AI backend. Increase this when using
|
||||
local or slow inference servers that need more time to generate responses.
|
||||
|
||||
Defaults to 120.
|
||||
|
||||
#### [`PAPERLESS_AI_LLM_BACKEND=<str>`](#PAPERLESS_AI_LLM_BACKEND) {#PAPERLESS_AI_LLM_BACKEND}
|
||||
|
||||
: The AI backend to use. This can be either "openai-like" or "ollama". If set to "ollama", the AI
|
||||
@@ -2120,7 +2131,7 @@ used with the OpenAI-compatible backend to target a custom provider or local gat
|
||||
|
||||
Defaults to true, which allows internal endpoints.
|
||||
|
||||
#### [`PAPERLESS_AI_LLM_INDEX_TASK_CRON=<cron expression>`](#PAPERLESS_AI_LLM_INDEX_TASK_CRON) {#PAPERLESS_AI_LLM_INDEX_TASK_CRON}
|
||||
#### [`PAPERLESS_LLM_INDEX_TASK_CRON=<cron expression>`](#PAPERLESS_LLM_INDEX_TASK_CRON) {#PAPERLESS_LLM_INDEX_TASK_CRON}
|
||||
|
||||
: Configures the schedule to update the AI embeddings of text content and metadata for all documents. Only performed if
|
||||
AI is enabled and the LLM embedding backend is set.
|
||||
|
||||
+13
-12
@@ -94,16 +94,16 @@ first-time setup.
|
||||
```
|
||||
|
||||
7. You can now either ...
|
||||
- install Redis or
|
||||
- install a Redis-compatible broker (e.g. Valkey or Redis) or
|
||||
|
||||
- use the included `scripts/start_services.sh` to use Docker to fire
|
||||
up a Redis instance (and some other services such as Tika,
|
||||
up a broker instance (and some other services such as Tika,
|
||||
Gotenberg and a database server) or
|
||||
|
||||
- spin up a bare Redis container
|
||||
- spin up a bare broker container
|
||||
|
||||
```bash
|
||||
docker run -d -p 6379:6379 --restart unless-stopped redis:latest
|
||||
docker run -d -p 6379:6379 --restart unless-stopped docker.io/valkey/valkey:9-alpine
|
||||
```
|
||||
|
||||
8. Continue with either back-end or front-end development – or both :-).
|
||||
@@ -132,7 +132,7 @@ uv run manage.py runserver & \
|
||||
```
|
||||
|
||||
You might need the front end to test your back end code.
|
||||
This assumes that you have AngularJS installed on your system.
|
||||
This assumes that you have Angular installed on your system.
|
||||
Go to the [Front end development](#front-end-development) section for further details.
|
||||
To build the front end once use this command:
|
||||
|
||||
@@ -174,7 +174,7 @@ To add a new development package `uv add --dev <package>`
|
||||
|
||||
## Front end development
|
||||
|
||||
The front end is built using AngularJS. In order to get started, you need Node.js (version 24+) and
|
||||
The front end is built using Angular. In order to get started, you need Node.js (version 24+) and
|
||||
`pnpm`.
|
||||
|
||||
!!! note
|
||||
@@ -248,12 +248,12 @@ that authentication is working.
|
||||
## Localization
|
||||
|
||||
Paperless-ngx is available in many different languages. Since Paperless-ngx
|
||||
consists both of a Django application and an AngularJS front end, both
|
||||
consists both of a Django application and an Angular front end, both
|
||||
these parts have to be translated separately.
|
||||
|
||||
### Front end localization
|
||||
|
||||
- The AngularJS front end does localization according to the [Angular
|
||||
- The Angular front end does localization according to the [Angular
|
||||
documentation](https://angular.io/guide/i18n).
|
||||
- The source language of the project is "en_US".
|
||||
- The source strings end up in the file `src-ui/messages.xlf`.
|
||||
@@ -495,7 +495,7 @@ class MyCustomParser:
|
||||
self._tempdir = Path(
|
||||
tempfile.mkdtemp(prefix="paperless-", dir=settings.SCRATCH_DIR)
|
||||
)
|
||||
self._text: str | None = None
|
||||
self._text: str = ""
|
||||
self._archive_path: Path | None = None
|
||||
|
||||
def __enter__(self) -> Self:
|
||||
@@ -553,7 +553,8 @@ def parse(
|
||||
**Result accessors**
|
||||
|
||||
```python
|
||||
def get_text(self) -> str | None:
|
||||
def get_text(self) -> str:
|
||||
# Return the extracted text, or an empty string if none was found.
|
||||
return self._text
|
||||
|
||||
def get_date(self) -> "datetime.datetime | None":
|
||||
@@ -684,7 +685,7 @@ class XmlDocumentParser:
|
||||
def __init__(self, logging_group: object = None) -> None:
|
||||
settings.SCRATCH_DIR.mkdir(parents=True, exist_ok=True)
|
||||
self._tempdir = Path(tempfile.mkdtemp(prefix="paperless-", dir=settings.SCRATCH_DIR))
|
||||
self._text: str | None = None
|
||||
self._text: str = ""
|
||||
|
||||
def __enter__(self) -> Self:
|
||||
return self
|
||||
@@ -702,7 +703,7 @@ class XmlDocumentParser:
|
||||
except ET.ParseError as e:
|
||||
raise ParseError(f"XML parse error: {e}") from e
|
||||
|
||||
def get_text(self) -> str | None:
|
||||
def get_text(self) -> str:
|
||||
return self._text
|
||||
|
||||
def get_date(self):
|
||||
|
||||
+29
-6
@@ -70,7 +70,16 @@ elsewhere. Here are a couple notes about that.
|
||||
Paperless-ngx determines the type of a file by inspecting its content
|
||||
rather than its file extensions. However, files processed via the
|
||||
consumption directory will be rejected if they have a file extension that
|
||||
not supported by any of the available parsers.
|
||||
is not supported by any of the available parsers.
|
||||
|
||||
## _Are duplicate documents rejected?_
|
||||
|
||||
**A:** Not by default. As of v3, a file whose contents match an existing document is still
|
||||
consumed, and the duplicate is flagged in the UI — open the document and check the
|
||||
**Duplicates** tab to review documents that share the same content. If you prefer the old
|
||||
behavior of rejecting duplicates during consumption, set
|
||||
[`PAPERLESS_CONSUMER_DELETE_DUPLICATES`](configuration.md#PAPERLESS_CONSUMER_DELETE_DUPLICATES)
|
||||
to `true`.
|
||||
|
||||
## _Will paperless-ngx run on Raspberry Pi?_
|
||||
|
||||
@@ -118,10 +127,24 @@ able to run paperless, you're a bit on your own. If you can't run the
|
||||
docker image, the documentation has instructions for bare metal
|
||||
installs.
|
||||
|
||||
## _What about the Redis licensing change and using one of the open source forks_?
|
||||
## _Does Paperless-ngx use AI, and is my data private?_
|
||||
|
||||
Currently (October 2024), forks of Redis such as Valkey or Redirect are not officially supported by our upstream
|
||||
libraries, so using one of these to replace Redis is not officially supported.
|
||||
**A:** Paperless-ngx includes optional AI features — LLM-based suggestions, document chat,
|
||||
and similar-document retrieval — that are **disabled by default**. They only run when you
|
||||
enable them and configure an LLM backend. The built-in tag/correspondent suggestions use a
|
||||
local, non-LLM machine-learning model and do not send your data anywhere. If you enable the
|
||||
LLM features, document content is sent to whichever backend you configure — this can be a
|
||||
fully local backend (e.g. Ollama) or a remote provider. See
|
||||
[AI features](advanced_usage.md#ai-features) for details.
|
||||
|
||||
However, they do claim to be compatible with the Redis protocol and will likely work, but we will
|
||||
not be updating from using Redis as the broker officially just yet.
|
||||
## _Which message broker should I use_?
|
||||
|
||||
Paperless-ngx talks to a Redis-compatible message broker, so any broker that
|
||||
implements the Redis protocol will work. The bundled Docker Compose files
|
||||
default to [Valkey](https://valkey.io/), the open-source fork created after
|
||||
Redis' licensing change, but Redis itself and other wire-compatible brokers
|
||||
(such as Microsoft's Garnet) are equally fine.
|
||||
|
||||
Existing installs can switch broker implementations in place: point
|
||||
[`PAPERLESS_REDIS`](configuration.md#PAPERLESS_REDIS) at the new instance and
|
||||
reuse the same data volume.
|
||||
|
||||
+2
-1
@@ -35,9 +35,10 @@ physical documents into a searchable online archive so you can keep, well, _less
|
||||
- _New!_ Supports remote OCR with Azure AI (opt-in).
|
||||
- Documents are saved as PDF/A format which is designed for long term storage, alongside the unaltered originals.
|
||||
- Uses machine-learning to automatically add tags, correspondents and document types to your documents.
|
||||
- **New**: Paperless-ngx can now leverage AI (Large Language Models or LLMs) for document suggestions. This is an optional feature that can be enabled (and is disabled by default).
|
||||
- **New**: Paperless-ngx can optionally leverage AI (Large Language Models or LLMs) for document suggestions, chatting with your documents, and similar-document retrieval. These features are opt-in and disabled by default.
|
||||
- Supports PDF documents, images, plain text files, Office documents (Word, Excel, PowerPoint, and LibreOffice equivalents)[^1] and more.
|
||||
- Paperless stores your documents plain on disk. Filenames and folders are managed by paperless and their format can be configured freely with different configurations assigned to different documents.
|
||||
- Keep multiple **versions** of a document's file under a single entry, sharing one set of metadata.
|
||||
- **Beautiful, modern web application** that features:
|
||||
- Customizable dashboard with statistics.
|
||||
- Filtering by tags, correspondents, types, and more.
|
||||
|
||||
+19
-12
@@ -178,7 +178,7 @@ to enable polling and disable inotify. See [here](configuration.md#polling).
|
||||
- `fonts-liberation` for generating thumbnails for plain text
|
||||
files
|
||||
- `imagemagick` >= 6 for PDF conversion
|
||||
- `gnupg` for handling encrypted documents
|
||||
- `gnupg` for decrypting GPG-encrypted email
|
||||
- `libpq-dev` for PostgreSQL
|
||||
- `libmagic-dev` for mime type detection
|
||||
- `mariadb-client` for MariaDB compile time
|
||||
@@ -226,7 +226,8 @@ to enable polling and disable inotify. See [here](configuration.md#polling).
|
||||
build-essential python3-setuptools python3-wheel
|
||||
```
|
||||
|
||||
2. Install `redis` >= 6.0 and configure it to start automatically.
|
||||
2. Install a Redis-compatible broker (a current release of Valkey or
|
||||
Redis) and configure it to start automatically.
|
||||
|
||||
3. Optional: Install `postgresql` and configure a database, user, and
|
||||
password for Paperless-ngx. If you do not wish to use PostgreSQL,
|
||||
@@ -268,10 +269,10 @@ to enable polling and disable inotify. See [here](configuration.md#polling).
|
||||
6. Configure Paperless-ngx. See [configuration](configuration.md) for details.
|
||||
Edit the included `paperless.conf` and adjust the settings to your
|
||||
needs. Required settings for getting Paperless-ngx running are:
|
||||
- [`PAPERLESS_REDIS`](configuration.md#PAPERLESS_REDIS) should point to your Redis server, such as
|
||||
- [`PAPERLESS_REDIS`](configuration.md#PAPERLESS_REDIS) should point to your broker, such as
|
||||
`redis://localhost:6379`.
|
||||
- [`PAPERLESS_DBENGINE`](configuration.md#PAPERLESS_DBENGINE) is optional, and should be one of `postgres`,
|
||||
`mariadb`, or `sqlite`
|
||||
- [`PAPERLESS_DBENGINE`](configuration.md#PAPERLESS_DBENGINE) should be one of `postgresql`,
|
||||
`mariadb`, or `sqlite`. PostgreSQL and MariaDB users must set this explicitly.
|
||||
- [`PAPERLESS_DBHOST`](configuration.md#PAPERLESS_DBHOST) should be the hostname on which your
|
||||
PostgreSQL server is running. Do not configure this to use
|
||||
SQLite instead. Also configure port, database name, user and
|
||||
@@ -297,7 +298,7 @@ to enable polling and disable inotify. See [here](configuration.md#polling).
|
||||
|
||||
!!! warning
|
||||
|
||||
Ensure your Redis instance [is secured](https://redis.io/docs/latest/operate/oss_and_stack/management/security/).
|
||||
Ensure your broker instance [is secured](https://valkey.io/topics/security/).
|
||||
|
||||
7. Create the following directories if they do not already exist:
|
||||
- `/opt/paperless/media`
|
||||
@@ -389,9 +390,9 @@ to enable polling and disable inotify. See [here](configuration.md#polling).
|
||||
`Require=paperless-webserver.socket` in the `webserver` script
|
||||
and configure `granian` to listen on port 80 (set `GRANIAN_PORT`).
|
||||
|
||||
These services rely on Redis and optionally the database server, but
|
||||
These services rely on the broker and optionally the database server, but
|
||||
don't need to be started in any particular order. The example files
|
||||
depend on Redis being started. If you use a database server, you
|
||||
depend on the broker being started. If you use a database server, you
|
||||
should add additional dependencies.
|
||||
|
||||
!!! note
|
||||
@@ -449,6 +450,12 @@ development documentation.
|
||||
You can migrate to Paperless-ngx from Paperless-ng or from the original
|
||||
Paperless project.
|
||||
|
||||
!!! note
|
||||
|
||||
Upgrading an existing Paperless-ngx installation from v2 to v3 has its own
|
||||
breaking changes and required steps. See the [v3 migration guide](migration-v3.md)
|
||||
before upgrading.
|
||||
|
||||
<h3 id="migration_ng">Migrating from Paperless-ng</h3>
|
||||
|
||||
Paperless-ngx is meant to be a drop-in replacement for Paperless-ng, and
|
||||
@@ -494,7 +501,7 @@ installation. Keep these points in mind:
|
||||
for other services, you might as well use it for Paperless as well.
|
||||
- The task scheduler of Paperless, which is used to execute periodic
|
||||
tasks such as email checking and maintenance, requires a
|
||||
[Redis](https://redis.io/) message broker instance. The
|
||||
Redis-compatible message broker instance (such as Valkey or Redis). The
|
||||
Docker Compose route takes care of that.
|
||||
- The layout of the folder structure for your documents and data
|
||||
remains the same, so you can plug your old Docker volumes into
|
||||
@@ -582,16 +589,16 @@ commands as well.
|
||||
|
||||
1. Stop and remove the Paperless container.
|
||||
2. If using an external database, stop that container.
|
||||
3. Update Redis configuration.
|
||||
3. Update broker configuration.
|
||||
1. If `REDIS_URL` is already set, change it to [`PAPERLESS_REDIS`](configuration.md#PAPERLESS_REDIS)
|
||||
and continue to step 4.
|
||||
|
||||
1. Otherwise, add a new Redis service in `docker-compose.yml`,
|
||||
1. Otherwise, add a new broker service in `docker-compose.yml`,
|
||||
following [the example compose
|
||||
files](https://github.com/paperless-ngx/paperless-ngx/tree/main/docker/compose)
|
||||
|
||||
1. Set the environment variable [`PAPERLESS_REDIS`](configuration.md#PAPERLESS_REDIS) so it points to
|
||||
the new Redis container.
|
||||
the new broker container.
|
||||
|
||||
4. Update user mapping.
|
||||
1. If set, change the environment variable `PUID` to `USERMAP_UID`.
|
||||
|
||||
+2
-33
@@ -10,9 +10,9 @@ Check for the following issues:
|
||||
`CONSUMPTION_DIR` setting. Don't adjust this setting if you're
|
||||
using docker.
|
||||
|
||||
- Ensure that redis is up and running. Paperless does its task
|
||||
- Ensure that the broker is up and running. Paperless does its task
|
||||
processing asynchronously, and for documents to arrive at the task
|
||||
processor, it needs redis to run.
|
||||
processor, it needs the broker to run.
|
||||
|
||||
- Ensure that the task processor is running. Docker does this
|
||||
automatically. Manually invoke the task processor by executing
|
||||
@@ -149,37 +149,6 @@ operating system, if these are different from `1000`. See [Docker setup](setup.m
|
||||
Also ensure that you are able to read and write to the consumption
|
||||
directory on the host.
|
||||
|
||||
## OSError: \[Errno 19\] No such device when consuming files
|
||||
|
||||
If you experience errors such as:
|
||||
|
||||
```shell-session
|
||||
File "/usr/local/lib/python3.7/site-packages/whoosh/codec/base.py", line 570, in open_compound_file
|
||||
return CompoundStorage(dbfile, use_mmap=storage.supports_mmap)
|
||||
File "/usr/local/lib/python3.7/site-packages/whoosh/filedb/compound.py", line 75, in __init__
|
||||
self._source = mmap.mmap(fileno, 0, access=mmap.ACCESS_READ)
|
||||
OSError: [Errno 19] No such device
|
||||
|
||||
During handling of the above exception, another exception occurred:
|
||||
|
||||
Traceback (most recent call last):
|
||||
File "/usr/local/lib/python3.7/site-packages/django_q/cluster.py", line 436, in worker
|
||||
res = f(*task["args"], **task["kwargs"])
|
||||
File "/usr/src/paperless/src/documents/tasks.py", line 73, in consume_file
|
||||
override_tag_ids=override_tag_ids)
|
||||
File "/usr/src/paperless/src/documents/consumer.py", line 271, in try_consume_file
|
||||
raise ConsumerError(e)
|
||||
```
|
||||
|
||||
Paperless uses a search index to provide better and faster full text
|
||||
searching. This search index is stored inside the `data` folder. The
|
||||
search index uses memory-mapped files (mmap). The above error indicates
|
||||
that paperless was unable to create and open these files.
|
||||
|
||||
This happens when you're trying to store the data directory on certain
|
||||
file systems (mostly network shares) that don't support memory-mapped
|
||||
files.
|
||||
|
||||
## Web-UI stuck at "Loading\..."
|
||||
|
||||
This might have multiple reasons.
|
||||
|
||||
+21
-2
@@ -292,6 +292,23 @@ Once setup, navigating to the email settings page in Paperless-ngx will allow yo
|
||||
You can also submit a document using the REST API, see [POSTing documents](api.md#file-uploads)
|
||||
for details.
|
||||
|
||||
### Duplicate documents
|
||||
|
||||
By default, Paperless-ngx **does not reject duplicates**. If you consume a file whose
|
||||
contents exactly match an existing document (same checksum), the new copy is still
|
||||
consumed and a warning is logged. The task entry for the upload also flags that a
|
||||
duplicate was detected and links to the existing document(s).
|
||||
|
||||
To review duplicates, open a document and switch to the **Duplicates** tab on the
|
||||
document detail page. It lists other documents that share the same content, including any
|
||||
that are in the trash (shown with a badge), and links to each so you can decide which to
|
||||
keep.
|
||||
|
||||
If you would rather reject duplicates at consumption time (the pre-v3 behavior), set
|
||||
[`PAPERLESS_CONSUMER_DELETE_DUPLICATES`](configuration.md#PAPERLESS_CONSUMER_DELETE_DUPLICATES)
|
||||
to `true`. The duplicate file is then deleted instead of consumed, and the task fails with
|
||||
a "document already exists" message.
|
||||
|
||||
## Document Suggestions
|
||||
|
||||
Paperless-ngx can suggest tags, correspondents, document types and storage paths for documents based on the content of the document. This is done using a (non-LLM) machine learning model that is trained on the documents in your database. The suggestions are shown in the document detail page and can be accepted or rejected by the user.
|
||||
@@ -306,7 +323,9 @@ Paperless-ngx includes several features that use AI to enhance the document mana
|
||||
so consider the privacy implications of using these features, especially if using a remote
|
||||
model or API provider instead of the default local model.
|
||||
|
||||
The AI features work by creating an embedding of the text content and metadata of documents, which is then used for various tasks such as similarity search and question answering. This uses the FAISS vector store.
|
||||
The AI features work by creating an embedding of the text content and metadata of documents, which is then used for various tasks such as similarity search and question answering.
|
||||
|
||||
See [AI features](advanced_usage.md#ai-features) for how to enable and configure these features, including choosing an LLM backend and setting up the LLM index for RAG.
|
||||
|
||||
### AI-Enhanced Suggestions
|
||||
|
||||
@@ -1097,7 +1116,7 @@ Paperless-ngx consists of the following components:
|
||||
errors (i.e., wrong email credentials, errors during consuming a
|
||||
specific file, etc).
|
||||
|
||||
- A [redis](https://redis.io/) message broker: This is a really
|
||||
- A message broker (such as Valkey or Redis): This is a really
|
||||
lightweight service that is responsible for getting the tasks from
|
||||
the webserver and the consumer to the task scheduler. These run in a
|
||||
different process (maybe even on different machines!), and
|
||||
|
||||
+27
-28
@@ -49,9 +49,8 @@ dependencies = [
|
||||
"ijson>=3.2",
|
||||
"imap-tools~=1.13.0",
|
||||
"jinja2~=3.1.5",
|
||||
"lancedb~=0.33.0",
|
||||
"langdetect~=1.0.9",
|
||||
"llama-index-core>=0.14.21",
|
||||
"llama-index-core>=0.14.22",
|
||||
"llama-index-embeddings-huggingface>=0.6.1",
|
||||
"llama-index-embeddings-ollama>=0.9",
|
||||
"llama-index-embeddings-openai-like>=0.2.2",
|
||||
@@ -62,7 +61,6 @@ dependencies = [
|
||||
"openai>=2.32",
|
||||
"pathvalidate~=3.3.1",
|
||||
"pdf2image~=1.17.0",
|
||||
"pyarrow>=16",
|
||||
"python-dateutil~=2.9.0",
|
||||
"python-dotenv~=1.2.1",
|
||||
"python-gnupg~=0.5.4",
|
||||
@@ -74,9 +72,10 @@ dependencies = [
|
||||
"scikit-learn~=1.8.0",
|
||||
"sentence-transformers>=5.4.1",
|
||||
"setproctitle~=1.3.4",
|
||||
"sqlite-vec==0.1.9",
|
||||
"tantivy~=0.26.0",
|
||||
"tika-client~=0.11.0",
|
||||
"torch~=2.11.0",
|
||||
"torch~=2.12.0",
|
||||
"watchfiles>=1.1.1",
|
||||
"whitenoise~=6.11",
|
||||
"zxing-cpp~=3.0.0",
|
||||
@@ -89,7 +88,7 @@ postgres = [
|
||||
"psycopg[c,pool]==3.3",
|
||||
# Direct dependency for proper resolution of the pre-built wheels
|
||||
"psycopg-c==3.3",
|
||||
"psycopg-pool==3.3",
|
||||
"psycopg-pool==3.3.1",
|
||||
]
|
||||
webserver = [
|
||||
"granian[uvloop]~=2.7.0",
|
||||
@@ -102,11 +101,11 @@ dev = [
|
||||
{ include-group = "testing" },
|
||||
]
|
||||
docs = [
|
||||
"zensical>=0.0.36",
|
||||
"zensical>=0.0.43",
|
||||
]
|
||||
lint = [
|
||||
"prek~=0.3.10",
|
||||
"ruff~=0.15.12",
|
||||
"ruff~=0.15.15",
|
||||
]
|
||||
testing = [
|
||||
"daphne",
|
||||
@@ -246,50 +245,38 @@ per-file-ignores."src/documents/models.py" = [
|
||||
isort.force-single-line = true
|
||||
|
||||
[tool.codespell]
|
||||
write-changes = true
|
||||
ignore-words-list = "criterias,afterall,valeu,ureue,equest,ure,assertIn,Oktober,commitish"
|
||||
skip = """\
|
||||
src-ui/src/locale/*,src-ui/pnpm-lock.yaml,src-ui/e2e/*,src/paperless_mail/tests/samples/*,src/paperless/tests/samples\
|
||||
/mail/*,src/documents/tests/samples/*,*.po,*.json\
|
||||
"""
|
||||
write-changes = true
|
||||
|
||||
[tool.pyproject-fmt]
|
||||
table_format = "long"
|
||||
|
||||
[tool.mypy]
|
||||
mypy_path = "src"
|
||||
disallow_any_generics = true
|
||||
disallow_untyped_defs = true
|
||||
disallow_incomplete_defs = true
|
||||
check_untyped_defs = true
|
||||
warn_redundant_casts = true
|
||||
warn_unused_ignores = true
|
||||
plugins = [
|
||||
"mypy_django_plugin.main",
|
||||
"mypy_drf_plugin.main",
|
||||
]
|
||||
check_untyped_defs = true
|
||||
disallow_any_generics = true
|
||||
disallow_incomplete_defs = true
|
||||
disallow_untyped_defs = true
|
||||
warn_redundant_casts = true
|
||||
warn_unused_ignores = true
|
||||
|
||||
[tool.pyrefly]
|
||||
search-path = [ "src" ]
|
||||
baseline = ".pyrefly-baseline.json"
|
||||
python-platform = "linux"
|
||||
search-path = [ "src" ]
|
||||
|
||||
[tool.django-stubs]
|
||||
django_settings_module = "paperless.settings"
|
||||
|
||||
[tool.pytest]
|
||||
minversion = "9.0"
|
||||
pythonpath = [ "src" ]
|
||||
strict_config = true
|
||||
strict_markers = true
|
||||
strict_parametrization_ids = true
|
||||
strict_xfail = true
|
||||
testpaths = [
|
||||
"src/documents/tests/",
|
||||
"src/paperless/tests/",
|
||||
"src/paperless_mail/tests/",
|
||||
"src/paperless_ai/tests",
|
||||
]
|
||||
addopts = [
|
||||
"--pythonwarnings=all",
|
||||
"--cov",
|
||||
@@ -304,7 +291,6 @@ addopts = [
|
||||
"-o",
|
||||
"junit_family=legacy",
|
||||
]
|
||||
norecursedirs = [ "src/locale/", ".venv/", "src-ui/" ]
|
||||
DJANGO_SETTINGS_MODULE = "paperless.settings"
|
||||
markers = [
|
||||
"live: Integration tests requiring external services (Gotenberg, Tika, nginx, etc)",
|
||||
@@ -317,6 +303,19 @@ markers = [
|
||||
"search: Tests for the Tantivy search backend",
|
||||
"api: Tests for REST API endpoints",
|
||||
]
|
||||
minversion = "9.0"
|
||||
norecursedirs = [ "src/locale/", ".venv/", "src-ui/" ]
|
||||
pythonpath = [ "src" ]
|
||||
strict_config = true
|
||||
strict_markers = true
|
||||
strict_parametrization_ids = true
|
||||
strict_xfail = true
|
||||
testpaths = [
|
||||
"src/documents/tests/",
|
||||
"src/paperless/tests/",
|
||||
"src/paperless_mail/tests/",
|
||||
"src/paperless_ai/tests",
|
||||
]
|
||||
|
||||
[tool.pytest_env]
|
||||
PAPERLESS_SECRET_KEY = "test-secret-key-do-not-use-in-production"
|
||||
|
||||
@@ -26,7 +26,7 @@ module.exports = {
|
||||
'abstract-paperless-service',
|
||||
],
|
||||
transformIgnorePatterns: [
|
||||
'node_modules/(?!.*(\\.mjs$|tslib|lodash-es|@angular/common/locales/.*\\.js$))',
|
||||
'node_modules/(?!.*(\\.mjs$|tslib|lodash-es|normalize-diacritics|@angular/common/locales/.*\\.js$))',
|
||||
],
|
||||
moduleNameMapper: {
|
||||
...esmPreset.moduleNameMapper,
|
||||
|
||||
+193
-139
File diff suppressed because it is too large
Load Diff
+4
-3
@@ -12,9 +12,9 @@
|
||||
"private": true,
|
||||
"dependencies": {
|
||||
"@angular/cdk": "^21.2.12",
|
||||
"@angular/common": "~21.2.14",
|
||||
"@angular/compiler": "~21.2.14",
|
||||
"@angular/core": "~21.2.14",
|
||||
"@angular/common": "~21.2.17",
|
||||
"@angular/compiler": "~21.2.17",
|
||||
"@angular/core": "~21.2.17",
|
||||
"@angular/forms": "~21.2.14",
|
||||
"@angular/localize": "~21.2.14",
|
||||
"@angular/platform-browser": "~21.2.14",
|
||||
@@ -32,6 +32,7 @@
|
||||
"ngx-cookie-service": "^21.3.1",
|
||||
"ngx-device-detector": "^11.0.0",
|
||||
"ngx-ui-tour-ng-bootstrap": "^18.0.0",
|
||||
"normalize-diacritics": "^5.0.0",
|
||||
"pdfjs-dist": "^5.7.284",
|
||||
"rxjs": "^7.8.2",
|
||||
"tslib": "^2.8.1",
|
||||
|
||||
Generated
+202
-134
@@ -10,40 +10,40 @@ importers:
|
||||
dependencies:
|
||||
'@angular/cdk':
|
||||
specifier: ^21.2.12
|
||||
version: 21.2.12(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/platform-browser@21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)))(rxjs@7.8.2)
|
||||
version: 21.2.12(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/platform-browser@21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)))(rxjs@7.8.2)
|
||||
'@angular/common':
|
||||
specifier: ~21.2.14
|
||||
version: 21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2)
|
||||
specifier: ~21.2.17
|
||||
version: 21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2)
|
||||
'@angular/compiler':
|
||||
specifier: ~21.2.14
|
||||
version: 21.2.14
|
||||
specifier: ~21.2.17
|
||||
version: 21.2.17
|
||||
'@angular/core':
|
||||
specifier: ~21.2.14
|
||||
version: 21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)
|
||||
specifier: ~21.2.17
|
||||
version: 21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)
|
||||
'@angular/forms':
|
||||
specifier: ~21.2.14
|
||||
version: 21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/platform-browser@21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)))(rxjs@7.8.2)
|
||||
version: 21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/platform-browser@21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)))(rxjs@7.8.2)
|
||||
'@angular/localize':
|
||||
specifier: ~21.2.14
|
||||
version: 21.2.14(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.14)(typescript@5.9.3))(@angular/compiler@21.2.14)
|
||||
version: 21.2.14(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.17)(typescript@5.9.3))(@angular/compiler@21.2.17)
|
||||
'@angular/platform-browser':
|
||||
specifier: ~21.2.14
|
||||
version: 21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))
|
||||
version: 21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))
|
||||
'@angular/platform-browser-dynamic':
|
||||
specifier: ~21.2.14
|
||||
version: 21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/compiler@21.2.14)(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/platform-browser@21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)))
|
||||
version: 21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/compiler@21.2.17)(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/platform-browser@21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)))
|
||||
'@angular/router':
|
||||
specifier: ~21.2.14
|
||||
version: 21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/platform-browser@21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)))(rxjs@7.8.2)
|
||||
version: 21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/platform-browser@21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)))(rxjs@7.8.2)
|
||||
'@ng-bootstrap/ng-bootstrap':
|
||||
specifier: ^20.0.0
|
||||
version: 20.0.0(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/forms@21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/platform-browser@21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)))(rxjs@7.8.2))(@angular/localize@21.2.14(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.14)(typescript@5.9.3))(@angular/compiler@21.2.14))(@popperjs/core@2.11.8)(rxjs@7.8.2)
|
||||
version: 20.0.0(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/forms@21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/platform-browser@21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)))(rxjs@7.8.2))(@angular/localize@21.2.14(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.17)(typescript@5.9.3))(@angular/compiler@21.2.17))(@popperjs/core@2.11.8)(rxjs@7.8.2)
|
||||
'@ng-select/ng-select':
|
||||
specifier: ^21.8.2
|
||||
version: 21.8.2(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/forms@21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/platform-browser@21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)))(rxjs@7.8.2))
|
||||
version: 21.8.2(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/forms@21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/platform-browser@21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)))(rxjs@7.8.2))
|
||||
'@ngneat/dirty-check-forms':
|
||||
specifier: ^3.0.3
|
||||
version: 3.0.3(be5de60320c5c6a3310af74f068bbe95)
|
||||
version: 3.0.3(ad2c8ff51b8ef8626e139c84727a024d)
|
||||
'@popperjs/core':
|
||||
specifier: ^2.11.8
|
||||
version: 2.11.8
|
||||
@@ -58,19 +58,22 @@ importers:
|
||||
version: 1.0.0
|
||||
ngx-bootstrap-icons:
|
||||
specifier: ^1.9.3
|
||||
version: 1.9.3(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))
|
||||
version: 1.9.3(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))
|
||||
ngx-color:
|
||||
specifier: ^10.1.0
|
||||
version: 10.1.0(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))
|
||||
version: 10.1.0(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))
|
||||
ngx-cookie-service:
|
||||
specifier: ^21.3.1
|
||||
version: 21.3.1(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))
|
||||
version: 21.3.1(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))
|
||||
ngx-device-detector:
|
||||
specifier: ^11.0.0
|
||||
version: 11.0.0(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))
|
||||
version: 11.0.0(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))
|
||||
ngx-ui-tour-ng-bootstrap:
|
||||
specifier: ^18.0.0
|
||||
version: 18.0.0(f910a33494d223bd6dd07ce1bf22a35e)
|
||||
version: 18.0.0(4ccfccfbcf381a309618492b31e99276)
|
||||
normalize-diacritics:
|
||||
specifier: ^5.0.0
|
||||
version: 5.0.0
|
||||
pdfjs-dist:
|
||||
specifier: ^5.7.284
|
||||
version: 5.7.284
|
||||
@@ -92,10 +95,10 @@ importers:
|
||||
devDependencies:
|
||||
'@angular-builders/custom-webpack':
|
||||
specifier: ^21.0.3
|
||||
version: 21.0.3(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.14)(typescript@5.9.3))(@angular/compiler@21.2.14)(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/localize@21.2.14(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.14)(typescript@5.9.3))(@angular/compiler@21.2.14))(@angular/platform-browser@21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)))(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)(@types/node@25.9.1)(chokidar@5.0.0)(jest-environment-jsdom@30.4.1(canvas@3.0.0))(jest@30.4.2(@types/node@25.9.1)(ts-node@10.9.2(@types/node@25.9.1)(typescript@5.9.3)))(jiti@2.6.1)(less@4.4.2)(postcss@8.5.15)(terser@5.44.1)(tslib@2.8.1)(typescript@5.9.3)(yaml@2.7.0)
|
||||
version: 21.0.3(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.17)(typescript@5.9.3))(@angular/compiler@21.2.17)(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/localize@21.2.14(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.17)(typescript@5.9.3))(@angular/compiler@21.2.17))(@angular/platform-browser@21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)))(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)(@types/node@25.9.1)(chokidar@5.0.0)(jest-environment-jsdom@30.4.1(canvas@3.0.0))(jest@30.4.2(@types/node@25.9.1)(ts-node@10.9.2(@types/node@25.9.1)(typescript@5.9.3)))(jiti@2.6.1)(less@4.4.2)(postcss@8.5.15)(terser@5.44.1)(tslib@2.8.1)(typescript@5.9.3)(yaml@2.7.0)
|
||||
'@angular-builders/jest':
|
||||
specifier: ^21.0.3
|
||||
version: 21.0.3(45beaf077858833b14ba9080c452c7e9)
|
||||
version: 21.0.3(6a682f4f002a31c8d3b52779d7b9ab9b)
|
||||
'@angular-devkit/core':
|
||||
specifier: ^21.2.12
|
||||
version: 21.2.12(chokidar@5.0.0)
|
||||
@@ -119,13 +122,13 @@ importers:
|
||||
version: 21.4.0(eslint@10.4.0(jiti@2.6.1))(typescript@5.9.3)
|
||||
'@angular/build':
|
||||
specifier: ^21.2.12
|
||||
version: 21.2.12(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.14)(typescript@5.9.3))(@angular/compiler@21.2.14)(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/localize@21.2.14(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.14)(typescript@5.9.3))(@angular/compiler@21.2.14))(@angular/platform-browser@21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)))(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)(@types/node@25.9.1)(chokidar@5.0.0)(jiti@2.6.1)(less@4.4.2)(postcss@8.5.15)(terser@5.44.1)(tslib@2.8.1)(typescript@5.9.3)(yaml@2.7.0)
|
||||
version: 21.2.12(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.17)(typescript@5.9.3))(@angular/compiler@21.2.17)(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/localize@21.2.14(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.17)(typescript@5.9.3))(@angular/compiler@21.2.17))(@angular/platform-browser@21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)))(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)(@types/node@25.9.1)(chokidar@5.0.0)(jiti@2.6.1)(less@4.4.2)(postcss@8.5.15)(terser@5.44.1)(tslib@2.8.1)(typescript@5.9.3)(yaml@2.7.0)
|
||||
'@angular/cli':
|
||||
specifier: ~21.2.12
|
||||
version: 21.2.12(@types/node@25.9.1)(chokidar@5.0.0)
|
||||
'@angular/compiler-cli':
|
||||
specifier: ~21.2.14
|
||||
version: 21.2.14(@angular/compiler@21.2.14)(typescript@5.9.3)
|
||||
version: 21.2.14(@angular/compiler@21.2.17)(typescript@5.9.3)
|
||||
'@codecov/webpack-plugin':
|
||||
specifier: ^2.0.1
|
||||
version: 2.0.1(webpack@5.107.2(postcss@8.5.15))
|
||||
@@ -161,7 +164,7 @@ importers:
|
||||
version: 17.0.0
|
||||
jest-preset-angular:
|
||||
specifier: ^16.1.5
|
||||
version: 16.1.5(43a2e4c530b4286e50e732293015d944)
|
||||
version: 16.1.5(26662f94407112e0967a16d5ea795956)
|
||||
jest-websocket-mock:
|
||||
specifier: ^2.5.0
|
||||
version: 2.5.0
|
||||
@@ -530,11 +533,11 @@ packages:
|
||||
engines: {node: ^20.19.0 || ^22.12.0 || >=24.0.0, npm: ^6.11.0 || ^7.5.6 || >=8.0.0, yarn: '>= 1.13.0'}
|
||||
hasBin: true
|
||||
|
||||
'@angular/common@21.2.14':
|
||||
resolution: {integrity: sha512-J6K7cE7uKOKmg4+sxLeGfsmaYDjP5l1XCiMMI0WPT0t68uxLk8g3MzV5Trqfb6ZnRxWcfp9c4c+XxAvMBB7ymA==}
|
||||
'@angular/common@21.2.17':
|
||||
resolution: {integrity: sha512-hqAQxRfi5ldFE42suAXRcY+JCANrUh7fuSQ/DtZ7L896id5BT/exuv6dWNBC1PyAfQmRbpD5Pt6/pd+tNLyhDQ==}
|
||||
engines: {node: ^20.19.0 || ^22.12.0 || >=24.0.0}
|
||||
peerDependencies:
|
||||
'@angular/core': 21.2.14
|
||||
'@angular/core': 21.2.17
|
||||
rxjs: ^6.5.3 || ^7.4.0
|
||||
|
||||
'@angular/compiler-cli@21.2.14':
|
||||
@@ -548,15 +551,15 @@ packages:
|
||||
typescript:
|
||||
optional: true
|
||||
|
||||
'@angular/compiler@21.2.14':
|
||||
resolution: {integrity: sha512-8mqgwRYfn2Z1vg/5YVt60dDBattnZL45nNJd2vTMwAiDTzhWhgKgRWKOeVL0aj2JqHeHiwuIlrLnz46acJMulQ==}
|
||||
'@angular/compiler@21.2.17':
|
||||
resolution: {integrity: sha512-p+NdjYiwAz9Zmu2yul0LlMXaFjMISVVa24+/MVMoKFeQeI82QE8jDywPlnOSHQHvdCcQVpS7saeEriZzX3JuBQ==}
|
||||
engines: {node: ^20.19.0 || ^22.12.0 || >=24.0.0}
|
||||
|
||||
'@angular/core@21.2.14':
|
||||
resolution: {integrity: sha512-Z1Ivjh7L2lT//8LA7vQ3tj7Rg6wl2XRA5kPSAukgn8u0Yu0XxG8NE8KG0Eypb3v9CEcbwATwpgnxzbJFZ8TFcw==}
|
||||
'@angular/core@21.2.17':
|
||||
resolution: {integrity: sha512-wYHpwIdnUnjQFOJJNqRcGx7LS3u64jT+R9L0TnMR/ViBM9dQgGYImlSikkftg2yrFCNo5aKRxhG2LLskQurVdg==}
|
||||
engines: {node: ^20.19.0 || ^22.12.0 || >=24.0.0}
|
||||
peerDependencies:
|
||||
'@angular/compiler': 21.2.14
|
||||
'@angular/compiler': 21.2.17
|
||||
rxjs: ^6.5.3 || ^7.4.0
|
||||
zone.js: ~0.15.0 || ~0.16.0
|
||||
peerDependenciesMeta:
|
||||
@@ -2392,30 +2395,35 @@ packages:
|
||||
engines: {node: '>= 10'}
|
||||
cpu: [arm64]
|
||||
os: [linux]
|
||||
libc: [glibc]
|
||||
|
||||
'@napi-rs/canvas-linux-arm64-musl@0.1.100':
|
||||
resolution: {integrity: sha512-K3mDW66N+xT2/V439u1alFANiBUjdEx2gLiNYnCmUsva5jZMxWTjafBYwTzYK+EMFMHrUoabuU+T1BIP5CgbYQ==}
|
||||
engines: {node: '>= 10'}
|
||||
cpu: [arm64]
|
||||
os: [linux]
|
||||
libc: [musl]
|
||||
|
||||
'@napi-rs/canvas-linux-riscv64-gnu@0.1.100':
|
||||
resolution: {integrity: sha512-mooqUBTIsccZpnoQC4NgrC1v6C1vof39etLNMnBwCY+p0gajWJvAHLGQ6g/gGyS5YrpDW+GefSN4+Cvcr08UWw==}
|
||||
engines: {node: '>= 10'}
|
||||
cpu: [riscv64]
|
||||
os: [linux]
|
||||
libc: [glibc]
|
||||
|
||||
'@napi-rs/canvas-linux-x64-gnu@0.1.100':
|
||||
resolution: {integrity: sha512-1eCvkDCazm7FFhsT7DfGOdSaHgZVK3bt/dSBl5EWHOWmnz+I7j8tPseJqqD81NF+MH21jKUK4wQSDjN0mdhnTg==}
|
||||
engines: {node: '>= 10'}
|
||||
cpu: [x64]
|
||||
os: [linux]
|
||||
libc: [glibc]
|
||||
|
||||
'@napi-rs/canvas-linux-x64-musl@0.1.100':
|
||||
resolution: {integrity: sha512-20arT6lnI19S68qNlii73TSEDbECNgzMz2EpldC1V3mZFuRkeujXkcebRk0LRJe9SEUAooYiLokfMViY8IX7yA==}
|
||||
engines: {node: '>= 10'}
|
||||
cpu: [x64]
|
||||
os: [linux]
|
||||
libc: [musl]
|
||||
|
||||
'@napi-rs/canvas-win32-arm64-msvc@0.1.100':
|
||||
resolution: {integrity: sha512-DZFFT1wIAg37LJw37yhMRFfjATd3vTQzjZ1Yki8u2vhO6Hi5VE6BVaGQ1aaDu7xb4iMErz+9EOwjpS7xcxFeBw==}
|
||||
@@ -2474,42 +2482,49 @@ packages:
|
||||
engines: {node: '>= 10'}
|
||||
cpu: [arm64]
|
||||
os: [linux]
|
||||
libc: [glibc]
|
||||
|
||||
'@napi-rs/nice-linux-arm64-musl@1.1.1':
|
||||
resolution: {integrity: sha512-+2Rzdb3nTIYZ0YJF43qf2twhqOCkiSrHx2Pg6DJaCPYhhaxbLcdlV8hCRMHghQ+EtZQWGNcS2xF4KxBhSGeutg==}
|
||||
engines: {node: '>= 10'}
|
||||
cpu: [arm64]
|
||||
os: [linux]
|
||||
libc: [musl]
|
||||
|
||||
'@napi-rs/nice-linux-ppc64-gnu@1.1.1':
|
||||
resolution: {integrity: sha512-4FS8oc0GeHpwvv4tKciKkw3Y4jKsL7FRhaOeiPei0X9T4Jd619wHNe4xCLmN2EMgZoeGg+Q7GY7BsvwKpL22Tg==}
|
||||
engines: {node: '>= 10'}
|
||||
cpu: [ppc64]
|
||||
os: [linux]
|
||||
libc: [glibc]
|
||||
|
||||
'@napi-rs/nice-linux-riscv64-gnu@1.1.1':
|
||||
resolution: {integrity: sha512-HU0nw9uD4FO/oGCCk409tCi5IzIZpH2agE6nN4fqpwVlCn5BOq0MS1dXGjXaG17JaAvrlpV5ZeyZwSon10XOXw==}
|
||||
engines: {node: '>= 10'}
|
||||
cpu: [riscv64]
|
||||
os: [linux]
|
||||
libc: [glibc]
|
||||
|
||||
'@napi-rs/nice-linux-s390x-gnu@1.1.1':
|
||||
resolution: {integrity: sha512-2YqKJWWl24EwrX0DzCQgPLKQBxYDdBxOHot1KWEq7aY2uYeX+Uvtv4I8xFVVygJDgf6/92h9N3Y43WPx8+PAgQ==}
|
||||
engines: {node: '>= 10'}
|
||||
cpu: [s390x]
|
||||
os: [linux]
|
||||
libc: [glibc]
|
||||
|
||||
'@napi-rs/nice-linux-x64-gnu@1.1.1':
|
||||
resolution: {integrity: sha512-/gaNz3R92t+dcrfCw/96pDopcmec7oCcAQ3l/M+Zxr82KT4DljD37CpgrnXV+pJC263JkW572pdbP3hP+KjcIg==}
|
||||
engines: {node: '>= 10'}
|
||||
cpu: [x64]
|
||||
os: [linux]
|
||||
libc: [glibc]
|
||||
|
||||
'@napi-rs/nice-linux-x64-musl@1.1.1':
|
||||
resolution: {integrity: sha512-xScCGnyj/oppsNPMnevsBe3pvNaoK7FGvMjT35riz9YdhB2WtTG47ZlbxtOLpjeO9SqqQ2J2igCmz6IJOD5JYw==}
|
||||
engines: {node: '>= 10'}
|
||||
cpu: [x64]
|
||||
os: [linux]
|
||||
libc: [musl]
|
||||
|
||||
'@napi-rs/nice-openharmony-arm64@1.1.1':
|
||||
resolution: {integrity: sha512-6uJPRVwVCLDeoOaNyeiW0gp2kFIM4r7PL2MczdZQHkFi9gVlgm+Vn+V6nTWRcu856mJ2WjYJiumEajfSm7arPQ==}
|
||||
@@ -2694,36 +2709,42 @@ packages:
|
||||
engines: {node: '>= 10.0.0'}
|
||||
cpu: [arm]
|
||||
os: [linux]
|
||||
libc: [glibc]
|
||||
|
||||
'@parcel/watcher-linux-arm-musl@2.5.6':
|
||||
resolution: {integrity: sha512-Ve3gUCG57nuUUSyjBq/MAM0CzArtuIOxsBdQ+ftz6ho8n7s1i9E1Nmk/xmP323r2YL0SONs1EuwqBp2u1k5fxg==}
|
||||
engines: {node: '>= 10.0.0'}
|
||||
cpu: [arm]
|
||||
os: [linux]
|
||||
libc: [musl]
|
||||
|
||||
'@parcel/watcher-linux-arm64-glibc@2.5.6':
|
||||
resolution: {integrity: sha512-f2g/DT3NhGPdBmMWYoxixqYr3v/UXcmLOYy16Bx0TM20Tchduwr4EaCbmxh1321TABqPGDpS8D/ggOTaljijOA==}
|
||||
engines: {node: '>= 10.0.0'}
|
||||
cpu: [arm64]
|
||||
os: [linux]
|
||||
libc: [glibc]
|
||||
|
||||
'@parcel/watcher-linux-arm64-musl@2.5.6':
|
||||
resolution: {integrity: sha512-qb6naMDGlbCwdhLj6hgoVKJl2odL34z2sqkC7Z6kzir8b5W65WYDpLB6R06KabvZdgoHI/zxke4b3zR0wAbDTA==}
|
||||
engines: {node: '>= 10.0.0'}
|
||||
cpu: [arm64]
|
||||
os: [linux]
|
||||
libc: [musl]
|
||||
|
||||
'@parcel/watcher-linux-x64-glibc@2.5.6':
|
||||
resolution: {integrity: sha512-kbT5wvNQlx7NaGjzPFu8nVIW1rWqV780O7ZtkjuWaPUgpv2NMFpjYERVi0UYj1msZNyCzGlaCWEtzc+exjMGbQ==}
|
||||
engines: {node: '>= 10.0.0'}
|
||||
cpu: [x64]
|
||||
os: [linux]
|
||||
libc: [glibc]
|
||||
|
||||
'@parcel/watcher-linux-x64-musl@2.5.6':
|
||||
resolution: {integrity: sha512-1JRFeC+h7RdXwldHzTsmdtYR/Ku8SylLgTU/reMuqdVD7CtLwf0VR1FqeprZ0eHQkO0vqsbvFLXUmYm/uNKJBg==}
|
||||
engines: {node: '>= 10.0.0'}
|
||||
cpu: [x64]
|
||||
os: [linux]
|
||||
libc: [musl]
|
||||
|
||||
'@parcel/watcher-win32-arm64@2.5.6':
|
||||
resolution: {integrity: sha512-3ukyebjc6eGlw9yRt678DxVF7rjXatWiHvTXqphZLvo7aC5NdEgFufVwjFfY51ijYEWpXbqF5jtrK275z52D4Q==}
|
||||
@@ -2828,48 +2849,56 @@ packages:
|
||||
engines: {node: ^20.19.0 || >=22.12.0}
|
||||
cpu: [arm64]
|
||||
os: [linux]
|
||||
libc: [glibc]
|
||||
|
||||
'@rolldown/binding-linux-arm64-gnu@1.0.0-rc.4':
|
||||
resolution: {integrity: sha512-AC1WsGdlV1MtGay/OQ4J9T7GRadVnpYRzTcygV1hKnypbYN20Yh4t6O1Sa2qRBMqv1etulUknqXjc3CTIsBu6A==}
|
||||
engines: {node: ^20.19.0 || >=22.12.0}
|
||||
cpu: [arm64]
|
||||
os: [linux]
|
||||
libc: [glibc]
|
||||
|
||||
'@rolldown/binding-linux-arm64-musl@1.0.0-beta.58':
|
||||
resolution: {integrity: sha512-l+p4QVtG72C7wI2SIkNQw/KQtSjuYwS3rV6AKcWrRBF62ClsFUcif5vLaZIEbPrCXu5OFRXigXFJnxYsVVZqdQ==}
|
||||
engines: {node: ^20.19.0 || >=22.12.0}
|
||||
cpu: [arm64]
|
||||
os: [linux]
|
||||
libc: [musl]
|
||||
|
||||
'@rolldown/binding-linux-arm64-musl@1.0.0-rc.4':
|
||||
resolution: {integrity: sha512-lU+6rgXXViO61B4EudxtVMXSOfiZONR29Sys5VGSetUY7X8mg9FCKIIjcPPj8xNDeYzKl+H8F/qSKOBVFJChCQ==}
|
||||
engines: {node: ^20.19.0 || >=22.12.0}
|
||||
cpu: [arm64]
|
||||
os: [linux]
|
||||
libc: [musl]
|
||||
|
||||
'@rolldown/binding-linux-x64-gnu@1.0.0-beta.58':
|
||||
resolution: {integrity: sha512-urzJX0HrXxIh0FfxwWRjfPCMeInU9qsImLQxHBgLp5ivji1EEUnOfux8KxPPnRQthJyneBrN2LeqUix9DYrNaQ==}
|
||||
engines: {node: ^20.19.0 || >=22.12.0}
|
||||
cpu: [x64]
|
||||
os: [linux]
|
||||
libc: [glibc]
|
||||
|
||||
'@rolldown/binding-linux-x64-gnu@1.0.0-rc.4':
|
||||
resolution: {integrity: sha512-DZaN1f0PGp/bSvKhtw50pPsnln4T13ycDq1FrDWRiHmWt1JeW+UtYg9touPFf8yt993p8tS2QjybpzKNTxYEwg==}
|
||||
engines: {node: ^20.19.0 || >=22.12.0}
|
||||
cpu: [x64]
|
||||
os: [linux]
|
||||
libc: [glibc]
|
||||
|
||||
'@rolldown/binding-linux-x64-musl@1.0.0-beta.58':
|
||||
resolution: {integrity: sha512-7ijfVK3GISnXIwq/1FZo+KyAUJjL3kWPJ7rViAL6MWeEBhEgRzJ0yEd9I8N9aut8Y8ab+EKFJyRNMWZuUBwQ0A==}
|
||||
engines: {node: ^20.19.0 || >=22.12.0}
|
||||
cpu: [x64]
|
||||
os: [linux]
|
||||
libc: [musl]
|
||||
|
||||
'@rolldown/binding-linux-x64-musl@1.0.0-rc.4':
|
||||
resolution: {integrity: sha512-RnGxwZLN7fhMMAItnD6dZ7lvy+TI7ba+2V54UF4dhaWa/p8I/ys1E73KO6HmPmgz92ZkfD8TXS1IMV8+uhbR9g==}
|
||||
engines: {node: ^20.19.0 || >=22.12.0}
|
||||
cpu: [x64]
|
||||
os: [linux]
|
||||
libc: [musl]
|
||||
|
||||
'@rolldown/binding-openharmony-arm64@1.0.0-beta.58':
|
||||
resolution: {integrity: sha512-/m7sKZCS+cUULbzyJTIlv8JbjNohxbpAOA6cM+lgWgqVzPee3U6jpwydrib328JFN/gF9A99IZEnuGYqEDJdww==}
|
||||
@@ -2957,66 +2986,79 @@ packages:
|
||||
resolution: {integrity: sha512-DxH0P3wxm+Yzs/p3zrk9dw1rURu8p0Nv5+MRK/L7OtnLNg5rLZraSBFZ8iUXOd9f2BlhJyEpIZUH/emjq4UJ4g==}
|
||||
cpu: [arm]
|
||||
os: [linux]
|
||||
libc: [glibc]
|
||||
|
||||
'@rollup/rollup-linux-arm-musleabihf@4.61.0':
|
||||
resolution: {integrity: sha512-T6ZvMNe84kAz6TBWHC7hGAoEtzP1LWYw/AqayGWEF6uISt3Abk/st06LqRD9THd7Xz3NxzurUpzAuEAUbZf+nw==}
|
||||
cpu: [arm]
|
||||
os: [linux]
|
||||
libc: [musl]
|
||||
|
||||
'@rollup/rollup-linux-arm64-gnu@4.61.0':
|
||||
resolution: {integrity: sha512-q/4hzvQkDs8b4jIBab1pnLiiM0ayTZsN2amBFPDzuyZxjEd4wDwx0UJFYM3cOZzSf5Kw8fnWSprJzIBMkcR44Q==}
|
||||
cpu: [arm64]
|
||||
os: [linux]
|
||||
libc: [glibc]
|
||||
|
||||
'@rollup/rollup-linux-arm64-musl@4.61.0':
|
||||
resolution: {integrity: sha512-vvYWX3akdEAY6km+9wAqFDnk6pQsbJKVnj7xawcvs/+fdlYBGp+U+Qq/lLfpIxYIZvZLHMAKD9HLdacSx/r3dw==}
|
||||
cpu: [arm64]
|
||||
os: [linux]
|
||||
libc: [musl]
|
||||
|
||||
'@rollup/rollup-linux-loong64-gnu@4.61.0':
|
||||
resolution: {integrity: sha512-DePa5cqOxDP/Zp0VOXpeWaGew5iIv5DXp9NYbzkX5PFQyWVX9184WCTh3hvr/7lhXo8ZVlbFLkz8+o/q1dU6gA==}
|
||||
cpu: [loong64]
|
||||
os: [linux]
|
||||
libc: [glibc]
|
||||
|
||||
'@rollup/rollup-linux-loong64-musl@4.61.0':
|
||||
resolution: {integrity: sha512-LV8aWMB8UChglMCEzs7RkN0GsH29RJaLLqwm9fCIjlqwxQTiWAqNcc7wjBkH31hV0PU/yVxGYvrYsgfea2qw6g==}
|
||||
cpu: [loong64]
|
||||
os: [linux]
|
||||
libc: [musl]
|
||||
|
||||
'@rollup/rollup-linux-ppc64-gnu@4.61.0':
|
||||
resolution: {integrity: sha512-QoNSnwQtaeNu5grdBbsL0tt1uyl5EnS8DA8Mr3nluMXbhdQNyhN+G4tBax7VCdxLKj8YJ0/4OO9Ho84jMnJtKA==}
|
||||
cpu: [ppc64]
|
||||
os: [linux]
|
||||
libc: [glibc]
|
||||
|
||||
'@rollup/rollup-linux-ppc64-musl@4.61.0':
|
||||
resolution: {integrity: sha512-/zZp5MKapIIApE8trN8qLGNSiRN9TUoaUZ1cmVu4XnVdd5LQLOXTtyi+vtfUbNnT3iyjzpPqYeKXmvJ+gJGYWw==}
|
||||
cpu: [ppc64]
|
||||
os: [linux]
|
||||
libc: [musl]
|
||||
|
||||
'@rollup/rollup-linux-riscv64-gnu@4.61.0':
|
||||
resolution: {integrity: sha512-RbrzcD3aJ1k3UbtMRRBNwojdVVyXjuVAFTfn/xPa6EEl6GE9Sm/akPgFTb9aAC9pMKGJ6CtWxaGrqWcabH+ySg==}
|
||||
cpu: [riscv64]
|
||||
os: [linux]
|
||||
libc: [glibc]
|
||||
|
||||
'@rollup/rollup-linux-riscv64-musl@4.61.0':
|
||||
resolution: {integrity: sha512-ZF+onDsBso8PJf1XaG9lB+O9RnBpKGnY6OrzC4CSHrtC1jb6jWLTKK4bRqdoCXHd22gyr2hiYmEAm8Wns/BOCw==}
|
||||
cpu: [riscv64]
|
||||
os: [linux]
|
||||
libc: [musl]
|
||||
|
||||
'@rollup/rollup-linux-s390x-gnu@4.61.0':
|
||||
resolution: {integrity: sha512-Atk0aSIk5Zx2Wuh9dgRQgLP0Koc8hOeYpbWryMXyk8G8/HmPkwPPkMqIIDhrXHHYqfUzSJA/I7IWSBv8xSmRBA==}
|
||||
cpu: [s390x]
|
||||
os: [linux]
|
||||
libc: [glibc]
|
||||
|
||||
'@rollup/rollup-linux-x64-gnu@4.61.0':
|
||||
resolution: {integrity: sha512-0uMOcf3eZ5K+K4cYHkdxShFMPlPXCOdfDFEFn9dNYAEEd2cVvmOfH7zFgRVoDgmtQ1m9k5q7qfrHzyMAubKYUA==}
|
||||
cpu: [x64]
|
||||
os: [linux]
|
||||
libc: [glibc]
|
||||
|
||||
'@rollup/rollup-linux-x64-musl@4.61.0':
|
||||
resolution: {integrity: sha512-mvFtE4A/t/7hRJ7X8Ozmu8FsIkAUat2nzl12pgU337BRmq87AQUJztwHz2Zv5/tjo9/C95E66CK03SI/ToEDJw==}
|
||||
cpu: [x64]
|
||||
os: [linux]
|
||||
libc: [musl]
|
||||
|
||||
'@rollup/rollup-openbsd-x64@4.61.0':
|
||||
resolution: {integrity: sha512-z9b9+aTxvt8n2rNltMPvyaUfB8NJ+CVyOrGK/MdIKHx7B+lXmZpm/XbRsU7Rpf3fRqJ2uS6mBJiJveCtq8LHDg==}
|
||||
@@ -3329,51 +3371,61 @@ packages:
|
||||
resolution: {integrity: sha512-zJc0H99FEPoFfSrNpa91HYfxzfAJCr502oxNK1cfdC9hlaFI43RT+JFCann9JUgZmLzzntChHyn13Sgn9ljHNg==}
|
||||
cpu: [arm64]
|
||||
os: [linux]
|
||||
libc: [glibc]
|
||||
|
||||
'@unrs/resolver-binding-linux-arm64-musl@1.12.2':
|
||||
resolution: {integrity: sha512-KQ3Lki6l+Pz1k/eBipN41ES+YUK30beLGb9YqcB1O542cyLCNE6GaxrfcY3T6EezmGGk84wb5XyO9loTM9tkcA==}
|
||||
cpu: [arm64]
|
||||
os: [linux]
|
||||
libc: [musl]
|
||||
|
||||
'@unrs/resolver-binding-linux-loong64-gnu@1.12.2':
|
||||
resolution: {integrity: sha512-3SJGEh1DborhG6pyxvhPzCT4bbSIVihsvgJc13P1bHG7KLdNDaF9T3gsTwFc7Jw/5Y5/iWOjkEx7Zy0NvCGX3Q==}
|
||||
cpu: [loong64]
|
||||
os: [linux]
|
||||
libc: [glibc]
|
||||
|
||||
'@unrs/resolver-binding-linux-loong64-musl@1.12.2':
|
||||
resolution: {integrity: sha512-jiuG/Obbel7uw1PwHNFfrkiKhLAF6mnyZ6aWlOAVN9WqKm8v0OFGnciJIHu8+CMvXLQ8AD51LPzAoUfT21D5Ew==}
|
||||
cpu: [loong64]
|
||||
os: [linux]
|
||||
libc: [musl]
|
||||
|
||||
'@unrs/resolver-binding-linux-ppc64-gnu@1.12.2':
|
||||
resolution: {integrity: sha512-q7xRvVpmcfeL+LlZg8Pbbo6QaTZwDU5BaGZbwfhkEsXJn3Was8xYfE0RBH266xZt0rM6B7i8xAYIvjthuUIWHg==}
|
||||
cpu: [ppc64]
|
||||
os: [linux]
|
||||
libc: [glibc]
|
||||
|
||||
'@unrs/resolver-binding-linux-riscv64-gnu@1.12.2':
|
||||
resolution: {integrity: sha512-0CVdx6lcnT3Q9inOH8tsMIOJ6ImndllMjqJHg8RLVdB7Vq4SfkEXl9mCSsVNuNA4MCYycRicCUxPCabVHJRr6A==}
|
||||
cpu: [riscv64]
|
||||
os: [linux]
|
||||
libc: [glibc]
|
||||
|
||||
'@unrs/resolver-binding-linux-riscv64-musl@1.12.2':
|
||||
resolution: {integrity: sha512-iOwlRo9vnp6R6ohHQS11n0NnfdXx/omhkocmIfaPRpQhKZ+3BDMkkdRVh53qjkFkpPddf+FETA28NwGN7l5l+w==}
|
||||
cpu: [riscv64]
|
||||
os: [linux]
|
||||
libc: [musl]
|
||||
|
||||
'@unrs/resolver-binding-linux-s390x-gnu@1.12.2':
|
||||
resolution: {integrity: sha512-HYJtLfXq94q8iZNFT1lknx258wlkkWhZeUXJRqzKBBUJ00CvZ+N33zgbCqimLjsyw5Va6uUxhVa12mI+kaveEw==}
|
||||
cpu: [s390x]
|
||||
os: [linux]
|
||||
libc: [glibc]
|
||||
|
||||
'@unrs/resolver-binding-linux-x64-gnu@1.12.2':
|
||||
resolution: {integrity: sha512-mPsUhunKKDih5O96Y6enDQyHc1SqBPlY1E/SfMWDM3EdJ95Z9CArPeCVwCCqbP45ljvivdEk8Fxn+SIb1rDAJQ==}
|
||||
cpu: [x64]
|
||||
os: [linux]
|
||||
libc: [glibc]
|
||||
|
||||
'@unrs/resolver-binding-linux-x64-musl@1.12.2':
|
||||
resolution: {integrity: sha512-azrt6+5ydLd8Vt210AAFis/lZevSfPw93EJRIJG+xPu4WCJ8K0kppCTpMyLPcKT7H15M4Jnt2tMp5bOvCkRC6A==}
|
||||
cpu: [x64]
|
||||
os: [linux]
|
||||
libc: [musl]
|
||||
|
||||
'@unrs/resolver-binding-openharmony-arm64@1.12.2':
|
||||
resolution: {integrity: sha512-YZ9hP4O0X9PQb8eO980qmLNGH4zT3I9+SZTdt0Pr0YyuGQhYKoOZkV02VzrzyOZJ5xIJ3UFIenKkUkGg8GjgWQ==}
|
||||
@@ -5516,6 +5568,10 @@ packages:
|
||||
engines: {node: ^20.17.0 || >=22.9.0}
|
||||
hasBin: true
|
||||
|
||||
normalize-diacritics@5.0.0:
|
||||
resolution: {integrity: sha512-t6czCJOpbAtckN1wCC2qPWnO3GQvNANb9bcUNbiOLEqojVuP31+ELIs5KhEG8jyz0TH7iD9BWxWz8O3ic2/rMQ==}
|
||||
engines: {node: '>= 14.x', npm: '>= 6.x'}
|
||||
|
||||
normalize-path@3.0.0:
|
||||
resolution: {integrity: sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==}
|
||||
engines: {node: '>=0.10.0'}
|
||||
@@ -6116,6 +6172,11 @@ packages:
|
||||
engines: {node: '>=10'}
|
||||
hasBin: true
|
||||
|
||||
semver@7.8.4:
|
||||
resolution: {integrity: sha512-rUCObTnP32Q08R2uuIrt7r9PlEonuTmtuXYcW6s5kjdlj3xbnwe+21yXptAUYcMAABLkYYTtnmzb3w3EDZfueA==}
|
||||
engines: {node: '>=10'}
|
||||
hasBin: true
|
||||
|
||||
send@0.19.2:
|
||||
resolution: {integrity: sha512-VMbMxbDeehAxpOtWJXlcUS5E8iXh6QmN+BkRX1GARS3wRaXEEgzCcB10gTQazO42tpNIya8xIyNx8fll1OFPrg==}
|
||||
engines: {node: '>= 0.8.0'}
|
||||
@@ -7137,14 +7198,14 @@ snapshots:
|
||||
- chokidar
|
||||
- typescript
|
||||
|
||||
'@angular-builders/custom-webpack@21.0.3(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.14)(typescript@5.9.3))(@angular/compiler@21.2.14)(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/localize@21.2.14(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.14)(typescript@5.9.3))(@angular/compiler@21.2.14))(@angular/platform-browser@21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)))(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)(@types/node@25.9.1)(chokidar@5.0.0)(jest-environment-jsdom@30.4.1(canvas@3.0.0))(jest@30.4.2(@types/node@25.9.1)(ts-node@10.9.2(@types/node@25.9.1)(typescript@5.9.3)))(jiti@2.6.1)(less@4.4.2)(postcss@8.5.15)(terser@5.44.1)(tslib@2.8.1)(typescript@5.9.3)(yaml@2.7.0)':
|
||||
'@angular-builders/custom-webpack@21.0.3(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.17)(typescript@5.9.3))(@angular/compiler@21.2.17)(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/localize@21.2.14(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.17)(typescript@5.9.3))(@angular/compiler@21.2.17))(@angular/platform-browser@21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)))(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)(@types/node@25.9.1)(chokidar@5.0.0)(jest-environment-jsdom@30.4.1(canvas@3.0.0))(jest@30.4.2(@types/node@25.9.1)(ts-node@10.9.2(@types/node@25.9.1)(typescript@5.9.3)))(jiti@2.6.1)(less@4.4.2)(postcss@8.5.15)(terser@5.44.1)(tslib@2.8.1)(typescript@5.9.3)(yaml@2.7.0)':
|
||||
dependencies:
|
||||
'@angular-builders/common': 5.0.3(@types/node@25.9.1)(chokidar@5.0.0)(typescript@5.9.3)
|
||||
'@angular-devkit/architect': 0.2101.2(chokidar@5.0.0)
|
||||
'@angular-devkit/build-angular': 21.1.2(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.14)(typescript@5.9.3))(@angular/compiler@21.2.14)(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/localize@21.2.14(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.14)(typescript@5.9.3))(@angular/compiler@21.2.14))(@angular/platform-browser@21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)))(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)(@types/node@25.9.1)(chokidar@5.0.0)(jest-environment-jsdom@30.4.1(canvas@3.0.0))(jest@30.4.2(@types/node@25.9.1)(ts-node@10.9.2(@types/node@25.9.1)(typescript@5.9.3)))(jiti@2.6.1)(typescript@5.9.3)(yaml@2.7.0)
|
||||
'@angular-devkit/build-angular': 21.1.2(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.17)(typescript@5.9.3))(@angular/compiler@21.2.17)(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/localize@21.2.14(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.17)(typescript@5.9.3))(@angular/compiler@21.2.17))(@angular/platform-browser@21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)))(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)(@types/node@25.9.1)(chokidar@5.0.0)(jest-environment-jsdom@30.4.1(canvas@3.0.0))(jest@30.4.2(@types/node@25.9.1)(ts-node@10.9.2(@types/node@25.9.1)(typescript@5.9.3)))(jiti@2.6.1)(typescript@5.9.3)(yaml@2.7.0)
|
||||
'@angular-devkit/core': 21.2.12(chokidar@5.0.0)
|
||||
'@angular/build': 21.2.12(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.14)(typescript@5.9.3))(@angular/compiler@21.2.14)(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/localize@21.2.14(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.14)(typescript@5.9.3))(@angular/compiler@21.2.14))(@angular/platform-browser@21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)))(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)(@types/node@25.9.1)(chokidar@5.0.0)(jiti@2.6.1)(less@4.4.2)(postcss@8.5.15)(terser@5.44.1)(tslib@2.8.1)(typescript@5.9.3)(yaml@2.7.0)
|
||||
'@angular/compiler-cli': 21.2.14(@angular/compiler@21.2.14)(typescript@5.9.3)
|
||||
'@angular/build': 21.2.12(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.17)(typescript@5.9.3))(@angular/compiler@21.2.17)(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/localize@21.2.14(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.17)(typescript@5.9.3))(@angular/compiler@21.2.17))(@angular/platform-browser@21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)))(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)(@types/node@25.9.1)(chokidar@5.0.0)(jiti@2.6.1)(less@4.4.2)(postcss@8.5.15)(terser@5.44.1)(tslib@2.8.1)(typescript@5.9.3)(yaml@2.7.0)
|
||||
'@angular/compiler-cli': 21.2.14(@angular/compiler@21.2.17)(typescript@5.9.3)
|
||||
lodash: 4.18.1
|
||||
webpack-merge: 6.0.1
|
||||
transitivePeerDependencies:
|
||||
@@ -7199,17 +7260,17 @@ snapshots:
|
||||
- webpack-cli
|
||||
- yaml
|
||||
|
||||
'@angular-builders/jest@21.0.3(45beaf077858833b14ba9080c452c7e9)':
|
||||
'@angular-builders/jest@21.0.3(6a682f4f002a31c8d3b52779d7b9ab9b)':
|
||||
dependencies:
|
||||
'@angular-builders/common': 5.0.3(@types/node@25.9.1)(chokidar@5.0.0)(typescript@5.9.3)
|
||||
'@angular-devkit/architect': 0.2101.2(chokidar@5.0.0)
|
||||
'@angular-devkit/build-angular': 21.1.2(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.14)(typescript@5.9.3))(@angular/compiler@21.2.14)(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/localize@21.2.14(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.14)(typescript@5.9.3))(@angular/compiler@21.2.14))(@angular/platform-browser@21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)))(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)(@types/node@25.9.1)(chokidar@5.0.0)(jest-environment-jsdom@30.4.1(canvas@3.0.0))(jest@30.4.2(@types/node@25.9.1)(ts-node@10.9.2(@types/node@25.9.1)(typescript@5.9.3)))(jiti@2.6.1)(typescript@5.9.3)(yaml@2.7.0)
|
||||
'@angular-devkit/build-angular': 21.1.2(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.17)(typescript@5.9.3))(@angular/compiler@21.2.17)(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/localize@21.2.14(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.17)(typescript@5.9.3))(@angular/compiler@21.2.17))(@angular/platform-browser@21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)))(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)(@types/node@25.9.1)(chokidar@5.0.0)(jest-environment-jsdom@30.4.1(canvas@3.0.0))(jest@30.4.2(@types/node@25.9.1)(ts-node@10.9.2(@types/node@25.9.1)(typescript@5.9.3)))(jiti@2.6.1)(typescript@5.9.3)(yaml@2.7.0)
|
||||
'@angular-devkit/core': 21.2.12(chokidar@5.0.0)
|
||||
'@angular/compiler-cli': 21.2.14(@angular/compiler@21.2.14)(typescript@5.9.3)
|
||||
'@angular/core': 21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)
|
||||
'@angular/platform-browser-dynamic': 21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/compiler@21.2.14)(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/platform-browser@21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)))
|
||||
'@angular/compiler-cli': 21.2.14(@angular/compiler@21.2.17)(typescript@5.9.3)
|
||||
'@angular/core': 21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)
|
||||
'@angular/platform-browser-dynamic': 21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/compiler@21.2.17)(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/platform-browser@21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)))
|
||||
jest: 30.4.2(@types/node@25.9.1)(ts-node@10.9.2(@types/node@25.9.1)(typescript@5.9.3))
|
||||
jest-preset-angular: 16.1.5(43a2e4c530b4286e50e732293015d944)
|
||||
jest-preset-angular: 16.1.5(26662f94407112e0967a16d5ea795956)
|
||||
lodash: 4.18.1
|
||||
transitivePeerDependencies:
|
||||
- '@angular/platform-browser'
|
||||
@@ -7246,14 +7307,14 @@ snapshots:
|
||||
transitivePeerDependencies:
|
||||
- chokidar
|
||||
|
||||
'@angular-devkit/build-angular@21.1.2(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.14)(typescript@5.9.3))(@angular/compiler@21.2.14)(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/localize@21.2.14(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.14)(typescript@5.9.3))(@angular/compiler@21.2.14))(@angular/platform-browser@21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)))(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)(@types/node@25.9.1)(chokidar@5.0.0)(jest-environment-jsdom@30.4.1(canvas@3.0.0))(jest@30.4.2(@types/node@25.9.1)(ts-node@10.9.2(@types/node@25.9.1)(typescript@5.9.3)))(jiti@2.6.1)(typescript@5.9.3)(yaml@2.7.0)':
|
||||
'@angular-devkit/build-angular@21.1.2(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.17)(typescript@5.9.3))(@angular/compiler@21.2.17)(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/localize@21.2.14(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.17)(typescript@5.9.3))(@angular/compiler@21.2.17))(@angular/platform-browser@21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)))(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)(@types/node@25.9.1)(chokidar@5.0.0)(jest-environment-jsdom@30.4.1(canvas@3.0.0))(jest@30.4.2(@types/node@25.9.1)(ts-node@10.9.2(@types/node@25.9.1)(typescript@5.9.3)))(jiti@2.6.1)(typescript@5.9.3)(yaml@2.7.0)':
|
||||
dependencies:
|
||||
'@ampproject/remapping': 2.3.0
|
||||
'@angular-devkit/architect': 0.2101.2(chokidar@5.0.0)
|
||||
'@angular-devkit/build-webpack': 0.2101.2(chokidar@5.0.0)(webpack-dev-server@5.2.2(tslib@2.8.1)(webpack@5.107.2(postcss@8.5.15)))(webpack@5.104.1(esbuild@0.27.2)(postcss@8.5.6))
|
||||
'@angular-devkit/core': 21.1.2(chokidar@5.0.0)
|
||||
'@angular/build': 21.1.2(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.14)(typescript@5.9.3))(@angular/compiler@21.2.14)(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/localize@21.2.14(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.14)(typescript@5.9.3))(@angular/compiler@21.2.14))(@angular/platform-browser@21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)))(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)(@types/node@25.9.1)(chokidar@5.0.0)(jiti@2.6.1)(less@4.4.2)(postcss@8.5.6)(terser@5.44.1)(tslib@2.8.1)(typescript@5.9.3)(yaml@2.7.0)
|
||||
'@angular/compiler-cli': 21.2.14(@angular/compiler@21.2.14)(typescript@5.9.3)
|
||||
'@angular/build': 21.1.2(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.17)(typescript@5.9.3))(@angular/compiler@21.2.17)(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/localize@21.2.14(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.17)(typescript@5.9.3))(@angular/compiler@21.2.17))(@angular/platform-browser@21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)))(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)(@types/node@25.9.1)(chokidar@5.0.0)(jiti@2.6.1)(less@4.4.2)(postcss@8.5.6)(terser@5.44.1)(tslib@2.8.1)(typescript@5.9.3)(yaml@2.7.0)
|
||||
'@angular/compiler-cli': 21.2.14(@angular/compiler@21.2.17)(typescript@5.9.3)
|
||||
'@babel/core': 7.28.5
|
||||
'@babel/generator': 7.28.5
|
||||
'@babel/helper-annotate-as-pure': 7.27.3
|
||||
@@ -7264,7 +7325,7 @@ snapshots:
|
||||
'@babel/preset-env': 7.28.5(@babel/core@7.28.5)
|
||||
'@babel/runtime': 7.28.4
|
||||
'@discoveryjs/json-ext': 0.6.3
|
||||
'@ngtools/webpack': 21.1.2(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.14)(typescript@5.9.3))(typescript@5.9.3)(webpack@5.104.1(esbuild@0.27.2)(postcss@8.5.6))
|
||||
'@ngtools/webpack': 21.1.2(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.17)(typescript@5.9.3))(typescript@5.9.3)(webpack@5.104.1(esbuild@0.27.2)(postcss@8.5.6))
|
||||
ansi-colors: 4.1.3
|
||||
autoprefixer: 10.4.23(postcss@8.5.6)
|
||||
babel-loader: 10.0.0(@babel/core@7.28.5)(webpack@5.104.1(esbuild@0.27.2)(postcss@8.5.6))
|
||||
@@ -7305,9 +7366,9 @@ snapshots:
|
||||
webpack-merge: 6.0.1
|
||||
webpack-subresource-integrity: 5.1.0(webpack@5.104.1(esbuild@0.27.2)(postcss@8.5.6))
|
||||
optionalDependencies:
|
||||
'@angular/core': 21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)
|
||||
'@angular/localize': 21.2.14(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.14)(typescript@5.9.3))(@angular/compiler@21.2.14)
|
||||
'@angular/platform-browser': 21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))
|
||||
'@angular/core': 21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)
|
||||
'@angular/localize': 21.2.14(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.17)(typescript@5.9.3))(@angular/compiler@21.2.17)
|
||||
'@angular/platform-browser': 21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))
|
||||
esbuild: 0.27.2
|
||||
jest: 30.4.2(@types/node@25.9.1)(ts-node@10.9.2(@types/node@25.9.1)(typescript@5.9.3))
|
||||
jest-environment-jsdom: 30.4.1(canvas@3.0.0)
|
||||
@@ -7460,12 +7521,12 @@ snapshots:
|
||||
eslint: 10.4.0(jiti@2.6.1)
|
||||
typescript: 5.9.3
|
||||
|
||||
'@angular/build@21.1.2(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.14)(typescript@5.9.3))(@angular/compiler@21.2.14)(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/localize@21.2.14(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.14)(typescript@5.9.3))(@angular/compiler@21.2.14))(@angular/platform-browser@21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)))(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)(@types/node@25.9.1)(chokidar@5.0.0)(jiti@2.6.1)(less@4.4.2)(postcss@8.5.6)(terser@5.44.1)(tslib@2.8.1)(typescript@5.9.3)(yaml@2.7.0)':
|
||||
'@angular/build@21.1.2(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.17)(typescript@5.9.3))(@angular/compiler@21.2.17)(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/localize@21.2.14(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.17)(typescript@5.9.3))(@angular/compiler@21.2.17))(@angular/platform-browser@21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)))(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)(@types/node@25.9.1)(chokidar@5.0.0)(jiti@2.6.1)(less@4.4.2)(postcss@8.5.6)(terser@5.44.1)(tslib@2.8.1)(typescript@5.9.3)(yaml@2.7.0)':
|
||||
dependencies:
|
||||
'@ampproject/remapping': 2.3.0
|
||||
'@angular-devkit/architect': 0.2101.2(chokidar@5.0.0)
|
||||
'@angular/compiler': 21.2.14
|
||||
'@angular/compiler-cli': 21.2.14(@angular/compiler@21.2.14)(typescript@5.9.3)
|
||||
'@angular/compiler': 21.2.17
|
||||
'@angular/compiler-cli': 21.2.14(@angular/compiler@21.2.17)(typescript@5.9.3)
|
||||
'@babel/core': 7.28.5
|
||||
'@babel/helper-annotate-as-pure': 7.27.3
|
||||
'@babel/helper-split-export-declaration': 7.24.7
|
||||
@@ -7494,9 +7555,9 @@ snapshots:
|
||||
vite: 7.3.0(@types/node@25.9.1)(jiti@2.6.1)(less@4.4.2)(sass@1.97.1)(terser@5.44.1)(yaml@2.7.0)
|
||||
watchpack: 2.5.0
|
||||
optionalDependencies:
|
||||
'@angular/core': 21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)
|
||||
'@angular/localize': 21.2.14(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.14)(typescript@5.9.3))(@angular/compiler@21.2.14)
|
||||
'@angular/platform-browser': 21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))
|
||||
'@angular/core': 21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)
|
||||
'@angular/localize': 21.2.14(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.17)(typescript@5.9.3))(@angular/compiler@21.2.17)
|
||||
'@angular/platform-browser': 21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))
|
||||
less: 4.4.2
|
||||
lmdb: 3.4.4
|
||||
postcss: 8.5.6
|
||||
@@ -7515,12 +7576,12 @@ snapshots:
|
||||
- tsx
|
||||
- yaml
|
||||
|
||||
'@angular/build@21.2.12(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.14)(typescript@5.9.3))(@angular/compiler@21.2.14)(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/localize@21.2.14(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.14)(typescript@5.9.3))(@angular/compiler@21.2.14))(@angular/platform-browser@21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)))(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)(@types/node@25.9.1)(chokidar@5.0.0)(jiti@2.6.1)(less@4.4.2)(postcss@8.5.15)(terser@5.44.1)(tslib@2.8.1)(typescript@5.9.3)(yaml@2.7.0)':
|
||||
'@angular/build@21.2.12(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.17)(typescript@5.9.3))(@angular/compiler@21.2.17)(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/localize@21.2.14(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.17)(typescript@5.9.3))(@angular/compiler@21.2.17))(@angular/platform-browser@21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)))(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)(@types/node@25.9.1)(chokidar@5.0.0)(jiti@2.6.1)(less@4.4.2)(postcss@8.5.15)(terser@5.44.1)(tslib@2.8.1)(typescript@5.9.3)(yaml@2.7.0)':
|
||||
dependencies:
|
||||
'@ampproject/remapping': 2.3.0
|
||||
'@angular-devkit/architect': 0.2102.12(chokidar@5.0.0)
|
||||
'@angular/compiler': 21.2.14
|
||||
'@angular/compiler-cli': 21.2.14(@angular/compiler@21.2.14)(typescript@5.9.3)
|
||||
'@angular/compiler': 21.2.17
|
||||
'@angular/compiler-cli': 21.2.14(@angular/compiler@21.2.17)(typescript@5.9.3)
|
||||
'@babel/core': 7.29.0
|
||||
'@babel/helper-annotate-as-pure': 7.27.3
|
||||
'@babel/helper-split-export-declaration': 7.24.7
|
||||
@@ -7549,9 +7610,9 @@ snapshots:
|
||||
vite: 7.3.2(@types/node@25.9.1)(jiti@2.6.1)(less@4.4.2)(sass@1.97.3)(terser@5.44.1)(yaml@2.7.0)
|
||||
watchpack: 2.5.1
|
||||
optionalDependencies:
|
||||
'@angular/core': 21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)
|
||||
'@angular/localize': 21.2.14(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.14)(typescript@5.9.3))(@angular/compiler@21.2.14)
|
||||
'@angular/platform-browser': 21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))
|
||||
'@angular/core': 21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)
|
||||
'@angular/localize': 21.2.14(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.17)(typescript@5.9.3))(@angular/compiler@21.2.17)
|
||||
'@angular/platform-browser': 21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))
|
||||
less: 4.4.2
|
||||
lmdb: 3.5.1
|
||||
postcss: 8.5.15
|
||||
@@ -7570,11 +7631,11 @@ snapshots:
|
||||
- tsx
|
||||
- yaml
|
||||
|
||||
'@angular/cdk@21.2.12(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/platform-browser@21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)))(rxjs@7.8.2)':
|
||||
'@angular/cdk@21.2.12(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/platform-browser@21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)))(rxjs@7.8.2)':
|
||||
dependencies:
|
||||
'@angular/common': 21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2)
|
||||
'@angular/core': 21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)
|
||||
'@angular/platform-browser': 21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))
|
||||
'@angular/common': 21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2)
|
||||
'@angular/core': 21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)
|
||||
'@angular/platform-browser': 21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))
|
||||
parse5: 8.0.1
|
||||
rxjs: 7.8.2
|
||||
tslib: 2.8.1
|
||||
@@ -7605,15 +7666,15 @@ snapshots:
|
||||
- chokidar
|
||||
- supports-color
|
||||
|
||||
'@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2)':
|
||||
'@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2)':
|
||||
dependencies:
|
||||
'@angular/core': 21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)
|
||||
'@angular/core': 21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)
|
||||
rxjs: 7.8.2
|
||||
tslib: 2.8.1
|
||||
|
||||
'@angular/compiler-cli@21.2.14(@angular/compiler@21.2.14)(typescript@5.9.3)':
|
||||
'@angular/compiler-cli@21.2.14(@angular/compiler@21.2.17)(typescript@5.9.3)':
|
||||
dependencies:
|
||||
'@angular/compiler': 21.2.14
|
||||
'@angular/compiler': 21.2.17
|
||||
'@babel/core': 7.29.0
|
||||
'@jridgewell/sourcemap-codec': 1.5.5
|
||||
chokidar: 5.0.0
|
||||
@@ -7627,31 +7688,31 @@ snapshots:
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
|
||||
'@angular/compiler@21.2.14':
|
||||
'@angular/compiler@21.2.17':
|
||||
dependencies:
|
||||
tslib: 2.8.1
|
||||
|
||||
'@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)':
|
||||
'@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)':
|
||||
dependencies:
|
||||
rxjs: 7.8.2
|
||||
tslib: 2.8.1
|
||||
optionalDependencies:
|
||||
'@angular/compiler': 21.2.14
|
||||
'@angular/compiler': 21.2.17
|
||||
zone.js: 0.16.2
|
||||
|
||||
'@angular/forms@21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/platform-browser@21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)))(rxjs@7.8.2)':
|
||||
'@angular/forms@21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/platform-browser@21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)))(rxjs@7.8.2)':
|
||||
dependencies:
|
||||
'@angular/common': 21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2)
|
||||
'@angular/core': 21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)
|
||||
'@angular/platform-browser': 21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))
|
||||
'@angular/common': 21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2)
|
||||
'@angular/core': 21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)
|
||||
'@angular/platform-browser': 21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))
|
||||
'@standard-schema/spec': 1.1.0
|
||||
rxjs: 7.8.2
|
||||
tslib: 2.8.1
|
||||
|
||||
'@angular/localize@21.2.14(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.14)(typescript@5.9.3))(@angular/compiler@21.2.14)':
|
||||
'@angular/localize@21.2.14(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.17)(typescript@5.9.3))(@angular/compiler@21.2.17)':
|
||||
dependencies:
|
||||
'@angular/compiler': 21.2.14
|
||||
'@angular/compiler-cli': 21.2.14(@angular/compiler@21.2.14)(typescript@5.9.3)
|
||||
'@angular/compiler': 21.2.17
|
||||
'@angular/compiler-cli': 21.2.14(@angular/compiler@21.2.17)(typescript@5.9.3)
|
||||
'@babel/core': 7.29.0
|
||||
'@types/babel__core': 7.20.5
|
||||
tinyglobby: 0.2.17
|
||||
@@ -7659,25 +7720,25 @@ snapshots:
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
|
||||
'@angular/platform-browser-dynamic@21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/compiler@21.2.14)(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/platform-browser@21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)))':
|
||||
'@angular/platform-browser-dynamic@21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/compiler@21.2.17)(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/platform-browser@21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)))':
|
||||
dependencies:
|
||||
'@angular/common': 21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2)
|
||||
'@angular/compiler': 21.2.14
|
||||
'@angular/core': 21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)
|
||||
'@angular/platform-browser': 21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))
|
||||
'@angular/common': 21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2)
|
||||
'@angular/compiler': 21.2.17
|
||||
'@angular/core': 21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)
|
||||
'@angular/platform-browser': 21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))
|
||||
tslib: 2.8.1
|
||||
|
||||
'@angular/platform-browser@21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))':
|
||||
'@angular/platform-browser@21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))':
|
||||
dependencies:
|
||||
'@angular/common': 21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2)
|
||||
'@angular/core': 21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)
|
||||
'@angular/common': 21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2)
|
||||
'@angular/core': 21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)
|
||||
tslib: 2.8.1
|
||||
|
||||
'@angular/router@21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/platform-browser@21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)))(rxjs@7.8.2)':
|
||||
'@angular/router@21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/platform-browser@21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)))(rxjs@7.8.2)':
|
||||
dependencies:
|
||||
'@angular/common': 21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2)
|
||||
'@angular/core': 21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)
|
||||
'@angular/platform-browser': 21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))
|
||||
'@angular/common': 21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2)
|
||||
'@angular/core': 21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)
|
||||
'@angular/platform-browser': 21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))
|
||||
rxjs: 7.8.2
|
||||
tslib: 2.8.1
|
||||
|
||||
@@ -9618,35 +9679,35 @@ snapshots:
|
||||
'@tybys/wasm-util': 0.10.2
|
||||
optional: true
|
||||
|
||||
'@ng-bootstrap/ng-bootstrap@20.0.0(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/forms@21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/platform-browser@21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)))(rxjs@7.8.2))(@angular/localize@21.2.14(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.14)(typescript@5.9.3))(@angular/compiler@21.2.14))(@popperjs/core@2.11.8)(rxjs@7.8.2)':
|
||||
'@ng-bootstrap/ng-bootstrap@20.0.0(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/forms@21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/platform-browser@21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)))(rxjs@7.8.2))(@angular/localize@21.2.14(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.17)(typescript@5.9.3))(@angular/compiler@21.2.17))(@popperjs/core@2.11.8)(rxjs@7.8.2)':
|
||||
dependencies:
|
||||
'@angular/common': 21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2)
|
||||
'@angular/core': 21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)
|
||||
'@angular/forms': 21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/platform-browser@21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)))(rxjs@7.8.2)
|
||||
'@angular/localize': 21.2.14(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.14)(typescript@5.9.3))(@angular/compiler@21.2.14)
|
||||
'@angular/common': 21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2)
|
||||
'@angular/core': 21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)
|
||||
'@angular/forms': 21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/platform-browser@21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)))(rxjs@7.8.2)
|
||||
'@angular/localize': 21.2.14(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.17)(typescript@5.9.3))(@angular/compiler@21.2.17)
|
||||
'@popperjs/core': 2.11.8
|
||||
rxjs: 7.8.2
|
||||
tslib: 2.8.1
|
||||
|
||||
'@ng-select/ng-select@21.8.2(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/forms@21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/platform-browser@21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)))(rxjs@7.8.2))':
|
||||
'@ng-select/ng-select@21.8.2(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/forms@21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/platform-browser@21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)))(rxjs@7.8.2))':
|
||||
dependencies:
|
||||
'@angular/common': 21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2)
|
||||
'@angular/core': 21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)
|
||||
'@angular/forms': 21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/platform-browser@21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)))(rxjs@7.8.2)
|
||||
'@angular/common': 21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2)
|
||||
'@angular/core': 21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)
|
||||
'@angular/forms': 21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/platform-browser@21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)))(rxjs@7.8.2)
|
||||
tslib: 2.8.1
|
||||
|
||||
'@ngneat/dirty-check-forms@3.0.3(be5de60320c5c6a3310af74f068bbe95)':
|
||||
'@ngneat/dirty-check-forms@3.0.3(ad2c8ff51b8ef8626e139c84727a024d)':
|
||||
dependencies:
|
||||
'@angular/core': 21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)
|
||||
'@angular/forms': 21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/platform-browser@21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)))(rxjs@7.8.2)
|
||||
'@angular/router': 21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/platform-browser@21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)))(rxjs@7.8.2)
|
||||
'@angular/core': 21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)
|
||||
'@angular/forms': 21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/platform-browser@21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)))(rxjs@7.8.2)
|
||||
'@angular/router': 21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/platform-browser@21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)))(rxjs@7.8.2)
|
||||
lodash-es: 4.17.21
|
||||
rxjs: 7.8.2
|
||||
tslib: 2.8.1
|
||||
|
||||
'@ngtools/webpack@21.1.2(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.14)(typescript@5.9.3))(typescript@5.9.3)(webpack@5.104.1(esbuild@0.27.2)(postcss@8.5.6))':
|
||||
'@ngtools/webpack@21.1.2(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.17)(typescript@5.9.3))(typescript@5.9.3)(webpack@5.104.1(esbuild@0.27.2)(postcss@8.5.6))':
|
||||
dependencies:
|
||||
'@angular/compiler-cli': 21.2.14(@angular/compiler@21.2.14)(typescript@5.9.3)
|
||||
'@angular/compiler-cli': 21.2.14(@angular/compiler@21.2.17)(typescript@5.9.3)
|
||||
typescript: 5.9.3
|
||||
webpack: 5.104.1(esbuild@0.27.2)(postcss@8.5.6)
|
||||
|
||||
@@ -12223,12 +12284,12 @@ snapshots:
|
||||
optionalDependencies:
|
||||
jest-resolve: 30.4.1
|
||||
|
||||
jest-preset-angular@16.1.5(43a2e4c530b4286e50e732293015d944):
|
||||
jest-preset-angular@16.1.5(26662f94407112e0967a16d5ea795956):
|
||||
dependencies:
|
||||
'@angular/compiler-cli': 21.2.14(@angular/compiler@21.2.14)(typescript@5.9.3)
|
||||
'@angular/core': 21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)
|
||||
'@angular/platform-browser': 21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))
|
||||
'@angular/platform-browser-dynamic': 21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/compiler@21.2.14)(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/platform-browser@21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)))
|
||||
'@angular/compiler-cli': 21.2.14(@angular/compiler@21.2.17)(typescript@5.9.3)
|
||||
'@angular/core': 21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)
|
||||
'@angular/platform-browser': 21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))
|
||||
'@angular/platform-browser-dynamic': 21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/compiler@21.2.17)(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/platform-browser@21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)))
|
||||
'@jest/environment-jsdom-abstract': 30.4.1(canvas@3.0.0)(jsdom@26.1.0(canvas@3.0.0))
|
||||
bs-logger: 0.2.6
|
||||
esbuild-wasm: 0.28.0
|
||||
@@ -12847,46 +12908,46 @@ snapshots:
|
||||
|
||||
neo-async@2.6.2: {}
|
||||
|
||||
ngx-bootstrap-icons@1.9.3(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)):
|
||||
ngx-bootstrap-icons@1.9.3(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)):
|
||||
dependencies:
|
||||
'@angular/common': 21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2)
|
||||
'@angular/core': 21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)
|
||||
'@angular/common': 21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2)
|
||||
'@angular/core': 21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)
|
||||
tslib: 2.8.1
|
||||
|
||||
ngx-color@10.1.0(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)):
|
||||
ngx-color@10.1.0(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)):
|
||||
dependencies:
|
||||
'@angular/common': 21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2)
|
||||
'@angular/core': 21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)
|
||||
'@angular/common': 21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2)
|
||||
'@angular/core': 21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)
|
||||
'@ctrl/tinycolor': 4.2.0
|
||||
material-colors: 1.2.6
|
||||
tslib: 2.8.1
|
||||
|
||||
ngx-cookie-service@21.3.1(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)):
|
||||
ngx-cookie-service@21.3.1(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)):
|
||||
dependencies:
|
||||
'@angular/common': 21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2)
|
||||
'@angular/core': 21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)
|
||||
'@angular/common': 21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2)
|
||||
'@angular/core': 21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)
|
||||
tslib: 2.8.1
|
||||
|
||||
ngx-device-detector@11.0.0(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)):
|
||||
ngx-device-detector@11.0.0(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)):
|
||||
dependencies:
|
||||
'@angular/common': 21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2)
|
||||
'@angular/core': 21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)
|
||||
'@angular/common': 21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2)
|
||||
'@angular/core': 21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)
|
||||
tslib: 2.8.1
|
||||
|
||||
ngx-ui-tour-core@16.0.0(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/router@21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/platform-browser@21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)))(rxjs@7.8.2))(rxjs@7.8.2):
|
||||
ngx-ui-tour-core@16.0.0(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/router@21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/platform-browser@21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)))(rxjs@7.8.2))(rxjs@7.8.2):
|
||||
dependencies:
|
||||
'@angular/common': 21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2)
|
||||
'@angular/core': 21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)
|
||||
'@angular/router': 21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/platform-browser@21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)))(rxjs@7.8.2)
|
||||
'@angular/common': 21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2)
|
||||
'@angular/core': 21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)
|
||||
'@angular/router': 21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/platform-browser@21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)))(rxjs@7.8.2)
|
||||
rxjs: 7.8.2
|
||||
tslib: 2.8.1
|
||||
|
||||
ngx-ui-tour-ng-bootstrap@18.0.0(f910a33494d223bd6dd07ce1bf22a35e):
|
||||
ngx-ui-tour-ng-bootstrap@18.0.0(4ccfccfbcf381a309618492b31e99276):
|
||||
dependencies:
|
||||
'@angular/common': 21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2)
|
||||
'@angular/core': 21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)
|
||||
'@ng-bootstrap/ng-bootstrap': 20.0.0(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/forms@21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/platform-browser@21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)))(rxjs@7.8.2))(@angular/localize@21.2.14(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.14)(typescript@5.9.3))(@angular/compiler@21.2.14))(@popperjs/core@2.11.8)(rxjs@7.8.2)
|
||||
ngx-ui-tour-core: 16.0.0(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/router@21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/platform-browser@21.2.14(@angular/common@21.2.14(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.14(@angular/compiler@21.2.14)(rxjs@7.8.2)(zone.js@0.16.2)))(rxjs@7.8.2))(rxjs@7.8.2)
|
||||
'@angular/common': 21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2)
|
||||
'@angular/core': 21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)
|
||||
'@ng-bootstrap/ng-bootstrap': 20.0.0(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/forms@21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/platform-browser@21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)))(rxjs@7.8.2))(@angular/localize@21.2.14(@angular/compiler-cli@21.2.14(@angular/compiler@21.2.17)(typescript@5.9.3))(@angular/compiler@21.2.17))(@popperjs/core@2.11.8)(rxjs@7.8.2)
|
||||
ngx-ui-tour-core: 16.0.0(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/router@21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(@angular/platform-browser@21.2.14(@angular/common@21.2.17(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2))(rxjs@7.8.2))(@angular/core@21.2.17(@angular/compiler@21.2.17)(rxjs@7.8.2)(zone.js@0.16.2)))(rxjs@7.8.2))(rxjs@7.8.2)
|
||||
tslib: 2.8.1
|
||||
transitivePeerDependencies:
|
||||
- '@angular/router'
|
||||
@@ -12894,7 +12955,7 @@ snapshots:
|
||||
|
||||
node-abi@3.92.0:
|
||||
dependencies:
|
||||
semver: 7.8.1
|
||||
semver: 7.8.4
|
||||
optional: true
|
||||
|
||||
node-addon-api@6.1.0:
|
||||
@@ -12931,6 +12992,10 @@ snapshots:
|
||||
dependencies:
|
||||
abbrev: 4.0.0
|
||||
|
||||
normalize-diacritics@5.0.0:
|
||||
dependencies:
|
||||
tslib: 2.8.1
|
||||
|
||||
normalize-path@3.0.0: {}
|
||||
|
||||
npm-bundled@5.0.0:
|
||||
@@ -13610,6 +13675,9 @@ snapshots:
|
||||
|
||||
semver@7.8.1: {}
|
||||
|
||||
semver@7.8.4:
|
||||
optional: true
|
||||
|
||||
send@0.19.2:
|
||||
dependencies:
|
||||
debug: 2.6.9
|
||||
|
||||
@@ -188,4 +188,14 @@ describe('ChatComponent', () => {
|
||||
component.searchInputKeyDown(event)
|
||||
expect(component.sendMessage).toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('should not send message on Enter key press while composing with IME', () => {
|
||||
jest.spyOn(component, 'sendMessage')
|
||||
const event = new KeyboardEvent('keydown', {
|
||||
key: 'Enter',
|
||||
isComposing: true,
|
||||
})
|
||||
component.searchInputKeyDown(event)
|
||||
expect(component.sendMessage).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -155,7 +155,10 @@ export class ChatComponent implements OnInit {
|
||||
}
|
||||
|
||||
public searchInputKeyDown(event: KeyboardEvent) {
|
||||
if (event.key === 'Enter') {
|
||||
if (
|
||||
event.key === 'Enter' &&
|
||||
!(event.isComposing || event.keyCode === 229)
|
||||
) {
|
||||
event.preventDefault()
|
||||
this.sendMessage()
|
||||
}
|
||||
|
||||
+2
-3
@@ -23,6 +23,7 @@ import {
|
||||
import { CustomFieldsService } from 'src/app/services/rest/custom-fields.service'
|
||||
import { ToastService } from 'src/app/services/toast.service'
|
||||
import { pngxPopperOptions } from 'src/app/utils/popper-options'
|
||||
import { matchesSearchText } from 'src/app/utils/text-search'
|
||||
import { LoadingComponentWithPermissions } from '../../loading-component/loading.component'
|
||||
import { CustomFieldEditDialogComponent } from '../edit-dialog/custom-field-edit-dialog/custom-field-edit-dialog.component'
|
||||
|
||||
@@ -69,9 +70,7 @@ export class CustomFieldsDropdownComponent extends LoadingComponentWithPermissio
|
||||
|
||||
public get filteredFields(): CustomField[] {
|
||||
return this.unusedFields.filter(
|
||||
(f) =>
|
||||
!this.filterText ||
|
||||
f.name.toLowerCase().includes(this.filterText.toLowerCase())
|
||||
(f) => !this.filterText || matchesSearchText(f.name, this.filterText)
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
+3
@@ -63,6 +63,7 @@
|
||||
[(ngModel)]="atom.value"
|
||||
[disabled]="disabled"
|
||||
[virtualScroll]="getSelectOptionsForField(atom.field)?.length > 100"
|
||||
[searchFn]="selectOptionSearchFn"
|
||||
(mousedown)="$event.stopImmediatePropagation()"
|
||||
></ng-select>
|
||||
} @else if (getCustomFieldByID(atom.field)?.data_type === CustomFieldDataType.DocumentLink) {
|
||||
@@ -81,6 +82,7 @@
|
||||
[disabled]="disabled"
|
||||
bindLabel="name"
|
||||
bindValue="id"
|
||||
[searchFn]="customFieldSearchFn"
|
||||
(mousedown)="$event.stopImmediatePropagation()"
|
||||
></ng-select>
|
||||
<select class="w-25 form-select" [(ngModel)]="atom.operator" [disabled]="disabled">
|
||||
@@ -125,6 +127,7 @@
|
||||
[(ngModel)]="atom.value"
|
||||
[disabled]="disabled"
|
||||
[multiple]="true"
|
||||
[searchFn]="selectOptionSearchFn"
|
||||
(mousedown)="$event.stopImmediatePropagation()"
|
||||
></ng-select>
|
||||
}
|
||||
|
||||
+9
@@ -36,6 +36,7 @@ import {
|
||||
CustomFieldQueryExpression,
|
||||
} from 'src/app/utils/custom-field-query-element'
|
||||
import { pngxPopperOptions } from 'src/app/utils/popper-options'
|
||||
import { matchesSearchText } from 'src/app/utils/text-search'
|
||||
import { LoadingComponentWithPermissions } from '../../loading-component/loading.component'
|
||||
import { ClearableBadgeComponent } from '../clearable-badge/clearable-badge.component'
|
||||
import { DocumentLinkComponent } from '../input/document-link/document-link.component'
|
||||
@@ -281,6 +282,14 @@ export class CustomFieldsQueryDropdownComponent extends LoadingComponentWithPerm
|
||||
|
||||
public readonly today: string = new Date().toLocaleDateString('en-CA')
|
||||
|
||||
public customFieldSearchFn = (term: string, field: CustomField): boolean =>
|
||||
matchesSearchText(field?.name, term)
|
||||
|
||||
public selectOptionSearchFn = (
|
||||
term: string,
|
||||
option: { id: string; label: string }
|
||||
): boolean => matchesSearchText(option?.label, term)
|
||||
|
||||
constructor() {
|
||||
super()
|
||||
this.selectionModel = new CustomFieldQueriesModel()
|
||||
|
||||
@@ -28,6 +28,7 @@
|
||||
[notFoundText]="notFoundText"
|
||||
[multiple]="multiple"
|
||||
[bindLabel]="bindLabel"
|
||||
[searchFn]="searchFn"
|
||||
bindValue="id"
|
||||
[virtualScroll]="items?.length > 100"
|
||||
(change)="onChange(value)"
|
||||
|
||||
@@ -112,6 +112,15 @@ describe('SelectComponent', () => {
|
||||
expect(createNewVal).toEqual('baz')
|
||||
})
|
||||
|
||||
it('should search items by independent normalized terms', () => {
|
||||
expect(
|
||||
component.searchFn('tax 26', { id: 11, name: 'Tax\u00e9s 2026' })
|
||||
).toBeTruthy()
|
||||
expect(
|
||||
component.searchFn('tax receipt', { id: 11, name: 'Tax\u00e9s 2026' })
|
||||
).toBeFalsy()
|
||||
})
|
||||
|
||||
it('should clear search term on blur after delay', fakeAsync(() => {
|
||||
const clearSpy = jest.spyOn(component, 'clearLastSearchTerm')
|
||||
component.onBlur()
|
||||
|
||||
@@ -13,6 +13,7 @@ import {
|
||||
import { RouterModule } from '@angular/router'
|
||||
import { NgSelectModule } from '@ng-select/ng-select'
|
||||
import { NgxBootstrapIconsModule } from 'ngx-bootstrap-icons'
|
||||
import { matchesSearchText } from 'src/app/utils/text-search'
|
||||
import { AbstractInputComponent } from '../abstract-input'
|
||||
|
||||
@Component({
|
||||
@@ -99,6 +100,9 @@ export class SelectComponent extends AbstractInputComponent<number> {
|
||||
@Input()
|
||||
bindLabel: string = 'name'
|
||||
|
||||
public searchFn = (term: string, item: any): boolean =>
|
||||
matchesSearchText(item?.[this.bindLabel], term)
|
||||
|
||||
@Input()
|
||||
showFilter: boolean = false
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
[clearSearchOnAdd]="true"
|
||||
[hideSelected]="tags.length > 0"
|
||||
[addTag]="allowCreate ? createTagRef : false"
|
||||
[searchFn]="searchFn"
|
||||
addTagText="Add tag"
|
||||
i18n-addTagText
|
||||
(add)="onAdd($event)"
|
||||
|
||||
@@ -171,6 +171,15 @@ describe('TagsComponent', () => {
|
||||
expect(component.getTag(4)).toBeUndefined()
|
||||
})
|
||||
|
||||
it('should search tags by independent normalized terms including parents', () => {
|
||||
const parent: Tag = { id: 11, name: 'Financ\u00e9' }
|
||||
const child: Tag = { id: 12, name: 'Taxes 2026', parent: parent.id }
|
||||
component.tags = [parent, child]
|
||||
|
||||
expect(component.searchFn('finance 26', child)).toBeTruthy()
|
||||
expect(component.searchFn('finance receipt', child)).toBeFalsy()
|
||||
})
|
||||
|
||||
it('should emit filtered documents', () => {
|
||||
component.value = [10]
|
||||
component.tags = tags
|
||||
|
||||
@@ -21,6 +21,7 @@ import { NgxBootstrapIconsModule } from 'ngx-bootstrap-icons'
|
||||
import { first, firstValueFrom, tap } from 'rxjs'
|
||||
import { Tag } from 'src/app/data/tag'
|
||||
import { TagService } from 'src/app/services/rest/tag.service'
|
||||
import { matchesSearchText } from 'src/app/utils/text-search'
|
||||
import { EditDialogMode } from '../../edit-dialog/edit-dialog.component'
|
||||
import { TagEditDialogComponent } from '../../edit-dialog/tag-edit-dialog/tag-edit-dialog.component'
|
||||
import { TagComponent } from '../../tag/tag.component'
|
||||
@@ -114,6 +115,14 @@ export class TagsComponent implements OnInit, ControlValueAccessor {
|
||||
|
||||
public createTagRef: (name) => void
|
||||
|
||||
public searchFn = (term: string, tag: Tag): boolean =>
|
||||
matchesSearchText(
|
||||
[this.getParentChain(tag?.id).map((parent) => parent.name), tag?.name]
|
||||
.flat()
|
||||
.join(' '),
|
||||
term
|
||||
)
|
||||
|
||||
getTag(id: number) {
|
||||
if (this.tags) {
|
||||
return this.tags.find((tag) => tag.id == id)
|
||||
|
||||
+3
-1
@@ -131,7 +131,9 @@
|
||||
@if (status.tasks.celery_status === 'OK') {
|
||||
<i-bs name="check-circle-fill" class="text-primary ms-2 lh-1"></i-bs>
|
||||
} @else {
|
||||
<i-bs name="exclamation-triangle-fill" class="text-danger ms-2 lh-1"></i-bs>
|
||||
<i-bs name="exclamation-triangle-fill" class="ms-2 lh-1"
|
||||
[class.text-danger]="status.tasks.celery_status === SystemStatusItemStatus.ERROR"
|
||||
[class.text-warning]="status.tasks.celery_status === SystemStatusItemStatus.WARNING"></i-bs>
|
||||
}
|
||||
</button>
|
||||
<ng-template #celeryStatus>
|
||||
|
||||
@@ -360,6 +360,14 @@ export const PaperlessConfigOptions: ConfigOption[] = [
|
||||
category: ConfigCategory.AI,
|
||||
note: $localize`Language to use for generated AI suggestions. When unset, AI suggestions use the user's display language if explicitly set.`,
|
||||
},
|
||||
{
|
||||
key: 'llm_request_timeout',
|
||||
title: $localize`LLM Request Timeout`,
|
||||
type: ConfigOptionType.Number,
|
||||
config_key: 'PAPERLESS_AI_LLM_REQUEST_TIMEOUT',
|
||||
category: ConfigCategory.AI,
|
||||
note: $localize`Timeout in seconds for LLM requests.`,
|
||||
},
|
||||
]
|
||||
|
||||
export interface PaperlessConfig extends ObjectWithId {
|
||||
@@ -401,4 +409,5 @@ export interface PaperlessConfig extends ObjectWithId {
|
||||
llm_api_key: string
|
||||
llm_endpoint: string
|
||||
llm_output_language: string
|
||||
llm_request_timeout: number
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { Pipe, PipeTransform } from '@angular/core'
|
||||
import { MatchingModel } from '../data/matching-model'
|
||||
import { matchesSearchText } from '../utils/text-search'
|
||||
|
||||
@Pipe({
|
||||
name: 'filter',
|
||||
@@ -21,9 +22,7 @@ export class FilterPipe implements PipeTransform {
|
||||
typeof item[key] === 'string' || typeof item[key] === 'number'
|
||||
)
|
||||
return keys.some((key) => {
|
||||
return String(item[key])
|
||||
.toLowerCase()
|
||||
.includes(searchText.toLowerCase())
|
||||
return matchesSearchText(item[key], searchText)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
import { matchesSearchText } from './text-search'
|
||||
|
||||
describe('text search utilities', () => {
|
||||
it('matches text accent-insensitively', () => {
|
||||
expect(matchesSearchText('R\u00e9sum\u00e9', 'resume')).toBeTruthy()
|
||||
expect(matchesSearchText('S\u00f8ren', 'soren')).toBeTruthy()
|
||||
expect(matchesSearchText('\u0152uvre', 'oeuvre')).toBeTruthy()
|
||||
expect(matchesSearchText('Invoice', 'receipt')).toBeFalsy()
|
||||
})
|
||||
|
||||
it('matches all whitespace-separated search terms independently', () => {
|
||||
expect(matchesSearchText('taxes 2026', 'tax 26')).toBeTruthy()
|
||||
expect(matchesSearchText('2026 taxes', 'tax 26')).toBeTruthy()
|
||||
expect(matchesSearchText('Tax\u00e9s 2026', 'taxe 26')).toBeTruthy()
|
||||
expect(matchesSearchText('taxes 2026', 'tax receipt')).toBeFalsy()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,23 @@
|
||||
import { normalizeSync } from 'normalize-diacritics'
|
||||
|
||||
export type SearchTextValue =
|
||||
| string
|
||||
| number
|
||||
| boolean
|
||||
| bigint
|
||||
| null
|
||||
| undefined
|
||||
|
||||
export function normalizeSearchText(value: SearchTextValue): string {
|
||||
return normalizeSync(String(value ?? '')).toLocaleLowerCase()
|
||||
}
|
||||
|
||||
export function matchesSearchText(
|
||||
value: SearchTextValue,
|
||||
searchText: SearchTextValue
|
||||
): boolean {
|
||||
const normalizedValue = normalizeSearchText(value)
|
||||
const searchTerms = normalizeSearchText(searchText).trim().split(/\s+/)
|
||||
|
||||
return searchTerms.every((term) => normalizedValue.includes(term))
|
||||
}
|
||||
@@ -70,13 +70,13 @@ def suggestions_last_modified(request, pk: int) -> datetime | None:
|
||||
|
||||
def metadata_etag(request, pk: int) -> str | None:
|
||||
"""
|
||||
Metadata is extracted from the original file, so use its checksum as the
|
||||
ETag
|
||||
Metadata responses include metadata as well as document fields, so include
|
||||
the modification time with the checksum so metadata-only changes invalidate cache.
|
||||
"""
|
||||
doc = resolve_effective_document_by_pk(pk, request).document
|
||||
if doc is None:
|
||||
return None
|
||||
return doc.checksum
|
||||
return f"{doc.checksum}:{doc.modified.isoformat()}"
|
||||
|
||||
|
||||
def metadata_last_modified(request, pk: int) -> datetime | None:
|
||||
|
||||
@@ -169,6 +169,10 @@ class FileStabilityTracker:
|
||||
self._tracked.pop(path, None)
|
||||
yield path
|
||||
|
||||
def is_tracking(self, path: Path) -> bool:
|
||||
"""Check whether a path is currently being tracked for stability."""
|
||||
return path.resolve() in self._tracked
|
||||
|
||||
def has_pending_files(self) -> bool:
|
||||
"""Check if there are files waiting for stability check."""
|
||||
return len(self._tracked) > 0
|
||||
@@ -370,6 +374,16 @@ class Command(BaseCommand):
|
||||
# Testing timeout in seconds
|
||||
testing_timeout_s: Final[float] = 0.5
|
||||
|
||||
# How often to perform a full-glob rescan of the consume directory as a
|
||||
# safety net. Each watchfiles watcher is torn down and recreated on every
|
||||
# batch to reconfigure its timeout, and a fresh watcher silently adopts the
|
||||
# current directory contents as its baseline. A file that appears between
|
||||
# one batch and the next watcher's baseline is therefore never reported and
|
||||
# would sit in the consume directory forever. This periodic rescan re-injects
|
||||
# such files into the stability tracker (see GH issue #13011). Not currently
|
||||
# user-configurable; instances may override for testing.
|
||||
rescan_interval_s: float = 300.0
|
||||
|
||||
def add_arguments(self, parser) -> None:
|
||||
parser.add_argument(
|
||||
"directory",
|
||||
@@ -425,7 +439,7 @@ class Command(BaseCommand):
|
||||
)
|
||||
|
||||
# Process existing files
|
||||
self._process_existing_files(
|
||||
queued = self._process_existing_files(
|
||||
directory=directory,
|
||||
recursive=recursive,
|
||||
subdirs_as_tags=subdirs_as_tags,
|
||||
@@ -445,6 +459,7 @@ class Command(BaseCommand):
|
||||
polling_interval=polling_interval,
|
||||
stability_delay=stability_delay,
|
||||
is_testing=is_testing,
|
||||
queued=queued,
|
||||
)
|
||||
|
||||
logger.debug("Consumer exiting")
|
||||
@@ -456,11 +471,18 @@ class Command(BaseCommand):
|
||||
recursive: bool,
|
||||
subdirs_as_tags: bool,
|
||||
consumer_filter: ConsumerFilter,
|
||||
) -> None:
|
||||
"""Process any existing files in the consumption directory."""
|
||||
) -> set[Path]:
|
||||
"""
|
||||
Process any existing files in the consumption directory.
|
||||
|
||||
Returns the set of resolved paths that were queued, so the watch loop
|
||||
can seed its in-flight set and avoid re-queuing them on the first
|
||||
rescan before the consume tasks have removed them from disk.
|
||||
"""
|
||||
logger.info(f"Processing existing files in {directory}")
|
||||
|
||||
glob_pattern = "**/*" if recursive else "*"
|
||||
queued: set[Path] = set()
|
||||
|
||||
for filepath in directory.glob(glob_pattern):
|
||||
# Use filter to check if file should be processed
|
||||
@@ -475,6 +497,48 @@ class Command(BaseCommand):
|
||||
consumption_dir=directory,
|
||||
subdirs_as_tags=subdirs_as_tags,
|
||||
)
|
||||
queued.add(filepath.resolve())
|
||||
|
||||
return queued
|
||||
|
||||
def _rescan_existing_files(
|
||||
self,
|
||||
*,
|
||||
directory: Path,
|
||||
recursive: bool,
|
||||
consumer_filter: ConsumerFilter,
|
||||
tracker: FileStabilityTracker,
|
||||
queued: set[Path],
|
||||
) -> None:
|
||||
"""
|
||||
Re-inject on-disk files the watcher never reported into the tracker.
|
||||
|
||||
Acts as a safety net for files stranded by the watcher-recreation gap
|
||||
(see ``rescan_interval_s``). Files already being tracked or already
|
||||
queued and awaiting consumption are skipped, so a file is never queued
|
||||
twice. Queued paths that have since left the directory are pruned so a
|
||||
later file reusing the same name is not skipped forever.
|
||||
"""
|
||||
# Prune in-flight paths that have left the directory
|
||||
for path in list(queued):
|
||||
if not path.exists():
|
||||
queued.discard(path)
|
||||
|
||||
glob_pattern = "**/*" if recursive else "*"
|
||||
|
||||
for filepath in directory.glob(glob_pattern):
|
||||
if not filepath.is_file():
|
||||
continue
|
||||
|
||||
if not consumer_filter(Change.added, str(filepath)):
|
||||
continue
|
||||
|
||||
resolved = filepath.resolve()
|
||||
if tracker.is_tracking(resolved) or resolved in queued:
|
||||
continue
|
||||
|
||||
logger.debug(f"Rescan found untracked file: {resolved}")
|
||||
tracker.track(resolved, Change.added)
|
||||
|
||||
def _watch_directory(
|
||||
self,
|
||||
@@ -486,11 +550,24 @@ class Command(BaseCommand):
|
||||
polling_interval: float,
|
||||
stability_delay: float,
|
||||
is_testing: bool,
|
||||
queued: set[Path] | None = None,
|
||||
) -> None:
|
||||
"""Watch directory for changes and process stable files."""
|
||||
use_polling = polling_interval > 0
|
||||
poll_delay_ms = int(polling_interval * 1000) if use_polling else 0
|
||||
|
||||
# Resolved paths that have been queued and are awaiting consumption.
|
||||
# Seeded from the startup scan so the first rescan does not re-queue
|
||||
# files whose consume tasks have not yet removed them from disk.
|
||||
queued = set() if queued is None else queued
|
||||
|
||||
# Full-glob safety net cadence (0 disables)
|
||||
rescan_interval_s = self.rescan_interval_s
|
||||
rescan_timeout_ms = (
|
||||
int(rescan_interval_s * 1000) if rescan_interval_s > 0 else 0
|
||||
)
|
||||
last_rescan = monotonic()
|
||||
|
||||
if use_polling:
|
||||
logger.info(
|
||||
f"Watching {directory} using polling (interval: {polling_interval}s)",
|
||||
@@ -505,6 +582,20 @@ class Command(BaseCommand):
|
||||
stability_timeout_ms = int(stability_delay * 1000)
|
||||
testing_timeout_ms = int(self.testing_timeout_s * 1000)
|
||||
|
||||
def cap_for_rescan(ms: int) -> int:
|
||||
"""
|
||||
Ensure the watch loop wakes often enough to run the rescan.
|
||||
|
||||
``watch()`` blocks for up to ``rust_timeout``, so the rescan can
|
||||
only run that often. A timeout of 0 means "wait indefinitely",
|
||||
which would never wake to rescan; cap it at the rescan interval.
|
||||
"""
|
||||
if rescan_timeout_ms <= 0:
|
||||
return ms
|
||||
if ms <= 0:
|
||||
return rescan_timeout_ms
|
||||
return min(ms, rescan_timeout_ms)
|
||||
|
||||
# Calculate appropriate timeout for watch loop
|
||||
# In polling mode, rust_timeout must be significantly longer than poll_delay_ms
|
||||
# to ensure poll cycles can complete before timing out
|
||||
@@ -522,6 +613,8 @@ class Command(BaseCommand):
|
||||
# Not testing, wait indefinitely for first event
|
||||
timeout_ms = 0
|
||||
|
||||
timeout_ms = cap_for_rescan(timeout_ms)
|
||||
|
||||
self.stop_flag.clear()
|
||||
|
||||
while not self.stop_flag.is_set():
|
||||
@@ -551,10 +644,26 @@ class Command(BaseCommand):
|
||||
consumption_dir=directory,
|
||||
subdirs_as_tags=subdirs_as_tags,
|
||||
)
|
||||
# Remember it so the rescan does not re-queue it while
|
||||
# the consume task has yet to remove it from disk
|
||||
queued.add(stable_path)
|
||||
|
||||
# Exit watch loop to reconfigure timeout
|
||||
break
|
||||
|
||||
# Periodic full-glob safety net for files the watcher missed
|
||||
if rescan_timeout_ms > 0 and (
|
||||
monotonic() - last_rescan >= rescan_interval_s
|
||||
):
|
||||
self._rescan_existing_files(
|
||||
directory=directory,
|
||||
recursive=recursive,
|
||||
consumer_filter=consumer_filter,
|
||||
tracker=tracker,
|
||||
queued=queued,
|
||||
)
|
||||
last_rescan = monotonic()
|
||||
|
||||
# Determine next timeout
|
||||
if tracker.has_pending_files():
|
||||
# Check pending files at stability interval
|
||||
@@ -572,6 +681,8 @@ class Command(BaseCommand):
|
||||
# No pending files, wait indefinitely
|
||||
timeout_ms = 0
|
||||
|
||||
timeout_ms = cap_for_rescan(timeout_ms)
|
||||
|
||||
except KeyboardInterrupt: # pragma: nocover
|
||||
logger.info("Received interrupt, stopping consumer")
|
||||
self.stop_flag.set()
|
||||
|
||||
@@ -45,6 +45,12 @@ class SanityCheckMessages:
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._messages: dict[int | None, list[MessageEntry]] = defaultdict(list)
|
||||
self._document_pks: set[int] = set()
|
||||
self._document_error_pks: set[int] = set()
|
||||
self._document_warning_pks: set[int] = set()
|
||||
self._document_info_pks: set[int] = set()
|
||||
self._document_error_issue_count: int = 0
|
||||
self._document_warning_issue_count: int = 0
|
||||
self.has_error: bool = False
|
||||
self.has_warning: bool = False
|
||||
self.has_info: bool = False
|
||||
@@ -56,20 +62,33 @@ class SanityCheckMessages:
|
||||
|
||||
# -- Recording ----------------------------------------------------------
|
||||
|
||||
def _add_document_issue(self, doc_pk: int, document_pks: set[int]) -> bool:
|
||||
if doc_pk not in self._document_pks:
|
||||
self._document_pks.add(doc_pk)
|
||||
self.document_count += 1
|
||||
|
||||
if doc_pk in document_pks:
|
||||
return False
|
||||
|
||||
document_pks.add(doc_pk)
|
||||
return True
|
||||
|
||||
def error(self, doc_pk: int | None, message: str) -> None:
|
||||
self._messages[doc_pk].append({"level": logging.ERROR, "message": message})
|
||||
self.has_error = True
|
||||
if doc_pk is not None:
|
||||
self.document_count += 1
|
||||
self.document_error_count += 1
|
||||
self._document_error_issue_count += 1
|
||||
if self._add_document_issue(doc_pk, self._document_error_pks):
|
||||
self.document_error_count += 1
|
||||
|
||||
def warning(self, doc_pk: int | None, message: str) -> None:
|
||||
self._messages[doc_pk].append({"level": logging.WARNING, "message": message})
|
||||
self.has_warning = True
|
||||
|
||||
if doc_pk is not None:
|
||||
self.document_count += 1
|
||||
self.document_warning_count += 1
|
||||
self._document_warning_issue_count += 1
|
||||
if self._add_document_issue(doc_pk, self._document_warning_pks):
|
||||
self.document_warning_count += 1
|
||||
else:
|
||||
# This is the only type of global message we do right now
|
||||
self.global_warning_count += 1
|
||||
@@ -78,8 +97,10 @@ class SanityCheckMessages:
|
||||
self._messages[doc_pk].append({"level": logging.INFO, "message": message})
|
||||
self.has_info = True
|
||||
|
||||
if doc_pk is not None:
|
||||
self.document_count += 1
|
||||
if doc_pk is not None and self._add_document_issue(
|
||||
doc_pk,
|
||||
self._document_info_pks,
|
||||
):
|
||||
self.document_info_count += 1
|
||||
|
||||
# -- Iteration / query --------------------------------------------------
|
||||
@@ -105,8 +126,8 @@ class SanityCheckMessages:
|
||||
def total_issue_count(self) -> int:
|
||||
"""Total number of error and warning messages across all documents and global."""
|
||||
return (
|
||||
self.document_error_count
|
||||
+ self.document_warning_count
|
||||
self._document_error_issue_count
|
||||
+ self._document_warning_issue_count
|
||||
+ self.global_warning_count
|
||||
)
|
||||
|
||||
|
||||
@@ -8,11 +8,15 @@ from documents.search._backend import get_backend
|
||||
from documents.search._backend import reset_backend
|
||||
from documents.search._schema import needs_rebuild
|
||||
from documents.search._schema import wipe_index
|
||||
from documents.search._translate import InvalidDateQuery
|
||||
from documents.search._translate import SearchQueryError
|
||||
|
||||
__all__ = [
|
||||
"InvalidDateQuery",
|
||||
"SearchHit",
|
||||
"SearchIndexLockError",
|
||||
"SearchMode",
|
||||
"SearchQueryError",
|
||||
"TantivyBackend",
|
||||
"TantivyRelevanceList",
|
||||
"WriteBatch",
|
||||
|
||||
@@ -866,8 +866,24 @@ class TantivyBackend:
|
||||
final_query = self._apply_permission_filter(mlt_query, user)
|
||||
|
||||
effective_limit = limit if limit is not None else searcher.num_docs
|
||||
# Fetch one extra to account for excluding the original document
|
||||
results = searcher.search(final_query, limit=effective_limit + 1)
|
||||
try:
|
||||
# Fetch one extra to account for excluding the original document
|
||||
results = searcher.search(final_query, limit=effective_limit + 1)
|
||||
except BaseException: # pragma: no cover
|
||||
# Tantivy 0.26 panics in BM25 idf scoring when the index holds
|
||||
# soft-deleted documents (doc_freq can exceed the alive doc count),
|
||||
# which only surfaces for the More Like This query. The panic crosses
|
||||
# the pyo3 boundary as a `pyo3_runtime.PanicException` — a
|
||||
# BaseException, not an Exception — so catch BaseException and degrade
|
||||
# to "no similar documents" instead of bubbling a 500 to the client.
|
||||
# Fixed upstream: https://github.com/quickwit-oss/tantivy/pull/2964
|
||||
# Remove once the bundled tantivy includes that fix.
|
||||
logger.warning(
|
||||
"More Like This scoring panicked (likely stale tantivy segment "
|
||||
"stats after deletions); returning no results. A search index "
|
||||
"reindex will rebuild consistent statistics.",
|
||||
)
|
||||
return []
|
||||
|
||||
addrs = [addr for _score, addr in results.hits]
|
||||
all_ids = cast("list[int]", searcher.fast_field_values("id", addrs))
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC
|
||||
from datetime import date
|
||||
from datetime import datetime
|
||||
from datetime import timedelta
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import Final
|
||||
|
||||
from dateutil.relativedelta import relativedelta
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from datetime import tzinfo
|
||||
|
||||
_DATE_ONLY_FIELDS = frozenset({"created"})
|
||||
|
||||
_TODAY: Final[str] = "today"
|
||||
_YESTERDAY: Final[str] = "yesterday"
|
||||
_PREVIOUS_WEEK: Final[str] = "previous week"
|
||||
_THIS_MONTH: Final[str] = "this month"
|
||||
_PREVIOUS_MONTH: Final[str] = "previous month"
|
||||
_THIS_YEAR: Final[str] = "this year"
|
||||
_PREVIOUS_YEAR: Final[str] = "previous year"
|
||||
_PREVIOUS_QUARTER: Final[str] = "previous quarter"
|
||||
|
||||
_DATE_KEYWORDS = frozenset(
|
||||
{
|
||||
_TODAY,
|
||||
_YESTERDAY,
|
||||
_PREVIOUS_WEEK,
|
||||
_THIS_MONTH,
|
||||
_PREVIOUS_MONTH,
|
||||
_THIS_YEAR,
|
||||
_PREVIOUS_YEAR,
|
||||
_PREVIOUS_QUARTER,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _fmt(dt: datetime) -> str:
|
||||
"""Format a datetime as an ISO 8601 UTC string for use in Tantivy range queries."""
|
||||
return dt.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
def _iso_range(lo: datetime, hi: datetime) -> str:
|
||||
"""Format a [lo TO hi] range string in ISO 8601 for Tantivy query syntax."""
|
||||
return f"[{_fmt(lo)} TO {_fmt(hi)}]"
|
||||
|
||||
|
||||
def _quarter_start(d: date) -> date:
|
||||
"""Return the first day of the calendar quarter containing ``d``."""
|
||||
return date(d.year, ((d.month - 1) // 3) * 3 + 1, 1)
|
||||
|
||||
|
||||
def _midnight(d: date, tz: tzinfo) -> datetime:
|
||||
"""Convert a calendar date at local-timezone midnight to a UTC datetime."""
|
||||
return datetime(d.year, d.month, d.day, tzinfo=tz).astimezone(UTC)
|
||||
|
||||
|
||||
def _keyword_bounds(keyword: str, tz: tzinfo) -> tuple[date, date]:
|
||||
"""
|
||||
Map a relative date keyword to ``(start, exclusive_end)`` calendar dates.
|
||||
|
||||
``tz`` only determines what "today" is; the caller decides how the returned
|
||||
dates become UTC datetime boundaries (date-only vs. local-midnight offset).
|
||||
"""
|
||||
today = datetime.now(tz).date()
|
||||
if keyword == _TODAY:
|
||||
return today, today + timedelta(days=1)
|
||||
if keyword == _YESTERDAY:
|
||||
return today - timedelta(days=1), today
|
||||
if keyword == _PREVIOUS_WEEK:
|
||||
this_monday = today - timedelta(days=today.weekday())
|
||||
return this_monday - timedelta(weeks=1), this_monday
|
||||
if keyword == _THIS_MONTH:
|
||||
first = today.replace(day=1)
|
||||
return first, first + relativedelta(months=1)
|
||||
if keyword == _PREVIOUS_MONTH:
|
||||
this_first = today.replace(day=1)
|
||||
return this_first - relativedelta(months=1), this_first
|
||||
if keyword == _THIS_YEAR:
|
||||
return date(today.year, 1, 1), date(today.year + 1, 1, 1)
|
||||
if keyword == _PREVIOUS_YEAR:
|
||||
return date(today.year - 1, 1, 1), date(today.year, 1, 1)
|
||||
if keyword == _PREVIOUS_QUARTER:
|
||||
this_quarter = _quarter_start(today)
|
||||
return this_quarter - relativedelta(months=3), this_quarter
|
||||
raise ValueError(f"Unknown keyword: {keyword}")
|
||||
|
||||
|
||||
def _date_only_range(keyword: str, tz: tzinfo) -> str:
|
||||
"""
|
||||
For `created` (DateField): use the local calendar date, converted to
|
||||
midnight UTC boundaries. No offset arithmetic — date only.
|
||||
"""
|
||||
start, end = _keyword_bounds(keyword, tz)
|
||||
lo = datetime(start.year, start.month, start.day, tzinfo=UTC)
|
||||
hi = datetime(end.year, end.month, end.day, tzinfo=UTC)
|
||||
return _iso_range(lo, hi)
|
||||
|
||||
|
||||
def _datetime_range(keyword: str, tz: tzinfo) -> str:
|
||||
"""
|
||||
For `added` / `modified` (DateTimeField, stored as UTC): convert local day
|
||||
boundaries to UTC — full offset arithmetic required.
|
||||
"""
|
||||
start, end = _keyword_bounds(keyword, tz)
|
||||
return _iso_range(_midnight(start, tz), _midnight(end, tz))
|
||||
|
||||
|
||||
def _precision_bounds(digits: str) -> tuple[date, date] | None:
|
||||
"""
|
||||
Map a 4/6/8-digit date token to (start, exclusive_end) calendar dates.
|
||||
|
||||
YYYY -> whole year, YYYYMM -> whole month, YYYYMMDD -> single day.
|
||||
Returns None for any unparsable or out-of-range value (e.g. month 23),
|
||||
so callers can emit a no-match clause instead of erroring (Whoosh parity).
|
||||
"""
|
||||
try:
|
||||
if len(digits) == 4:
|
||||
year = int(digits)
|
||||
return date(year, 1, 1), date(year + 1, 1, 1)
|
||||
if len(digits) == 6:
|
||||
year, month = int(digits[:4]), int(digits[4:6])
|
||||
start = date(year, month, 1)
|
||||
end = date(year + 1, 1, 1) if month == 12 else date(year, month + 1, 1)
|
||||
return start, end
|
||||
if len(digits) == 8:
|
||||
start = date(int(digits[:4]), int(digits[4:6]), int(digits[6:8]))
|
||||
return start, start + timedelta(days=1)
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _utc_bounds_for_field(
|
||||
field: str,
|
||||
start: date,
|
||||
end: date,
|
||||
tz: tzinfo,
|
||||
) -> tuple[datetime, datetime]:
|
||||
"""
|
||||
Convert calendar-date bounds to UTC datetimes per the field's storage type.
|
||||
|
||||
For DateField (``created``) the bounds are UTC midnight (no offset). For
|
||||
DateTimeField (``added``/``modified``) the bounds are local-tz midnight
|
||||
converted to UTC, matching how each field is indexed.
|
||||
"""
|
||||
if field in _DATE_ONLY_FIELDS:
|
||||
return (
|
||||
datetime(start.year, start.month, start.day, tzinfo=UTC),
|
||||
datetime(end.year, end.month, end.day, tzinfo=UTC),
|
||||
)
|
||||
return (
|
||||
datetime(start.year, start.month, start.day, tzinfo=tz).astimezone(UTC),
|
||||
datetime(end.year, end.month, end.day, tzinfo=tz).astimezone(UTC),
|
||||
)
|
||||
|
||||
|
||||
def _field_range_from_dates(field: str, start: date, end: date, tz: tzinfo) -> str:
|
||||
"""Build a Tantivy ``field:[lo TO hi]`` ISO range from calendar-date bounds."""
|
||||
lo, hi = _utc_bounds_for_field(field, start, end, tz)
|
||||
return f"{field}:{_iso_range(lo, hi)}"
|
||||
+27
-405
@@ -1,88 +1,35 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import UTC
|
||||
from datetime import date
|
||||
from datetime import datetime
|
||||
from datetime import timedelta
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import Final
|
||||
|
||||
import regex
|
||||
import tantivy
|
||||
from dateutil.relativedelta import relativedelta
|
||||
from django.conf import settings
|
||||
|
||||
from documents.search._dates import (
|
||||
_date_only_range, # noqa: F401 — re-exported for test imports
|
||||
)
|
||||
from documents.search._dates import (
|
||||
_datetime_range, # noqa: F401 — re-exported for test imports
|
||||
)
|
||||
from documents.search._tokenizer import simple_search_tokens
|
||||
from documents.search._translate import SearchQueryError
|
||||
from documents.search._translate import translate_query
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from datetime import tzinfo
|
||||
|
||||
from django.contrib.auth.base_user import AbstractBaseUser
|
||||
|
||||
logger = logging.getLogger("paperless.search")
|
||||
|
||||
# Maximum seconds any single regex substitution may run.
|
||||
# Prevents ReDoS on adversarial user-supplied query strings.
|
||||
_REGEX_TIMEOUT: Final[float] = 1.0
|
||||
|
||||
_DATE_ONLY_FIELDS = frozenset({"created"})
|
||||
|
||||
_TODAY: Final[str] = "today"
|
||||
_YESTERDAY: Final[str] = "yesterday"
|
||||
_PREVIOUS_WEEK: Final[str] = "previous week"
|
||||
_THIS_MONTH: Final[str] = "this month"
|
||||
_PREVIOUS_MONTH: Final[str] = "previous month"
|
||||
_THIS_YEAR: Final[str] = "this year"
|
||||
_PREVIOUS_YEAR: Final[str] = "previous year"
|
||||
_PREVIOUS_QUARTER: Final[str] = "previous quarter"
|
||||
|
||||
_DATE_KEYWORDS = frozenset(
|
||||
{
|
||||
_TODAY,
|
||||
_YESTERDAY,
|
||||
_PREVIOUS_WEEK,
|
||||
_THIS_MONTH,
|
||||
_PREVIOUS_MONTH,
|
||||
_THIS_YEAR,
|
||||
_PREVIOUS_YEAR,
|
||||
_PREVIOUS_QUARTER,
|
||||
},
|
||||
)
|
||||
|
||||
_DATE_KEYWORD_PATTERN = "|".join(
|
||||
sorted((regex.escape(k) for k in _DATE_KEYWORDS), key=len, reverse=True),
|
||||
)
|
||||
|
||||
_FIELD_DATE_RE = regex.compile(
|
||||
rf"""(?<!\w)(?P<field>created|modified|added)\s*:\s*(?:
|
||||
(?P<quote>["'])(?P<quoted>{_DATE_KEYWORD_PATTERN})(?P=quote)
|
||||
|
|
||||
(?P<bare>{_DATE_KEYWORD_PATTERN})(?![\w-])
|
||||
)""",
|
||||
regex.IGNORECASE | regex.VERBOSE,
|
||||
)
|
||||
_COMPACT_DATE_RE = regex.compile(r"\b(\d{14})\b")
|
||||
_RELATIVE_RANGE_RE = regex.compile(
|
||||
r"\[now([+-]\d+[dhm])?\s+TO\s+now([+-]\d+[dhm])?\]",
|
||||
regex.IGNORECASE,
|
||||
)
|
||||
# Whoosh-style relative date range: e.g. [-1 week to now], [-7 days to now]
|
||||
_WHOOSH_REL_RANGE_RE = regex.compile(
|
||||
r"\[-(?P<n>\d+)\s+(?P<unit>second|minute|hour|day|week|month|year)s?\s+to\s+now\]",
|
||||
regex.IGNORECASE,
|
||||
)
|
||||
# Whoosh-style 8-digit date: field:YYYYMMDD — field-aware so timezone can be applied correctly.
|
||||
# Scoped to date fields only; numeric fields (asn, id, page_count, ...) must not be rewritten.
|
||||
_DATE8_RE = regex.compile(
|
||||
r"(?<!\w)(?P<field>created|modified|added):(?P<date8>\d{8})\b",
|
||||
)
|
||||
_YEAR_RANGE_RE = regex.compile(
|
||||
r"(?<!\w)(?P<field>created|modified|added):\[(?P<y1>\d{4})\s+TO\s+(?P<y2>\d{4})\]",
|
||||
regex.IGNORECASE,
|
||||
)
|
||||
# Tantivy syntax error: " - " and " + " with spaces on both sides are invalid because
|
||||
# the NOT/MUST operators require no space between the operator and the term.
|
||||
# In natural-language queries (e.g., "H52.1 - Kurzsichtigkeit"), the dash is a separator.
|
||||
_SPACED_OPERATOR_RE = regex.compile(r"\s+[-+]\s+")
|
||||
_TRAILING_OPERATOR_RE = regex.compile(r"\s+[-+]+\s*$")
|
||||
# Matches CJK/Hangul characters so queries can be routed to bigram fields.
|
||||
# Uses Unicode properties to cover all blocks including Extension B+ planes.
|
||||
_CJK_RE: Final = regex.compile(r"[\p{Han}\p{Hiragana}\p{Katakana}\p{Hangul}]+")
|
||||
@@ -117,303 +64,12 @@ def _build_cjk_query(
|
||||
return None
|
||||
|
||||
|
||||
def _fmt(dt: datetime) -> str:
|
||||
"""Format a datetime as an ISO 8601 UTC string for use in Tantivy range queries."""
|
||||
return dt.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
def _iso_range(lo: datetime, hi: datetime) -> str:
|
||||
"""Format a [lo TO hi] range string in ISO 8601 for Tantivy query syntax."""
|
||||
return f"[{_fmt(lo)} TO {_fmt(hi)}]"
|
||||
|
||||
|
||||
def _date_only_range(keyword: str, tz: tzinfo) -> str:
|
||||
"""
|
||||
For `created` (DateField): use the local calendar date, converted to
|
||||
midnight UTC boundaries. No offset arithmetic — date only.
|
||||
"""
|
||||
|
||||
today = datetime.now(tz).date()
|
||||
|
||||
def _quarter_start(d: date) -> date:
|
||||
return date(d.year, ((d.month - 1) // 3) * 3 + 1, 1)
|
||||
|
||||
if keyword == _TODAY:
|
||||
lo = datetime(today.year, today.month, today.day, tzinfo=UTC)
|
||||
return _iso_range(lo, lo + timedelta(days=1))
|
||||
if keyword == _YESTERDAY:
|
||||
y = today - timedelta(days=1)
|
||||
lo = datetime(y.year, y.month, y.day, tzinfo=UTC)
|
||||
hi = datetime(today.year, today.month, today.day, tzinfo=UTC)
|
||||
return _iso_range(lo, hi)
|
||||
if keyword == _PREVIOUS_WEEK:
|
||||
this_mon = today - timedelta(days=today.weekday())
|
||||
last_mon = this_mon - timedelta(weeks=1)
|
||||
lo = datetime(last_mon.year, last_mon.month, last_mon.day, tzinfo=UTC)
|
||||
hi = datetime(this_mon.year, this_mon.month, this_mon.day, tzinfo=UTC)
|
||||
return _iso_range(lo, hi)
|
||||
if keyword == _THIS_MONTH:
|
||||
lo = datetime(today.year, today.month, 1, tzinfo=UTC)
|
||||
if today.month == 12:
|
||||
hi = datetime(today.year + 1, 1, 1, tzinfo=UTC)
|
||||
else:
|
||||
hi = datetime(today.year, today.month + 1, 1, tzinfo=UTC)
|
||||
return _iso_range(lo, hi)
|
||||
if keyword == _PREVIOUS_MONTH:
|
||||
if today.month == 1:
|
||||
lo = datetime(today.year - 1, 12, 1, tzinfo=UTC)
|
||||
else:
|
||||
lo = datetime(today.year, today.month - 1, 1, tzinfo=UTC)
|
||||
hi = datetime(today.year, today.month, 1, tzinfo=UTC)
|
||||
return _iso_range(lo, hi)
|
||||
if keyword == _THIS_YEAR:
|
||||
lo = datetime(today.year, 1, 1, tzinfo=UTC)
|
||||
return _iso_range(lo, datetime(today.year + 1, 1, 1, tzinfo=UTC))
|
||||
if keyword == _PREVIOUS_YEAR:
|
||||
lo = datetime(today.year - 1, 1, 1, tzinfo=UTC)
|
||||
return _iso_range(lo, datetime(today.year, 1, 1, tzinfo=UTC))
|
||||
if keyword == _PREVIOUS_QUARTER:
|
||||
this_quarter = _quarter_start(today)
|
||||
last_quarter = this_quarter - relativedelta(months=3)
|
||||
lo = datetime(
|
||||
last_quarter.year,
|
||||
last_quarter.month,
|
||||
last_quarter.day,
|
||||
tzinfo=UTC,
|
||||
)
|
||||
hi = datetime(
|
||||
this_quarter.year,
|
||||
this_quarter.month,
|
||||
this_quarter.day,
|
||||
tzinfo=UTC,
|
||||
)
|
||||
return _iso_range(lo, hi)
|
||||
raise ValueError(f"Unknown keyword: {keyword}")
|
||||
|
||||
|
||||
def _datetime_range(keyword: str, tz: tzinfo) -> str:
|
||||
"""
|
||||
For `added` / `modified` (DateTimeField, stored as UTC): convert local day
|
||||
boundaries to UTC — full offset arithmetic required.
|
||||
"""
|
||||
|
||||
now_local = datetime.now(tz)
|
||||
today = now_local.date()
|
||||
|
||||
def _midnight(d: date) -> datetime:
|
||||
return datetime(d.year, d.month, d.day, tzinfo=tz).astimezone(UTC)
|
||||
|
||||
def _quarter_start(d: date) -> date:
|
||||
return date(d.year, ((d.month - 1) // 3) * 3 + 1, 1)
|
||||
|
||||
if keyword == _TODAY:
|
||||
return _iso_range(_midnight(today), _midnight(today + timedelta(days=1)))
|
||||
if keyword == _YESTERDAY:
|
||||
y = today - timedelta(days=1)
|
||||
return _iso_range(_midnight(y), _midnight(today))
|
||||
if keyword == _PREVIOUS_WEEK:
|
||||
this_mon = today - timedelta(days=today.weekday())
|
||||
last_mon = this_mon - timedelta(weeks=1)
|
||||
return _iso_range(_midnight(last_mon), _midnight(this_mon))
|
||||
if keyword == _THIS_MONTH:
|
||||
first = today.replace(day=1)
|
||||
if today.month == 12:
|
||||
next_first = date(today.year + 1, 1, 1)
|
||||
else:
|
||||
next_first = date(today.year, today.month + 1, 1)
|
||||
return _iso_range(_midnight(first), _midnight(next_first))
|
||||
if keyword == _PREVIOUS_MONTH:
|
||||
this_first = today.replace(day=1)
|
||||
if today.month == 1:
|
||||
last_first = date(today.year - 1, 12, 1)
|
||||
else:
|
||||
last_first = date(today.year, today.month - 1, 1)
|
||||
return _iso_range(_midnight(last_first), _midnight(this_first))
|
||||
if keyword == _THIS_YEAR:
|
||||
return _iso_range(
|
||||
_midnight(date(today.year, 1, 1)),
|
||||
_midnight(date(today.year + 1, 1, 1)),
|
||||
)
|
||||
if keyword == _PREVIOUS_YEAR:
|
||||
return _iso_range(
|
||||
_midnight(date(today.year - 1, 1, 1)),
|
||||
_midnight(date(today.year, 1, 1)),
|
||||
)
|
||||
if keyword == _PREVIOUS_QUARTER:
|
||||
this_quarter = _quarter_start(today)
|
||||
last_quarter = this_quarter - relativedelta(months=3)
|
||||
return _iso_range(_midnight(last_quarter), _midnight(this_quarter))
|
||||
raise ValueError(f"Unknown keyword: {keyword}")
|
||||
|
||||
|
||||
def _rewrite_compact_date(query: str) -> str:
|
||||
"""Rewrite Whoosh compact date tokens (14-digit YYYYMMDDHHmmss) to ISO 8601."""
|
||||
|
||||
def _sub(m: regex.Match[str]) -> str:
|
||||
raw = m.group(1)
|
||||
try:
|
||||
dt = datetime(
|
||||
int(raw[0:4]),
|
||||
int(raw[4:6]),
|
||||
int(raw[6:8]),
|
||||
int(raw[8:10]),
|
||||
int(raw[10:12]),
|
||||
int(raw[12:14]),
|
||||
tzinfo=UTC,
|
||||
)
|
||||
return dt.strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
except ValueError:
|
||||
return str(m.group(0))
|
||||
|
||||
try:
|
||||
return _COMPACT_DATE_RE.sub(_sub, query, timeout=_REGEX_TIMEOUT)
|
||||
except TimeoutError: # pragma: no cover
|
||||
raise ValueError(
|
||||
"Query too complex to process (compact date rewrite timed out)",
|
||||
)
|
||||
|
||||
|
||||
def _rewrite_relative_range(query: str) -> str:
|
||||
"""Rewrite Whoosh relative ranges ([now-7d TO now]) to concrete ISO 8601 UTC boundaries."""
|
||||
|
||||
def _sub(m: regex.Match[str]) -> str:
|
||||
now = datetime.now(UTC)
|
||||
|
||||
def _offset(s: str | None) -> timedelta:
|
||||
if not s:
|
||||
return timedelta(0)
|
||||
sign = 1 if s[0] == "+" else -1
|
||||
n, unit = int(s[1:-1]), s[-1]
|
||||
return (
|
||||
sign
|
||||
* {
|
||||
"d": timedelta(days=n),
|
||||
"h": timedelta(hours=n),
|
||||
"m": timedelta(minutes=n),
|
||||
}[unit]
|
||||
)
|
||||
|
||||
lo, hi = now + _offset(m.group(1)), now + _offset(m.group(2))
|
||||
if lo > hi:
|
||||
lo, hi = hi, lo
|
||||
return f"[{_fmt(lo)} TO {_fmt(hi)}]"
|
||||
|
||||
try:
|
||||
return _RELATIVE_RANGE_RE.sub(_sub, query, timeout=_REGEX_TIMEOUT)
|
||||
except TimeoutError: # pragma: no cover
|
||||
raise ValueError(
|
||||
"Query too complex to process (relative range rewrite timed out)",
|
||||
)
|
||||
|
||||
|
||||
def _rewrite_whoosh_relative_range(query: str) -> str:
|
||||
"""Rewrite Whoosh-style relative date ranges ([-N unit to now]) to ISO 8601.
|
||||
|
||||
Supports: second, minute, hour, day, week, month, year (singular and plural).
|
||||
Example: ``added:[-1 week to now]`` → ``added:[2025-01-01T… TO 2025-01-08T…]``
|
||||
"""
|
||||
now = datetime.now(UTC)
|
||||
|
||||
def _sub(m: regex.Match[str]) -> str:
|
||||
n = int(m.group("n"))
|
||||
unit = m.group("unit").lower()
|
||||
delta_map: dict[str, timedelta | relativedelta] = {
|
||||
"second": timedelta(seconds=n),
|
||||
"minute": timedelta(minutes=n),
|
||||
"hour": timedelta(hours=n),
|
||||
"day": timedelta(days=n),
|
||||
"week": timedelta(weeks=n),
|
||||
"month": relativedelta(months=n),
|
||||
"year": relativedelta(years=n),
|
||||
}
|
||||
lo = now - delta_map[unit]
|
||||
return f"[{_fmt(lo)} TO {_fmt(now)}]"
|
||||
|
||||
try:
|
||||
return _WHOOSH_REL_RANGE_RE.sub(_sub, query, timeout=_REGEX_TIMEOUT)
|
||||
except TimeoutError: # pragma: no cover
|
||||
raise ValueError(
|
||||
"Query too complex to process (Whoosh relative range rewrite timed out)",
|
||||
)
|
||||
|
||||
|
||||
def _rewrite_8digit_date(query: str, tz: tzinfo) -> str:
|
||||
"""Rewrite field:YYYYMMDD date tokens to an ISO 8601 day range.
|
||||
|
||||
Runs after ``_rewrite_compact_date`` so 14-digit timestamps are already
|
||||
converted and won't spuriously match here.
|
||||
|
||||
For DateField fields (e.g. ``created``) uses UTC midnight boundaries.
|
||||
For DateTimeField fields (e.g. ``added``, ``modified``) uses local TZ
|
||||
midnight boundaries converted to UTC — matching the ``_datetime_range``
|
||||
behaviour for keyword dates.
|
||||
"""
|
||||
|
||||
def _sub(m: regex.Match[str]) -> str:
|
||||
field = m.group("field")
|
||||
raw = m.group("date8")
|
||||
try:
|
||||
year, month, day = int(raw[0:4]), int(raw[4:6]), int(raw[6:8])
|
||||
d = date(year, month, day)
|
||||
if field in _DATE_ONLY_FIELDS:
|
||||
lo = datetime(d.year, d.month, d.day, tzinfo=UTC)
|
||||
hi = lo + timedelta(days=1)
|
||||
else:
|
||||
# DateTimeField: use local-timezone midnight → UTC
|
||||
lo = datetime(d.year, d.month, d.day, tzinfo=tz).astimezone(UTC)
|
||||
hi = datetime(
|
||||
(d + timedelta(days=1)).year,
|
||||
(d + timedelta(days=1)).month,
|
||||
(d + timedelta(days=1)).day,
|
||||
tzinfo=tz,
|
||||
).astimezone(UTC)
|
||||
return f"{field}:[{_fmt(lo)} TO {_fmt(hi)}]"
|
||||
except ValueError:
|
||||
return m.group(0)
|
||||
|
||||
try:
|
||||
return _DATE8_RE.sub(_sub, query, timeout=_REGEX_TIMEOUT)
|
||||
except TimeoutError: # pragma: no cover
|
||||
raise ValueError(
|
||||
"Query too complex to process (8-digit date rewrite timed out)",
|
||||
)
|
||||
|
||||
|
||||
def _rewrite_year_range(query: str) -> str:
|
||||
"""Rewrite Whoosh-style year-only date ranges to ISO 8601 UTC boundaries.
|
||||
|
||||
Converts ``field:[YYYY TO YYYY]`` to a full ISO 8601 datetime range.
|
||||
The upper bound is the start of the year after the end year (exclusive),
|
||||
matching the Whoosh convention of treating year-only ranges as full-year spans.
|
||||
"""
|
||||
|
||||
def _sub(m: regex.Match[str]) -> str:
|
||||
field = m.group("field")
|
||||
y1, y2 = int(m.group("y1")), int(m.group("y2"))
|
||||
# Whoosh swaps a reversed range when both years are explicit
|
||||
# (whoosh.util.times.timespan.disambiguated); match that so a backwards
|
||||
# range spans the intended years instead of matching nothing.
|
||||
lo_year, hi_year = min(y1, y2), max(y1, y2)
|
||||
lo = datetime(lo_year, 1, 1, tzinfo=UTC)
|
||||
hi = datetime(hi_year + 1, 1, 1, tzinfo=UTC)
|
||||
return f"{field}:[{_fmt(lo)} TO {_fmt(hi)}]"
|
||||
|
||||
try:
|
||||
return _YEAR_RANGE_RE.sub(_sub, query, timeout=_REGEX_TIMEOUT)
|
||||
except TimeoutError: # pragma: no cover
|
||||
raise ValueError("Query too complex to process (year range rewrite timed out)")
|
||||
|
||||
|
||||
def rewrite_natural_date_keywords(query: str, tz: tzinfo) -> str:
|
||||
"""
|
||||
Rewrite natural date syntax to ISO 8601 format for Tantivy compatibility.
|
||||
|
||||
Performs the first stage of query preprocessing, converting various date
|
||||
formats and keywords to ISO 8601 datetime ranges that Tantivy can parse:
|
||||
- Compact 14-digit dates (YYYYMMDDHHmmss)
|
||||
- Whoosh relative ranges ([-7 days to now], [now-1h TO now+2h])
|
||||
- 8-digit dates with field awareness (created:20240115)
|
||||
- Natural keywords (field:today, field:"previous quarter", etc.)
|
||||
Delegates to ``translate_query`` which handles all date forms, comma
|
||||
expansion, field aliasing, relative ranges, and operator normalization.
|
||||
|
||||
Args:
|
||||
query: Raw user query string
|
||||
@@ -425,35 +81,15 @@ def rewrite_natural_date_keywords(query: str, tz: tzinfo) -> str:
|
||||
Note:
|
||||
Bare keywords without field prefixes pass through unchanged.
|
||||
"""
|
||||
query = _rewrite_compact_date(query)
|
||||
query = _rewrite_whoosh_relative_range(query)
|
||||
query = _rewrite_year_range(query)
|
||||
query = _rewrite_8digit_date(query, tz)
|
||||
query = _rewrite_relative_range(query)
|
||||
|
||||
def _replace(m: regex.Match[str]) -> str:
|
||||
field = m.group("field")
|
||||
keyword = (m.group("quoted") or m.group("bare")).lower()
|
||||
if field in _DATE_ONLY_FIELDS:
|
||||
return f"{field}:{_date_only_range(keyword, tz)}"
|
||||
return f"{field}:{_datetime_range(keyword, tz)}"
|
||||
|
||||
try:
|
||||
return _FIELD_DATE_RE.sub(_replace, query, timeout=_REGEX_TIMEOUT)
|
||||
except TimeoutError: # pragma: no cover
|
||||
raise ValueError(
|
||||
"Query too complex to process (date keyword rewrite timed out)",
|
||||
)
|
||||
return translate_query(query, tz)
|
||||
|
||||
|
||||
def normalize_query(query: str) -> str:
|
||||
"""
|
||||
Normalize query syntax for better search behavior.
|
||||
|
||||
Expands comma-separated field values to explicit AND clauses and
|
||||
collapses excessive whitespace for cleaner parsing:
|
||||
- tag:foo,bar → tag:foo AND tag:bar
|
||||
- multiple spaces → single spaces
|
||||
Delegates to ``translate_query`` which handles comma expansion, whitespace
|
||||
collapsing, operator normalization, and field aliasing.
|
||||
|
||||
Args:
|
||||
query: Query string after date rewriting
|
||||
@@ -461,29 +97,7 @@ def normalize_query(query: str) -> str:
|
||||
Returns:
|
||||
Normalized query string ready for Tantivy parsing
|
||||
"""
|
||||
|
||||
def _expand(m: regex.Match[str]) -> str:
|
||||
field = m.group(1)
|
||||
values = [v.strip() for v in m.group(2).split(",") if v.strip()]
|
||||
return " AND ".join(f"{field}:{v}" for v in values)
|
||||
|
||||
try:
|
||||
query = regex.sub(
|
||||
r"(\w+):([^\s\[\]]+(?:,[^\s\[\]]+)+)",
|
||||
_expand,
|
||||
query,
|
||||
timeout=_REGEX_TIMEOUT,
|
||||
)
|
||||
query = regex.sub(r" {2,}", " ", query, timeout=_REGEX_TIMEOUT).strip()
|
||||
# Strip trailing dangling operators before Tantivy sees them.
|
||||
query = _TRAILING_OPERATOR_RE.sub("", query, timeout=_REGEX_TIMEOUT).strip()
|
||||
# Replace " - " / " + " with a space: Tantivy requires no space between
|
||||
# the operator and its operand (-term / +term), so spaces on both sides
|
||||
# means this is a natural-language separator, not a query operator.
|
||||
query = _SPACED_OPERATOR_RE.sub(" ", query, timeout=_REGEX_TIMEOUT).strip()
|
||||
return query
|
||||
except TimeoutError: # pragma: no cover
|
||||
raise ValueError("Query too complex to process (normalization timed out)")
|
||||
return translate_query(query, UTC)
|
||||
|
||||
|
||||
def build_permission_filter(
|
||||
@@ -603,8 +217,16 @@ def parse_user_query(
|
||||
as a post-search score filter, not during query construction.
|
||||
"""
|
||||
|
||||
query_str = rewrite_natural_date_keywords(raw_query, tz)
|
||||
query_str = normalize_query(query_str)
|
||||
try:
|
||||
query_str = translate_query(raw_query, tz)
|
||||
except SearchQueryError:
|
||||
# Intentional, user-fixable error (e.g. an unparsable date). Propagate so
|
||||
# the view can return a 400 with a helpful message rather than falling
|
||||
# back to the raw (still-invalid) query.
|
||||
raise
|
||||
except Exception: # pragma: no cover - defensive
|
||||
logger.warning("Query translation failed; using raw query", exc_info=True)
|
||||
query_str = raw_query
|
||||
|
||||
exact = index.parse_query(
|
||||
query_str,
|
||||
|
||||
@@ -48,6 +48,9 @@ _LANGUAGE_MAP: dict[str, str] = {
|
||||
}
|
||||
|
||||
SUPPORTED_LANGUAGES: frozenset[str] = frozenset(_LANGUAGE_MAP)
|
||||
# Document.title is max_length=128, so use 129 as the limit for
|
||||
# Tantivy's remove_long filter
|
||||
_TOKEN_REMOVE_LONG_LIMIT: Final[int] = 129
|
||||
|
||||
|
||||
def register_tokenizers(index: tantivy.Index, language: str | None) -> None:
|
||||
@@ -77,10 +80,10 @@ def register_tokenizers(index: tantivy.Index, language: str | None) -> None:
|
||||
|
||||
|
||||
def _paperless_text(language: str | None) -> tantivy.TextAnalyzer:
|
||||
"""Main full-text tokenizer for content, title, etc: simple -> remove_long(65) -> lowercase -> ascii_fold [-> stemmer]"""
|
||||
"""Main full-text tokenizer for content, title, etc: simple -> remove_long(129) -> lowercase -> ascii_fold [-> stemmer]"""
|
||||
builder = (
|
||||
tantivy.TextAnalyzerBuilder(tantivy.Tokenizer.simple())
|
||||
.filter(tantivy.Filter.remove_long(65))
|
||||
.filter(tantivy.Filter.remove_long(_TOKEN_REMOVE_LONG_LIMIT))
|
||||
.filter(tantivy.Filter.lowercase())
|
||||
.filter(tantivy.Filter.ascii_fold())
|
||||
)
|
||||
@@ -119,12 +122,12 @@ def _bigram_analyzer() -> tantivy.TextAnalyzer:
|
||||
|
||||
|
||||
def _simple_search_analyzer() -> tantivy.TextAnalyzer:
|
||||
"""Tokenizer for simple substring search fields: non-whitespace chunks -> remove_long(65) -> lowercase -> ascii_fold."""
|
||||
"""Tokenizer for simple substring search fields: non-whitespace chunks -> remove_long(129) -> lowercase -> ascii_fold."""
|
||||
return (
|
||||
tantivy.TextAnalyzerBuilder(
|
||||
tantivy.Tokenizer.regex(r"\S+"),
|
||||
)
|
||||
.filter(tantivy.Filter.remove_long(65))
|
||||
.filter(tantivy.Filter.remove_long(_TOKEN_REMOVE_LONG_LIMIT))
|
||||
.filter(tantivy.Filter.lowercase())
|
||||
.filter(tantivy.Filter.ascii_fold())
|
||||
.build()
|
||||
|
||||
@@ -0,0 +1,566 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC
|
||||
from datetime import datetime
|
||||
from datetime import timedelta
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TypeAlias
|
||||
|
||||
import regex
|
||||
from dateutil.relativedelta import relativedelta
|
||||
|
||||
from documents.search._dates import _DATE_KEYWORDS
|
||||
from documents.search._dates import _DATE_ONLY_FIELDS
|
||||
from documents.search._dates import _date_only_range
|
||||
from documents.search._dates import _datetime_range
|
||||
from documents.search._dates import _field_range_from_dates
|
||||
from documents.search._dates import _fmt
|
||||
from documents.search._dates import _precision_bounds
|
||||
from documents.search._dates import _utc_bounds_for_field
|
||||
|
||||
# Compiled regex that matches any known multi-word (or single-word) date keyword
|
||||
# at the start of a match position, longest alternatives first so "previous week"
|
||||
# wins over a hypothetical shorter "previous".
|
||||
_KEYWORD_VALUE_RE = regex.compile(
|
||||
"|".join(sorted((regex.escape(k) for k in _DATE_KEYWORDS), key=len, reverse=True)),
|
||||
regex.IGNORECASE,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from datetime import tzinfo
|
||||
|
||||
# TODO: this module translates date queries into Tantivy *string* syntax, which
|
||||
# forces a workaround for something Tantivy's string parser cannot express on
|
||||
# date fields: open-ended ranges use far-past/far-future string sentinels
|
||||
# (OPEN_LO/OPEN_HI). These can be replaced with a real tantivy.Query object
|
||||
# (Query.range_query(..., None) for open bounds) once tantivy-py accepts Python
|
||||
# datetimes in range_query/term_query on Date fields. That support exists on
|
||||
# tantivy-py master (PRs #655 + #666) but postdates the pinned 0.26.0 wheel, so
|
||||
# it is blocked only on a published release > 0.26.0 and a dependency bump.
|
||||
# (Unparsable dates now raise InvalidDateQuery -> HTTP 400 rather than using a
|
||||
# no-match string sentinel.)
|
||||
|
||||
# Fields that store exact, non-analyzed comma-joined tokens in the index and so
|
||||
# need explicit comma->AND expansion (Whoosh KEYWORD(commas=True) set).
|
||||
MULTI_VALUE_FIELDS = frozenset({"tag", "tag_id", "viewer_id"})
|
||||
|
||||
# Date fields whose values/ranges get rewritten to RFC3339 Tantivy ranges.
|
||||
DATE_FIELDS = frozenset({"created", "modified", "added"})
|
||||
|
||||
# Field aliases: Whoosh (v2) field names that were renamed in the Tantivy schema.
|
||||
# Preserved here so v2 queries using the old names continue to work without 400
|
||||
# errors instead of silently failing. Applied by _render to non-date field tokens.
|
||||
FIELD_ALIASES: dict[str, str] = {
|
||||
"type": "document_type",
|
||||
"type_id": "document_type_id",
|
||||
"path": "storage_path",
|
||||
"path_id": "storage_path_id",
|
||||
}
|
||||
|
||||
# Known schema fields: a comma immediately followed by ``<known>:`` is a clause
|
||||
# separator. Restricting to known fields prevents URL-like ``http:`` misfires.
|
||||
KNOWN_FIELDS = frozenset(
|
||||
{
|
||||
"title",
|
||||
"content",
|
||||
"correspondent",
|
||||
"document_type",
|
||||
"type", # v2 alias -> document_type
|
||||
"storage_path",
|
||||
"path", # v2 alias -> storage_path
|
||||
"tag",
|
||||
"tag_id",
|
||||
"correspondent_id",
|
||||
"document_type_id",
|
||||
"type_id", # v2 alias -> document_type_id
|
||||
"storage_path_id",
|
||||
"path_id", # v2 alias -> storage_path_id
|
||||
"owner_id",
|
||||
"viewer_id",
|
||||
"asn",
|
||||
"page_count",
|
||||
"num_notes",
|
||||
"created",
|
||||
"modified",
|
||||
"added",
|
||||
"original_filename",
|
||||
"checksum",
|
||||
"notes",
|
||||
"custom_fields",
|
||||
},
|
||||
)
|
||||
|
||||
_FIELD_RE = regex.compile(r"(?P<field>\w+):")
|
||||
|
||||
# Matches the TO separator inside a range bracket. Handles three forms:
|
||||
# middle: "lo TO hi" (either lo or hi may be empty)
|
||||
# trailing: "lo TO" (open upper bound)
|
||||
# leading: "TO hi" (open lower bound)
|
||||
# Bounds MAY contain internal spaces (e.g. "-7 days"), so we use .*? / .+?
|
||||
# and split on the whitespace-delimited " TO " / " to " separator.
|
||||
_RANGE_RE = regex.compile(
|
||||
r"^\s*(?P<lo>.*?)\s+[Tt][Oo]\s+(?P<hi>.+?)\s*$"
|
||||
r"|"
|
||||
r"^\s*(?P<lo2>.+?)\s+[Tt][Oo]\s*$"
|
||||
r"|"
|
||||
r"^\s*[Tt][Oo]\s+(?P<hi2>.+?)\s*$",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class FieldValue:
|
||||
field: str
|
||||
value: str
|
||||
|
||||
|
||||
# Produced by the comma-resolution pass (not by scan()).
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class FieldValueList:
|
||||
field: str
|
||||
values: tuple[str, ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class FieldRange:
|
||||
field: str
|
||||
open: str
|
||||
lo: str
|
||||
hi: str
|
||||
close: str
|
||||
|
||||
|
||||
# Produced by the comma-resolution pass (not by scan()).
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Comma:
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Passthrough:
|
||||
raw: str
|
||||
|
||||
|
||||
Token: TypeAlias = FieldValue | FieldValueList | FieldRange | Comma | Passthrough
|
||||
|
||||
_CLOSE: dict[str, str] = {"[": "]", "{": "}"}
|
||||
|
||||
|
||||
def scan(query: str) -> list[Token]:
|
||||
"""
|
||||
Tokenize a raw query into date/comma-aware tokens, leaving everything else
|
||||
as verbatim ``Passthrough`` runs. Non-recursive: finds the first matching
|
||||
close bracket/quote. Nested brackets are not valid Tantivy range syntax and
|
||||
pass through verbatim on mismatch.
|
||||
"""
|
||||
tokens: list[Token] = []
|
||||
buf: list[str] = [] # accumulates passthrough chars
|
||||
i, n = 0, len(query)
|
||||
while i < n:
|
||||
matched = _match_field_token(query, i)
|
||||
if matched is None:
|
||||
buf.append(query[i])
|
||||
i += 1
|
||||
continue
|
||||
token, i = matched
|
||||
_flush(buf, tokens)
|
||||
tokens.append(token)
|
||||
i = _maybe_comma(query, i, tokens)
|
||||
_flush(buf, tokens)
|
||||
return tokens
|
||||
|
||||
|
||||
def _flush(buf: list[str], tokens: list[Token]) -> None:
|
||||
"""Emit any accumulated passthrough characters as a single token."""
|
||||
if buf:
|
||||
tokens.append(Passthrough("".join(buf)))
|
||||
buf.clear()
|
||||
|
||||
|
||||
def _at_word_boundary(query: str, i: int) -> bool:
|
||||
"""A field token may begin only at the start or after a non-word character."""
|
||||
return i == 0 or not (query[i - 1].isalnum() or query[i - 1] == "_")
|
||||
|
||||
|
||||
def _match_field_token(query: str, i: int) -> tuple[Token, int] | None:
|
||||
"""
|
||||
If a known ``field:`` token starts at ``i``, consume it and return
|
||||
``(token, end_index)``; otherwise return None so the caller treats the
|
||||
character as passthrough. Handles both ``field:[range]`` and ``field:value``,
|
||||
and returns None when the range/value cannot be consumed.
|
||||
"""
|
||||
m = _FIELD_RE.match(query, i)
|
||||
if m is None or m.group("field") not in KNOWN_FIELDS:
|
||||
return None
|
||||
if not _at_word_boundary(query, i):
|
||||
return None
|
||||
field = m.group("field")
|
||||
j = m.end()
|
||||
if j < len(query) and query[j] in "[{":
|
||||
return _consume_range(query, j, field)
|
||||
consumed = _consume_field_value(query, field, j)
|
||||
if consumed is None:
|
||||
return None
|
||||
value, end = consumed
|
||||
return FieldValue(field, value), end
|
||||
|
||||
|
||||
def _consume_field_value(query: str, field: str, start: int) -> tuple[str, int] | None:
|
||||
"""
|
||||
Consume a field value starting at ``start``: a multi-word date keyword phrase
|
||||
(date fields only), or a bare/quoted value, then absorb any comma-joined
|
||||
continuation that is not a clause separator. ``resolve_commas`` later splits a
|
||||
multi-value field's joined value into a ``FieldValueList``; for other fields
|
||||
the comma stays literal.
|
||||
"""
|
||||
n = len(query)
|
||||
consumed = None
|
||||
if field in DATE_FIELDS:
|
||||
km = _KEYWORD_VALUE_RE.match(query, start)
|
||||
if km is not None and (km.end() >= n or query[km.end()] in " \t),"):
|
||||
consumed = (km.group(0), km.end())
|
||||
if consumed is None:
|
||||
consumed = _consume_value(query, start)
|
||||
if consumed is None:
|
||||
return None
|
||||
value, k = consumed
|
||||
while k < n and query[k] == ",":
|
||||
if _looks_like_known_field(query, k + 1):
|
||||
break # clause separator: left for _maybe_comma to emit a Comma()
|
||||
more = _consume_value(query, k + 1)
|
||||
if more is None:
|
||||
break
|
||||
value = f"{value},{more[0]}"
|
||||
k = more[1]
|
||||
return value, k
|
||||
|
||||
|
||||
def _consume_range(
|
||||
query: str,
|
||||
start: int,
|
||||
field: str,
|
||||
) -> tuple[FieldRange, int] | None:
|
||||
"""Consume ``[lo TO hi]`` / ``{lo TO hi}`` from ``start`` (the bracket)."""
|
||||
open_br = query[start]
|
||||
close_br = _CLOSE[open_br]
|
||||
end = query.find(close_br, start + 1)
|
||||
if end == -1:
|
||||
return None
|
||||
inner = query[start + 1 : end]
|
||||
m = _RANGE_RE.match(inner)
|
||||
if m is not None:
|
||||
if m.group("lo") is not None or m.group("hi") is not None:
|
||||
# Middle form: "lo TO hi" (either may be empty string)
|
||||
lo = (m.group("lo") or "").strip()
|
||||
hi = (m.group("hi") or "").strip()
|
||||
elif m.group("lo2") is not None:
|
||||
# Trailing form: "lo TO"
|
||||
lo = m.group("lo2").strip()
|
||||
hi = ""
|
||||
else:
|
||||
# Leading form: "TO hi"
|
||||
lo = ""
|
||||
hi = (m.group("hi2") or "").strip()
|
||||
else:
|
||||
lo, hi = inner.strip(), ""
|
||||
return FieldRange(field, open_br, lo, hi, close_br), end + 1
|
||||
|
||||
|
||||
def _consume_value(query: str, start: int) -> tuple[str, int] | None:
|
||||
"""Consume a bare or quoted field value from ``start``, stopping at comma."""
|
||||
n = len(query)
|
||||
if start >= n or query[start] in " \t":
|
||||
return None
|
||||
if query[start] in "\"'":
|
||||
quote = query[start]
|
||||
end = query.find(quote, start + 1)
|
||||
if end == -1:
|
||||
return None
|
||||
return query[start : end + 1], end + 1
|
||||
j = start
|
||||
while j < n and query[j] not in " \t),":
|
||||
j += 1
|
||||
return query[start:j], j
|
||||
|
||||
|
||||
def _looks_like_known_field(query: str, pos: int) -> bool:
|
||||
"""True if a known ``field:`` token starts at ``pos``."""
|
||||
m = _FIELD_RE.match(query, pos)
|
||||
return bool(m and m.group("field") in KNOWN_FIELDS)
|
||||
|
||||
|
||||
def _maybe_comma(query: str, i: int, tokens: list) -> int:
|
||||
"""If a clause-separator comma follows at ``i``, emit ``Comma()`` and advance."""
|
||||
if i < len(query) and query[i] == "," and _looks_like_known_field(query, i + 1):
|
||||
tokens.append(Comma())
|
||||
return i + 1
|
||||
return i
|
||||
|
||||
|
||||
def resolve_commas(tokens: list) -> list:
|
||||
"""
|
||||
Collapse value-list commas into ``FieldValueList`` and keep clause-separator
|
||||
commas as ``Comma``. (Clause-sep commas are already emitted by ``scan`` via
|
||||
the value-stop logic; this pass folds value-lists.)
|
||||
"""
|
||||
out: list = []
|
||||
for tok in tokens:
|
||||
if (
|
||||
isinstance(tok, FieldValue)
|
||||
and tok.field in MULTI_VALUE_FIELDS
|
||||
and "," in tok.value
|
||||
):
|
||||
values = tuple(v for v in tok.value.split(",") if v)
|
||||
out.append(FieldValueList(tok.field, values))
|
||||
else:
|
||||
out.append(tok)
|
||||
return out
|
||||
|
||||
|
||||
class SearchQueryError(ValueError):
|
||||
"""
|
||||
Base for user-fixable search query errors.
|
||||
|
||||
Carries a message safe to surface to the user (no internal details). The view
|
||||
layer catches this and returns an HTTP 400, so any future subclass (unknown
|
||||
field, malformed range, wrapped parser errors) gets the same treatment.
|
||||
"""
|
||||
|
||||
|
||||
class InvalidDateQuery(SearchQueryError):
|
||||
"""Raised when a date field value or range bound cannot be parsed."""
|
||||
|
||||
def __init__(self, field: str, value: str) -> None:
|
||||
self.field = field
|
||||
self.value = value
|
||||
super().__init__(f"Invalid date value {value!r} for field {field!r}.")
|
||||
|
||||
|
||||
_DIGITS_RE = regex.compile(r"^\d{4}(?:\d{2}){0,2}$")
|
||||
_ISO_RE = regex.compile(r"^\d{4}(?:-\d{2}(?:-\d{2})?)?$")
|
||||
|
||||
|
||||
def translate_scalar(field: str, value: str, tz: tzinfo) -> str:
|
||||
"""Translate a bare date-field value to a Tantivy range string."""
|
||||
bare = value.strip("\"'").lower()
|
||||
if bare in _DATE_KEYWORDS:
|
||||
if field in _DATE_ONLY_FIELDS:
|
||||
return f"{field}:{_date_only_range(bare, tz)}"
|
||||
return f"{field}:{_datetime_range(bare, tz)}"
|
||||
digits = value.replace("-", "")
|
||||
if _DIGITS_RE.match(value) or _ISO_RE.match(value):
|
||||
bounds = _precision_bounds(digits)
|
||||
if bounds is None:
|
||||
raise InvalidDateQuery(field, value)
|
||||
return _field_range_from_dates(field, bounds[0], bounds[1], tz)
|
||||
if regex.fullmatch(r"\d{14}", value):
|
||||
try:
|
||||
dt = datetime(
|
||||
int(value[0:4]),
|
||||
int(value[4:6]),
|
||||
int(value[6:8]),
|
||||
int(value[8:10]),
|
||||
int(value[10:12]),
|
||||
int(value[12:14]),
|
||||
tzinfo=UTC,
|
||||
)
|
||||
except ValueError:
|
||||
raise InvalidDateQuery(field, value) from None
|
||||
iso = _fmt(dt)
|
||||
return f"{field}:[{iso} TO {iso}]"
|
||||
# Unrecognized shape -> tell the user their date is malformed rather than
|
||||
# silently matching nothing or emitting invalid Tantivy syntax.
|
||||
raise InvalidDateQuery(field, value)
|
||||
|
||||
|
||||
# Open-bound sentinels for date ranges. These far-past/far-future strings allow
|
||||
# open-ended ranges to be expressed as Tantivy string queries until tantivy-py
|
||||
# exposes Query.range_query(..., None) on Date fields (see module TODO).
|
||||
OPEN_LO = "0001-01-01T00:00:00Z"
|
||||
OPEN_HI = "9999-12-31T23:59:59Z"
|
||||
|
||||
|
||||
# Matches compact now-offset tokens like now-7d, now+1h, now-30m.
|
||||
_NOW_COMPACT_RE = regex.compile(
|
||||
r"^now(?P<sign>[+-])(?P<n>\d+)(?P<unit>[dhm])$",
|
||||
regex.IGNORECASE,
|
||||
)
|
||||
|
||||
# Matches "±N <unit>" Whoosh-style offsets (e.g. -7 days, -1 week, +3 hours)
|
||||
# Unit is singular or plural; sign prefix is mandatory.
|
||||
_NOW_SPACED_RE = regex.compile(
|
||||
r"^(?P<sign>[+-])(?P<n>\d+)\s*"
|
||||
r"(?P<unit>second|minute|hour|day|week|month|year)s?$",
|
||||
regex.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _resolve_relative_bound(token: str) -> datetime | None:
|
||||
"""
|
||||
Resolve a relative bound token to an exact UTC instant, or return None.
|
||||
|
||||
Supported forms:
|
||||
- ``now`` -> current UTC instant
|
||||
- ``now+/-<n>d/h/m`` -> now +/- timedelta (d=days, h=hours, m=minutes)
|
||||
- ``±N <unit>`` -> now +/- delta; month/year use relativedelta
|
||||
"""
|
||||
stripped = token.strip()
|
||||
low = stripped.lower()
|
||||
now = datetime.now(UTC)
|
||||
|
||||
if low == "now":
|
||||
return now
|
||||
|
||||
m = _NOW_COMPACT_RE.match(stripped)
|
||||
if m:
|
||||
sign = 1 if m.group("sign") == "+" else -1
|
||||
n = int(m.group("n"))
|
||||
unit = m.group("unit").lower()
|
||||
delta = (
|
||||
sign
|
||||
* {
|
||||
"d": timedelta(days=n),
|
||||
"h": timedelta(hours=n),
|
||||
"m": timedelta(minutes=n),
|
||||
}[unit]
|
||||
)
|
||||
return now + delta
|
||||
|
||||
m = _NOW_SPACED_RE.match(stripped)
|
||||
if m:
|
||||
sign = 1 if m.group("sign") == "+" else -1
|
||||
n = int(m.group("n"))
|
||||
unit = m.group("unit").lower()
|
||||
delta_map: dict[str, timedelta | relativedelta] = {
|
||||
"second": timedelta(seconds=n),
|
||||
"minute": timedelta(minutes=n),
|
||||
"hour": timedelta(hours=n),
|
||||
"day": timedelta(days=n),
|
||||
"week": timedelta(weeks=n),
|
||||
"month": relativedelta(months=n),
|
||||
"year": relativedelta(years=n),
|
||||
}
|
||||
return now - delta_map[unit] if sign == -1 else now + delta_map[unit]
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _bound_datetimes(
|
||||
field: str,
|
||||
token: str,
|
||||
tz: tzinfo,
|
||||
) -> tuple[datetime, datetime] | None:
|
||||
"""
|
||||
Return (floor_dt, ceil_dt) UTC datetimes for a single range bound token, or
|
||||
None if the token is unparsable. ``now`` and relative offsets resolve to the
|
||||
current instant (floor == ceil == that instant; no day-flooring).
|
||||
"""
|
||||
token = token.strip()
|
||||
|
||||
# Try relative/now forms first (before stripping hyphens which would mangle them).
|
||||
rel = _resolve_relative_bound(token)
|
||||
if rel is not None:
|
||||
return rel, rel
|
||||
|
||||
# Full ISO datetime token (contains "T"): parse directly and return an exact
|
||||
# instant (floor == ceil). Python 3.11+ datetime.fromisoformat accepts trailing Z.
|
||||
if "T" in token:
|
||||
try:
|
||||
dt = datetime.fromisoformat(token)
|
||||
# Ensure timezone-aware UTC result.
|
||||
dt = dt.replace(tzinfo=UTC) if dt.tzinfo is None else dt.astimezone(UTC)
|
||||
return dt, dt
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
digits = token.replace("-", "")
|
||||
bounds = _precision_bounds(digits)
|
||||
if bounds is None:
|
||||
return None
|
||||
start, end = bounds
|
||||
return _utc_bounds_for_field(field, start, end, tz)
|
||||
|
||||
|
||||
def _render(tok: Token, tz: tzinfo) -> str:
|
||||
"""Render a single token back to a Tantivy query string fragment."""
|
||||
if isinstance(tok, Passthrough):
|
||||
return tok.raw
|
||||
if isinstance(tok, Comma):
|
||||
return " AND "
|
||||
if isinstance(tok, FieldValueList):
|
||||
field = FIELD_ALIASES.get(tok.field, tok.field)
|
||||
return " AND ".join(f"{field}:{v}" for v in tok.values)
|
||||
if isinstance(tok, FieldValue):
|
||||
field = FIELD_ALIASES.get(tok.field, tok.field)
|
||||
if field in DATE_FIELDS:
|
||||
return translate_scalar(field, tok.value, tz)
|
||||
return f"{field}:{tok.value}"
|
||||
if isinstance(tok, FieldRange):
|
||||
field = FIELD_ALIASES.get(tok.field, tok.field)
|
||||
if field in DATE_FIELDS:
|
||||
return translate_range(field, tok.lo, tok.hi, tz)
|
||||
return f"{field}:{tok.open}{tok.lo} TO {tok.hi}{tok.close}"
|
||||
return "" # pragma: no cover
|
||||
|
||||
|
||||
# Post-render operator normalization patterns: collapse repeated whitespace and
|
||||
# strip spaced/trailing Tantivy boolean operators that would otherwise be invalid.
|
||||
_MULTI_SPACE_RE = regex.compile(r" {2,}")
|
||||
_TRAILING_OP_RE = regex.compile(r"\s+[-+]+\s*$")
|
||||
_SPACED_OP_RE = regex.compile(r"\s+[-+]\s+")
|
||||
|
||||
|
||||
def _normalize_operators(text: str) -> str:
|
||||
"""
|
||||
Collapse multiple spaces, strip trailing dangling operators, and replace
|
||||
spaced operators (`` - `` / `` + ``) with a single space.
|
||||
|
||||
Applied only to Passthrough fragments (the rendered output is scanned for
|
||||
operator artifacts outside bracketed ranges) via a post-render pass on the
|
||||
full rendered string. This preserves date ranges (``[... TO ...]``) verbatim
|
||||
while cleaning natural-language separators in the surrounding text.
|
||||
"""
|
||||
text = _MULTI_SPACE_RE.sub(" ", text)
|
||||
text = _TRAILING_OP_RE.sub("", text).strip()
|
||||
text = _SPACED_OP_RE.sub(" ", text).strip()
|
||||
return text
|
||||
|
||||
|
||||
def translate_query(raw: str, tz: tzinfo) -> str:
|
||||
"""Translate a raw Whoosh-style query into Tantivy-compatible syntax."""
|
||||
tokens = resolve_commas(scan(raw))
|
||||
rendered = "".join(_render(t, tz) for t in tokens)
|
||||
return _normalize_operators(rendered)
|
||||
|
||||
|
||||
def translate_range(field: str, lo: str, hi: str, tz: tzinfo) -> str:
|
||||
"""Translate a date-field ``[lo TO hi]`` range to a Tantivy ISO range string.
|
||||
|
||||
Handles partial-date bounds (YYYY, YYYYMM, YYYYMMDD, ISO dash variants),
|
||||
open bounds (empty string -> OPEN_LO/OPEN_HI), ``now``, and reversed ranges
|
||||
(swaps tokens before computing floor/ceil so the span is always correct).
|
||||
"""
|
||||
lo_s = lo.strip()
|
||||
hi_s = hi.strip()
|
||||
|
||||
# Parse both bounds to (floor, ceil) pairs when present.
|
||||
lo_pair: tuple[datetime, datetime] | None = None
|
||||
hi_pair: tuple[datetime, datetime] | None = None
|
||||
|
||||
if lo_s:
|
||||
lo_pair = _bound_datetimes(field, lo_s, tz)
|
||||
if lo_pair is None:
|
||||
raise InvalidDateQuery(field, lo_s)
|
||||
if hi_s:
|
||||
hi_pair = _bound_datetimes(field, hi_s, tz)
|
||||
if hi_pair is None:
|
||||
raise InvalidDateQuery(field, hi_s)
|
||||
|
||||
# Detect a reversed range: only swap when BOTH bounds are present.
|
||||
if lo_pair is not None and hi_pair is not None and lo_pair[0] > hi_pair[0]:
|
||||
lo_pair, hi_pair = hi_pair, lo_pair
|
||||
|
||||
lo_iso = _fmt(lo_pair[0]) if lo_pair is not None else OPEN_LO
|
||||
hi_iso = _fmt(hi_pair[1]) if hi_pair is not None else OPEN_HI
|
||||
|
||||
return f"{field}:[{lo_iso} TO {hi_iso}]"
|
||||
@@ -1,6 +1,7 @@
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import unicodedata
|
||||
from collections.abc import Iterable
|
||||
from pathlib import PurePath
|
||||
|
||||
@@ -36,10 +37,12 @@ class FilePathTemplate(Template):
|
||||
def clean_filepath(value: str) -> str:
|
||||
"""
|
||||
Clean up a filepath by:
|
||||
1. Removing newlines and carriage returns
|
||||
2. Removing extra spaces before and after forward slashes
|
||||
3. Preserving spaces in other parts of the path
|
||||
1. Normalizing Unicode to NFC form to prevent byte-level mismatches
|
||||
2. Removing newlines and carriage returns
|
||||
3. Removing extra spaces before and after forward slashes
|
||||
4. Preserving spaces in other parts of the path
|
||||
"""
|
||||
value = unicodedata.normalize("NFC", value)
|
||||
value = value.replace("\n", "").replace("\r", "")
|
||||
value = re.sub(r"\s*/\s*", "/", value)
|
||||
|
||||
@@ -181,17 +184,17 @@ def get_basic_metadata_context(
|
||||
"""
|
||||
return {
|
||||
"title": pathvalidate.sanitize_filename(
|
||||
document.title,
|
||||
unicodedata.normalize("NFC", document.title),
|
||||
replacement_text="-",
|
||||
),
|
||||
"correspondent": pathvalidate.sanitize_filename(
|
||||
document.correspondent.name,
|
||||
unicodedata.normalize("NFC", document.correspondent.name),
|
||||
replacement_text="-",
|
||||
)
|
||||
if document.correspondent
|
||||
else no_value_default,
|
||||
"document_type": pathvalidate.sanitize_filename(
|
||||
document.document_type.name,
|
||||
unicodedata.normalize("NFC", document.document_type.name),
|
||||
replacement_text="-",
|
||||
)
|
||||
if document.document_type
|
||||
@@ -202,7 +205,10 @@ def get_basic_metadata_context(
|
||||
"owner_username": document.owner.username
|
||||
if document.owner
|
||||
else no_value_default,
|
||||
"original_name": PurePath(document.original_filename).with_suffix("").name
|
||||
"original_name": unicodedata.normalize(
|
||||
"NFC",
|
||||
PurePath(document.original_filename).with_suffix("").name,
|
||||
)
|
||||
if document.original_filename
|
||||
else no_value_default,
|
||||
"doc_pk": f"{document.pk:07}",
|
||||
@@ -269,12 +275,12 @@ def get_tags_context(tags: Iterable[Tag]) -> dict[str, str | list[str]]:
|
||||
return {
|
||||
"tag_list": pathvalidate.sanitize_filename(
|
||||
",".join(
|
||||
sorted(tag.name for tag in tags),
|
||||
sorted(unicodedata.normalize("NFC", tag.name) for tag in tags),
|
||||
),
|
||||
replacement_text="-",
|
||||
),
|
||||
# Assumed to be ordered, but a template could loop through to find what they want
|
||||
"tag_name_list": [x.name for x in tags],
|
||||
"tag_name_list": [unicodedata.normalize("NFC", x.name) for x in tags],
|
||||
}
|
||||
|
||||
|
||||
@@ -301,7 +307,7 @@ def get_custom_fields_context(
|
||||
CustomField.FieldDataType.LONG_TEXT,
|
||||
}:
|
||||
value = pathvalidate.sanitize_filename(
|
||||
field_instance.value,
|
||||
unicodedata.normalize("NFC", field_instance.value),
|
||||
replacement_text="-",
|
||||
)
|
||||
elif (
|
||||
@@ -310,10 +316,13 @@ def get_custom_fields_context(
|
||||
):
|
||||
options = field_instance.field.extra_data["select_options"]
|
||||
value = pathvalidate.sanitize_filename(
|
||||
next(
|
||||
option["label"]
|
||||
for option in options
|
||||
if option["id"] == field_instance.value
|
||||
unicodedata.normalize(
|
||||
"NFC",
|
||||
next(
|
||||
option["label"]
|
||||
for option in options
|
||||
if option["id"] == field_instance.value
|
||||
),
|
||||
),
|
||||
replacement_text="-",
|
||||
)
|
||||
@@ -321,7 +330,7 @@ def get_custom_fields_context(
|
||||
value = field_instance.value
|
||||
field_data["custom_fields"][
|
||||
pathvalidate.sanitize_filename(
|
||||
field_instance.field.name,
|
||||
unicodedata.normalize("NFC", field_instance.field.name),
|
||||
replacement_text="-",
|
||||
)
|
||||
] = {
|
||||
|
||||
@@ -1,11 +1,15 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
import tantivy
|
||||
|
||||
from documents.search._backend import TantivyBackend
|
||||
from documents.search._backend import reset_backend
|
||||
from documents.search._schema import build_schema
|
||||
from documents.search._tokenizer import register_tokenizers
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Generator
|
||||
@@ -31,3 +35,11 @@ def backend() -> Generator[TantivyBackend, None, None]:
|
||||
finally:
|
||||
b.close()
|
||||
reset_backend()
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def index() -> tantivy.Index:
|
||||
"""A real Tantivy index for parse-acceptance tests (module scope for speed)."""
|
||||
idx = tantivy.Index(build_schema(), path=tempfile.mkdtemp())
|
||||
register_tokenizers(idx, "english")
|
||||
return idx
|
||||
|
||||
@@ -261,6 +261,36 @@ class TestSearch:
|
||||
== 1
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("search_mode", "query"),
|
||||
[
|
||||
pytest.param(SearchMode.TITLE, "12345", id="title_search"),
|
||||
pytest.param(SearchMode.TEXT, "12345", id="text_search"),
|
||||
pytest.param(SearchMode.QUERY, None, id="query_title_exact"),
|
||||
],
|
||||
)
|
||||
def test_search_modes_match_model_limit_title_tokens(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
search_mode: SearchMode,
|
||||
query: str | None,
|
||||
) -> None:
|
||||
"""Search must keep filename-like title tokens up to the model limit."""
|
||||
long_title = "1234567890" * 12 + "12345678"
|
||||
doc = Document.objects.create(
|
||||
title=long_title,
|
||||
content="ordinary content",
|
||||
checksum="TXT12",
|
||||
pk=18,
|
||||
)
|
||||
backend.add_or_update(doc)
|
||||
|
||||
assert backend.search_ids(
|
||||
query or f"title:{long_title}",
|
||||
user=None,
|
||||
search_mode=search_mode,
|
||||
) == [doc.pk]
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mode", "title", "content", "hits", "misses"),
|
||||
[
|
||||
|
||||
@@ -13,7 +13,6 @@ import time_machine
|
||||
|
||||
from documents.search._query import _date_only_range
|
||||
from documents.search._query import _datetime_range
|
||||
from documents.search._query import _rewrite_compact_date
|
||||
from documents.search._query import build_permission_filter
|
||||
from documents.search._query import normalize_query
|
||||
from documents.search._query import parse_simple_text_highlight_query
|
||||
@@ -21,6 +20,7 @@ from documents.search._query import parse_user_query
|
||||
from documents.search._query import rewrite_natural_date_keywords
|
||||
from documents.search._schema import build_schema
|
||||
from documents.search._tokenizer import register_tokenizers
|
||||
from documents.search._translate import InvalidDateQuery
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from django.contrib.auth.base_user import AbstractBaseUser
|
||||
@@ -405,12 +405,14 @@ class TestWhooshQueryRewriting:
|
||||
assert lo == "2023-12-01T05:00:00Z"
|
||||
assert hi == "2023-12-02T05:00:00Z"
|
||||
|
||||
def test_8digit_invalid_date_passes_through_unchanged(self) -> None:
|
||||
assert rewrite_natural_date_keywords("added:20231340", UTC) == "added:20231340"
|
||||
|
||||
def test_compact_14digit_invalid_date_passes_through_unchanged(self) -> None:
|
||||
# Month=13 makes datetime() raise ValueError; the token must be left as-is
|
||||
assert _rewrite_compact_date("20231300120000") == "20231300120000"
|
||||
def test_8digit_invalid_date_raises(self) -> None:
|
||||
# The translation pipeline raises InvalidDateQuery for unparsable dates
|
||||
# (e.g. month=13) so the API can surface a 400 telling the user the date
|
||||
# is malformed instead of silently returning zero results.
|
||||
with pytest.raises(InvalidDateQuery) as exc_info:
|
||||
rewrite_natural_date_keywords("added:20231340", UTC)
|
||||
assert exc_info.value.field == "added"
|
||||
assert exc_info.value.value == "20231340"
|
||||
|
||||
|
||||
class TestParseUserQuery:
|
||||
@@ -463,6 +465,67 @@ class TestParseUserQuery:
|
||||
) -> None:
|
||||
assert isinstance(parse_user_query(query_index, raw_query, UTC), tantivy.Query)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"raw_query",
|
||||
[
|
||||
# Partial date scalar (year only)
|
||||
pytest.param("created:2020", id="created_year_scalar"),
|
||||
# 8-digit compact date range in brackets
|
||||
pytest.param(
|
||||
"created:[20200101 TO 20201231]",
|
||||
id="created_8digit_bracket_range",
|
||||
),
|
||||
# Comma-separated field + date range (Whoosh v2 multi-clause syntax)
|
||||
pytest.param(
|
||||
"title:x,created:[2020 TO 2021]",
|
||||
id="title_comma_created_range",
|
||||
),
|
||||
# Field alias: type -> document_type
|
||||
pytest.param("type:invoice", id="type_alias"),
|
||||
# Multi-word date keyword
|
||||
pytest.param("created:previous week", id="created_previous_week"),
|
||||
# Full ISO datetime range
|
||||
pytest.param(
|
||||
"created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z]",
|
||||
id="created_iso_range",
|
||||
),
|
||||
# Comma-separated ISO ranges (Whoosh v2 syntax)
|
||||
pytest.param(
|
||||
"created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z],"
|
||||
"added:[2026-05-01T00:00:00Z TO 2026-06-01T00:00:00Z]",
|
||||
id="comma_iso_ranges",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_advanced_search_queries_do_not_raise(
|
||||
self,
|
||||
query_index: tantivy.Index,
|
||||
raw_query: str,
|
||||
) -> None:
|
||||
"""
|
||||
End-to-end: queries that the frontend sends must parse without raising.
|
||||
|
||||
This tests the full pipeline: translate_query -> tantivy parse_query.
|
||||
Equivalent to asserting HTTP 200 (not 400) for each query form.
|
||||
"""
|
||||
with time_machine.travel(datetime(2026, 6, 15, 12, 0, tzinfo=UTC), tick=False):
|
||||
assert isinstance(
|
||||
parse_user_query(query_index, raw_query, UTC),
|
||||
tantivy.Query,
|
||||
)
|
||||
|
||||
def test_invalid_date_propagates_not_swallowed(
|
||||
self,
|
||||
query_index: tantivy.Index,
|
||||
) -> None:
|
||||
# parse_user_query falls back to the raw query on unexpected translation
|
||||
# errors, but an InvalidDateQuery is intentional and must propagate so the
|
||||
# view can return a 400 instead of silently parsing the raw (invalid) date.
|
||||
with pytest.raises(InvalidDateQuery) as exc_info:
|
||||
parse_user_query(query_index, "created:202023", UTC)
|
||||
assert exc_info.value.field == "created"
|
||||
assert exc_info.value.value == "202023"
|
||||
|
||||
|
||||
class TestYearRangeRewriting:
|
||||
"""Whoosh-style year-only date ranges must be rewritten to ISO 8601."""
|
||||
@@ -542,11 +605,16 @@ class TestYearRangeRewriting:
|
||||
assert rewrite_natural_date_keywords(original, UTC) == original
|
||||
|
||||
def test_8digit_in_brackets_not_matched_as_year_range(self) -> None:
|
||||
# [YYYYMMDD TO YYYYMMDD] has 8-digit values - must not be caught by year rewriter
|
||||
# [YYYYMMDD TO YYYYMMDD]: the translation layer converts 8-digit bounds to
|
||||
# ISO day ranges. 20200101 -> 2020-01-01T00:00:00Z (lo of that day);
|
||||
# 20201231 -> the ceil of Dec 31 = 2021-01-01T00:00:00Z (exclusive end).
|
||||
# This is the correct and accepted behavior: old compact form becomes a
|
||||
# proper Tantivy-parseable ISO range.
|
||||
original = "created:[20200101 TO 20201231]"
|
||||
result = rewrite_natural_date_keywords(original, UTC)
|
||||
assert "20200101" in result or "2020-01-01" in result
|
||||
assert "20201231" in result or "2020-12-31" in result
|
||||
lo, hi = _range(result, "created")
|
||||
assert lo == "2020-01-01T00:00:00Z"
|
||||
assert hi == "2021-01-01T00:00:00Z"
|
||||
|
||||
|
||||
class TestNonDateFieldsNotRewritten:
|
||||
@@ -606,6 +674,16 @@ class TestNormalizeQuery:
|
||||
def test_normalize_expands_comma_separated_tags(self) -> None:
|
||||
assert normalize_query("tag:foo,bar") == "tag:foo AND tag:bar"
|
||||
|
||||
def test_normalize_comma_between_range_expressions(self) -> None:
|
||||
# Comma-separated field range expressions (Whoosh v2 syntax) must be
|
||||
# converted to AND so Tantivy does not receive an invalid comma.
|
||||
q = "created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z],added:[2026-05-01T00:00:00Z TO 2026-06-01T00:00:00Z]"
|
||||
assert normalize_query(q) == (
|
||||
"created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z]"
|
||||
" AND "
|
||||
"added:[2026-05-01T00:00:00Z TO 2026-06-01T00:00:00Z]"
|
||||
)
|
||||
|
||||
def test_normalize_expands_three_values(self) -> None:
|
||||
assert normalize_query("tag:foo,bar,baz") == "tag:foo AND tag:bar AND tag:baz"
|
||||
|
||||
|
||||
@@ -99,6 +99,25 @@ class TestTokenizers:
|
||||
)
|
||||
assert simple_search_index.searcher().search(q, limit=5).count == 1
|
||||
|
||||
def test_simple_search_analyzer_supports_model_limit_token_substrings(
|
||||
self,
|
||||
simple_search_index: tantivy.Index,
|
||||
) -> None:
|
||||
"""Simple substring search keeps tokens up to Document.title's model limit."""
|
||||
long_token = "abcdefghij" * 12 + "abcdefgh"
|
||||
writer = simple_search_index.writer()
|
||||
doc = tantivy.Document()
|
||||
doc.add_text("simple_content", long_token)
|
||||
writer.add_document(doc)
|
||||
writer.commit()
|
||||
simple_search_index.reload()
|
||||
q = tantivy.Query.regex_query(
|
||||
simple_search_index.schema,
|
||||
"simple_content",
|
||||
".*cdefg.*",
|
||||
)
|
||||
assert simple_search_index.searcher().search(q, limit=5).count == 1
|
||||
|
||||
def test_unsupported_language_logs_warning(self, caplog: LogCaptureFixture) -> None:
|
||||
"""Unsupported language codes should log a warning and disable stemming gracefully."""
|
||||
sb = tantivy.SchemaBuilder()
|
||||
|
||||
@@ -0,0 +1,742 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
import pytest
|
||||
import time_machine
|
||||
|
||||
from documents.search._dates import _precision_bounds
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tantivy
|
||||
from documents.search._query import _FIELD_BOOSTS
|
||||
from documents.search._query import DEFAULT_SEARCH_FIELDS
|
||||
from documents.search._translate import OPEN_HI
|
||||
from documents.search._translate import OPEN_LO
|
||||
from documents.search._translate import Comma
|
||||
from documents.search._translate import FieldRange
|
||||
from documents.search._translate import FieldValue
|
||||
from documents.search._translate import FieldValueList
|
||||
from documents.search._translate import InvalidDateQuery
|
||||
from documents.search._translate import Passthrough
|
||||
from documents.search._translate import resolve_commas
|
||||
from documents.search._translate import scan
|
||||
from documents.search._translate import translate_query
|
||||
from documents.search._translate import translate_range
|
||||
from documents.search._translate import translate_scalar
|
||||
|
||||
|
||||
@pytest.mark.search
|
||||
class TestPrecisionBounds:
|
||||
@pytest.mark.parametrize(
|
||||
("digits", "expected"),
|
||||
[
|
||||
("2020", ((2020, 1, 1), (2021, 1, 1))),
|
||||
("202003", ((2020, 3, 1), (2020, 4, 1))),
|
||||
("202012", ((2020, 12, 1), (2021, 1, 1))),
|
||||
("20200115", ((2020, 1, 15), (2020, 1, 16))),
|
||||
("20201231", ((2020, 12, 31), (2021, 1, 1))),
|
||||
],
|
||||
)
|
||||
def test_valid(self, digits, expected):
|
||||
lo, hi = _precision_bounds(digits)
|
||||
assert (lo.year, lo.month, lo.day) == expected[0]
|
||||
assert (hi.year, hi.month, hi.day) == expected[1]
|
||||
|
||||
@pytest.mark.parametrize("digits", ["202023", "20200230", "20201301", "20", "abcd"])
|
||||
def test_invalid_returns_none(self, digits):
|
||||
assert _precision_bounds(digits) is None
|
||||
|
||||
|
||||
@pytest.mark.search
|
||||
class TestScan:
|
||||
def test_plain_words_are_passthrough(self):
|
||||
assert scan("bank statement") == [Passthrough("bank statement")]
|
||||
|
||||
def test_field_value(self):
|
||||
assert scan("created:2020") == [FieldValue("created", "2020")]
|
||||
|
||||
def test_field_value_in_boolean(self):
|
||||
toks = scan("created:2020 OR foo")
|
||||
assert toks == [
|
||||
FieldValue("created", "2020"),
|
||||
Passthrough(" OR foo"),
|
||||
]
|
||||
|
||||
def test_field_value_in_parens(self):
|
||||
toks = scan("(created:2020 OR foo)")
|
||||
assert toks == [
|
||||
Passthrough("("),
|
||||
FieldValue("created", "2020"),
|
||||
Passthrough(" OR foo)"),
|
||||
]
|
||||
|
||||
def test_quoted_value(self):
|
||||
assert scan('correspondent:"A B"') == [FieldValue("correspondent", '"A B"')]
|
||||
|
||||
def test_field_range(self):
|
||||
assert scan("created:[2020 TO 2021]") == [
|
||||
FieldRange("created", "[", "2020", "2021", "]"),
|
||||
]
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("query", "expected"),
|
||||
[
|
||||
pytest.param(
|
||||
"created:[2020 to]",
|
||||
FieldRange("created", "[", "2020", "", "]"),
|
||||
id="open_upper",
|
||||
),
|
||||
pytest.param(
|
||||
"created:[to 2020]",
|
||||
FieldRange("created", "[", "", "2020", "]"),
|
||||
id="open_lower",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_open_range(self, query, expected):
|
||||
assert scan(query) == [expected]
|
||||
|
||||
def test_comma_inside_range_not_split(self):
|
||||
# No depth-0 comma here; the whole thing is one range token.
|
||||
toks = scan("created:[2020 TO 2021]")
|
||||
assert len(toks) == 1
|
||||
|
||||
# --- Edge-case / regression tests (scan must never raise) ---
|
||||
|
||||
def test_url_is_passthrough(self):
|
||||
# "http" is not a known field; the whole URL must pass through verbatim.
|
||||
assert scan("http://example.com") == [Passthrough("http://example.com")]
|
||||
|
||||
def test_unterminated_quote_is_passthrough(self):
|
||||
# title is a known field but the quoted value has no closing quote;
|
||||
# _consume_value returns None so the whole string falls into passthrough.
|
||||
assert scan('title:"abc') == [Passthrough('title:"abc')]
|
||||
|
||||
def test_unterminated_bracket_is_passthrough(self):
|
||||
# created is a known field but the range bracket is never closed;
|
||||
# _consume_range returns None so the whole string falls into passthrough.
|
||||
assert scan("created:[2020") == [Passthrough("created:[2020")]
|
||||
|
||||
def test_empty_value_at_end_is_passthrough(self):
|
||||
# created is a known field but there is no value after the colon
|
||||
# (_consume_value returns None for start >= n), so passthrough.
|
||||
assert scan("created:") == [Passthrough("created:")]
|
||||
|
||||
def test_value_containing_colon(self):
|
||||
# The bare-word value reader stops at whitespace/paren, not at colon,
|
||||
# so "2020:30" is consumed as a single value token.
|
||||
assert scan("created:2020:30") == [FieldValue("created", "2020:30")]
|
||||
|
||||
def test_comma_followed_by_unconsumable_value_stops(self):
|
||||
# A comma followed by whitespace is neither a value-list continuation nor a
|
||||
# clause separator: the value stops and the comma stays as passthrough.
|
||||
assert scan("tag:foo, bar") == [
|
||||
FieldValue("tag", "foo"),
|
||||
Passthrough(", bar"),
|
||||
]
|
||||
|
||||
def test_bracket_without_to_is_open_upper_bound(self):
|
||||
# A bracketed value with no TO falls back to (value, "") -> open upper bound.
|
||||
assert scan("created:[2020]") == [
|
||||
FieldRange("created", "[", "2020", "", "]"),
|
||||
]
|
||||
|
||||
def test_known_field_name_midword_is_passthrough(self):
|
||||
# A known field name embedded mid-word is not a field token (the
|
||||
# word-boundary guard); the whole run stays passthrough.
|
||||
assert scan("xtag:foo") == [Passthrough("xtag:foo")]
|
||||
|
||||
|
||||
@pytest.mark.search
|
||||
class TestCommaResolution:
|
||||
def test_value_list_multi_value_field(self):
|
||||
toks = resolve_commas(scan("tag:foo,bar"))
|
||||
assert toks == [FieldValueList("tag", ("foo", "bar"))]
|
||||
|
||||
def test_value_list_three(self):
|
||||
toks = resolve_commas(scan("tag_id:1,2,3"))
|
||||
assert toks == [FieldValueList("tag_id", ("1", "2", "3"))]
|
||||
|
||||
def test_text_field_comma_is_literal(self):
|
||||
# correspondent is not multi-value: comma stays inside the value.
|
||||
toks = resolve_commas(scan("correspondent:foo,bar"))
|
||||
assert toks == [FieldValue("correspondent", "foo,bar")]
|
||||
|
||||
def test_clause_separator_before_known_field(self):
|
||||
toks = resolve_commas(scan("tag:foo,type:bar"))
|
||||
assert toks == [FieldValue("tag", "foo"), Comma(), FieldValue("type", "bar")]
|
||||
|
||||
def test_clause_separator_after_range(self):
|
||||
toks = resolve_commas(scan("created:[2020 TO 2021],added:[2022 TO 2023]"))
|
||||
assert toks == [
|
||||
FieldRange("created", "[", "2020", "2021", "]"),
|
||||
Comma(),
|
||||
FieldRange("added", "[", "2022", "2023", "]"),
|
||||
]
|
||||
|
||||
def test_clause_separator_after_quote(self):
|
||||
toks = resolve_commas(scan('correspondent:"A B",created:[2020 TO 2021]'))
|
||||
assert toks == [
|
||||
FieldValue("correspondent", '"A B"'),
|
||||
Comma(),
|
||||
FieldRange("created", "[", "2020", "2021", "]"),
|
||||
]
|
||||
|
||||
def test_url_comma_is_literal_passthrough(self):
|
||||
toks = resolve_commas(scan("http://example.com/a,b"))
|
||||
assert toks == [Passthrough("http://example.com/a,b")]
|
||||
|
||||
def test_non_multi_value_comma_is_literal(self):
|
||||
# title is not in MULTI_VALUE_FIELDS: comma stays inside the value.
|
||||
toks = resolve_commas(scan("title:10,20"))
|
||||
assert toks == [FieldValue("title", "10,20")]
|
||||
|
||||
def test_clause_separator_before_known_date_field(self):
|
||||
# The comma between a bare value and a known date field acts as a
|
||||
# clause separator; both sides survive as distinct tokens.
|
||||
toks = resolve_commas(scan("correspondent:foo,created:[2020 TO 2021]"))
|
||||
assert toks == [
|
||||
FieldValue("correspondent", "foo"),
|
||||
Comma(),
|
||||
FieldRange("created", "[", "2020", "2021", "]"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.search
|
||||
class TestTranslateScalar:
|
||||
@pytest.mark.parametrize(
|
||||
("field", "value", "expected"),
|
||||
[
|
||||
(
|
||||
"created",
|
||||
"2020",
|
||||
"created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z]",
|
||||
),
|
||||
(
|
||||
"created",
|
||||
"202003",
|
||||
"created:[2020-03-01T00:00:00Z TO 2020-04-01T00:00:00Z]",
|
||||
),
|
||||
(
|
||||
"created",
|
||||
"20200115",
|
||||
"created:[2020-01-15T00:00:00Z TO 2020-01-16T00:00:00Z]",
|
||||
),
|
||||
(
|
||||
"created",
|
||||
"2020-01-15",
|
||||
"created:[2020-01-15T00:00:00Z TO 2020-01-16T00:00:00Z]",
|
||||
),
|
||||
(
|
||||
"created",
|
||||
"2020-03",
|
||||
"created:[2020-03-01T00:00:00Z TO 2020-04-01T00:00:00Z]",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_partial_and_iso_dates(self, field: str, value: str, expected: str) -> None:
|
||||
assert translate_scalar(field, value, UTC) == expected
|
||||
|
||||
def test_invalid_date_raises(self) -> None:
|
||||
with pytest.raises(InvalidDateQuery) as exc_info:
|
||||
translate_scalar("created", "202023", UTC)
|
||||
assert exc_info.value.field == "created"
|
||||
assert exc_info.value.value == "202023"
|
||||
|
||||
def test_keyword_delegates(self) -> None:
|
||||
# keyword path produces a range; just assert it is a created range
|
||||
out = translate_scalar("created", "today", UTC)
|
||||
assert out.startswith("created:[") and out.endswith("]")
|
||||
|
||||
def test_14digit_compact_datetime(self) -> None:
|
||||
out = translate_scalar("created", "20240115120000", UTC)
|
||||
assert "20240115120000" not in out
|
||||
assert out.startswith("created:")
|
||||
assert out == "created:[2024-01-15T12:00:00Z TO 2024-01-15T12:00:00Z]"
|
||||
|
||||
def test_14digit_invalid_month_raises(self) -> None:
|
||||
with pytest.raises(InvalidDateQuery) as exc_info:
|
||||
translate_scalar("created", "20231300120000", UTC)
|
||||
assert exc_info.value.field == "created"
|
||||
assert exc_info.value.value == "20231300120000"
|
||||
|
||||
def test_unrecognized_value_raises(self) -> None:
|
||||
# A value that is not a keyword, digits, ISO date, or compact timestamp
|
||||
# raises rather than producing invalid Tantivy syntax or silently matching
|
||||
# nothing.
|
||||
with pytest.raises(InvalidDateQuery) as exc_info:
|
||||
translate_scalar("created", "garbage", UTC)
|
||||
assert exc_info.value.field == "created"
|
||||
assert exc_info.value.value == "garbage"
|
||||
|
||||
|
||||
@pytest.mark.search
|
||||
class TestTranslateRange:
|
||||
@pytest.mark.parametrize(
|
||||
("lo", "hi", "expected"),
|
||||
[
|
||||
("2005", "2009", "created:[2005-01-01T00:00:00Z TO 2010-01-01T00:00:00Z]"),
|
||||
(
|
||||
"202001",
|
||||
"202006",
|
||||
"created:[2020-01-01T00:00:00Z TO 2020-07-01T00:00:00Z]",
|
||||
),
|
||||
(
|
||||
"20200101",
|
||||
"20201231",
|
||||
"created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z]",
|
||||
),
|
||||
(
|
||||
"2020-01-01",
|
||||
"2020-12-31",
|
||||
"created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z]",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_absolute_ranges(self, lo, hi, expected):
|
||||
assert translate_range("created", lo, hi, UTC) == expected
|
||||
|
||||
def test_reversed_swaps(self):
|
||||
assert translate_range("created", "2009", "2005", UTC) == (
|
||||
"created:[2005-01-01T00:00:00Z TO 2010-01-01T00:00:00Z]"
|
||||
)
|
||||
|
||||
def test_open_upper(self):
|
||||
out = translate_range("created", "2020", "", UTC)
|
||||
assert out == f"created:[2020-01-01T00:00:00Z TO {OPEN_HI}]"
|
||||
|
||||
def test_open_lower(self):
|
||||
out = translate_range("created", "", "2020", UTC)
|
||||
assert out == f"created:[{OPEN_LO} TO 2021-01-01T00:00:00Z]"
|
||||
|
||||
def test_invalid_bound_raises(self):
|
||||
with pytest.raises(InvalidDateQuery) as exc_info:
|
||||
translate_range("created", "202023", "2025", UTC)
|
||||
assert exc_info.value.field == "created"
|
||||
assert exc_info.value.value == "202023"
|
||||
|
||||
def test_invalid_high_bound_raises(self):
|
||||
# Low bound parses, high bound does not -> raise on the high bound.
|
||||
with pytest.raises(InvalidDateQuery) as exc_info:
|
||||
translate_range("created", "2020", "garbage", UTC)
|
||||
assert exc_info.value.field == "created"
|
||||
assert exc_info.value.value == "garbage"
|
||||
|
||||
|
||||
@pytest.mark.search
|
||||
class TestTranslateQuery:
|
||||
@pytest.mark.parametrize(
|
||||
("raw", "expected"),
|
||||
[
|
||||
(
|
||||
"created:2020",
|
||||
"created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z]",
|
||||
),
|
||||
("tag:foo,bar", "tag:foo AND tag:bar"),
|
||||
# 'type' is a user-facing alias rewritten to 'document_type' (the real schema field)
|
||||
("tag:foo,type:bar", "tag:foo AND document_type:bar"),
|
||||
(
|
||||
"created:[2020 TO 2021],added:[2022 TO 2023]",
|
||||
"created:[2020-01-01T00:00:00Z TO 2022-01-01T00:00:00Z]"
|
||||
" AND "
|
||||
"added:[2022-01-01T00:00:00Z TO 2024-01-01T00:00:00Z]",
|
||||
),
|
||||
# correspondent is not multi-value: comma stays literal inside the value
|
||||
("correspondent:foo,bar", "correspondent:foo,bar"),
|
||||
],
|
||||
)
|
||||
def test_golden(self, raw: str, expected: str) -> None:
|
||||
assert translate_query(raw, UTC) == expected
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"raw",
|
||||
[
|
||||
"created:2020",
|
||||
"created:202003",
|
||||
"created:[20200101 TO 20201231]",
|
||||
"created:[2020-01-01 TO 2020-12-31]",
|
||||
"created:[2020 to]",
|
||||
"created:[to 2020]",
|
||||
"title:x,created:[2020 TO 2021]",
|
||||
"created:2020 OR foo",
|
||||
"(created:2020 OR invoice)",
|
||||
"tag:foo,type:bar",
|
||||
"bank statement",
|
||||
],
|
||||
)
|
||||
def test_parse_acceptance(self, index: tantivy.Index, raw: str) -> None:
|
||||
translated = translate_query(raw, UTC)
|
||||
# Must not raise:
|
||||
index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
|
||||
|
||||
|
||||
@pytest.mark.search
|
||||
class TestFieldAliasing:
|
||||
"""Whoosh->Tantivy field-name aliasing (type/path -> document_type/storage_path)."""
|
||||
|
||||
def test_type_alias(self) -> None:
|
||||
assert translate_query("type:invoice", UTC) == "document_type:invoice"
|
||||
|
||||
def test_path_alias(self) -> None:
|
||||
assert translate_query("path:/foo/bar", UTC) == "storage_path:/foo/bar"
|
||||
|
||||
def test_type_id_alias(self) -> None:
|
||||
assert translate_query("type_id:5", UTC) == "document_type_id:5"
|
||||
|
||||
def test_path_id_alias(self) -> None:
|
||||
assert translate_query("path_id:7", UTC) == "storage_path_id:7"
|
||||
|
||||
def test_clause_separator_plus_alias(self) -> None:
|
||||
# Comma between known fields acts as AND separator; alias still applied.
|
||||
assert (
|
||||
translate_query("tag:foo,type:bar", UTC) == "tag:foo AND document_type:bar"
|
||||
)
|
||||
|
||||
def test_type_range_alias(self) -> None:
|
||||
# type is not a date field; range passes through verbatim with alias applied.
|
||||
assert (
|
||||
translate_query("type:[2020 TO 2021]", UTC)
|
||||
== "document_type:[2020 TO 2021]"
|
||||
)
|
||||
|
||||
def test_parse_acceptance_type(self, index: tantivy.Index) -> None:
|
||||
# Translated output must be accepted by the real Tantivy parser.
|
||||
translated = translate_query("type:invoice", UTC)
|
||||
index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
|
||||
|
||||
def test_parse_acceptance_path(self, index: tantivy.Index) -> None:
|
||||
translated = translate_query("path:foo", UTC)
|
||||
index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
|
||||
|
||||
|
||||
# Freeze time so relative-date tests are deterministic.
|
||||
_FROZEN_NOW = datetime(2026, 3, 28, 12, 0, 0, tzinfo=UTC)
|
||||
|
||||
|
||||
@pytest.mark.search
|
||||
class TestRelativeRanges:
|
||||
"""Relative date-range tokens resolved against a frozen clock."""
|
||||
|
||||
@time_machine.travel(_FROZEN_NOW, tick=False)
|
||||
def test_minus_7_days_to_now(self) -> None:
|
||||
assert translate_query("added:[-7 days to now]", UTC) == (
|
||||
"added:[2026-03-21T12:00:00Z TO 2026-03-28T12:00:00Z]"
|
||||
)
|
||||
|
||||
@time_machine.travel(_FROZEN_NOW, tick=False)
|
||||
def test_minus_1_week_to_now(self) -> None:
|
||||
assert translate_query("added:[-1 week to now]", UTC) == (
|
||||
"added:[2026-03-21T12:00:00Z TO 2026-03-28T12:00:00Z]"
|
||||
)
|
||||
|
||||
@time_machine.travel(_FROZEN_NOW, tick=False)
|
||||
def test_minus_1_month_to_now(self) -> None:
|
||||
assert translate_query("created:[-1 month to now]", UTC) == (
|
||||
"created:[2026-02-28T12:00:00Z TO 2026-03-28T12:00:00Z]"
|
||||
)
|
||||
|
||||
@time_machine.travel(_FROZEN_NOW, tick=False)
|
||||
def test_minus_1_year_to_now(self) -> None:
|
||||
assert translate_query("modified:[-1 year to now]", UTC) == (
|
||||
"modified:[2025-03-28T12:00:00Z TO 2026-03-28T12:00:00Z]"
|
||||
)
|
||||
|
||||
@time_machine.travel(_FROZEN_NOW, tick=False)
|
||||
def test_minus_3_hours_to_now(self) -> None:
|
||||
assert translate_query("added:[-3 hours to now]", UTC) == (
|
||||
"added:[2026-03-28T09:00:00Z TO 2026-03-28T12:00:00Z]"
|
||||
)
|
||||
|
||||
@time_machine.travel(_FROZEN_NOW, tick=False)
|
||||
def test_uppercase_units(self) -> None:
|
||||
assert translate_query("added:[-1 WEEK TO NOW]", UTC) == (
|
||||
"added:[2026-03-21T12:00:00Z TO 2026-03-28T12:00:00Z]"
|
||||
)
|
||||
|
||||
@time_machine.travel(_FROZEN_NOW, tick=False)
|
||||
def test_now_minus_7d_compact(self) -> None:
|
||||
assert translate_query("added:[now-7d TO now]", UTC) == (
|
||||
"added:[2026-03-21T12:00:00Z TO 2026-03-28T12:00:00Z]"
|
||||
)
|
||||
|
||||
@time_machine.travel(_FROZEN_NOW, tick=False)
|
||||
def test_reversed_range_swapped(self) -> None:
|
||||
# now+1h TO now-1h is reversed; translate_range swaps -> lo=now-1h, hi=now+1h
|
||||
assert translate_query("added:[now+1h TO now-1h]", UTC) == (
|
||||
"added:[2026-03-28T11:00:00Z TO 2026-03-28T13:00:00Z]"
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"raw",
|
||||
[
|
||||
"added:[-7 days to now]",
|
||||
"added:[-1 week to now]",
|
||||
"created:[-1 month to now]",
|
||||
"modified:[-1 year to now]",
|
||||
"added:[-3 hours to now]",
|
||||
"added:[now-7d TO now]",
|
||||
"added:[now+1h TO now-1h]",
|
||||
],
|
||||
)
|
||||
@time_machine.travel(_FROZEN_NOW, tick=False)
|
||||
def test_parse_acceptance(self, index: tantivy.Index, raw: str) -> None:
|
||||
translated = translate_query(raw, UTC)
|
||||
index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
|
||||
|
||||
|
||||
@pytest.mark.search
|
||||
class TestOperatorNormalization:
|
||||
"""Post-render operator normalization in translate_query."""
|
||||
|
||||
def test_spaced_dash_removed(self) -> None:
|
||||
assert (
|
||||
translate_query("H52.1 - Kurzsichtigkeit", UTC) == "H52.1 Kurzsichtigkeit"
|
||||
)
|
||||
|
||||
def test_spaced_dash_simple(self) -> None:
|
||||
assert translate_query("bar - baz", UTC) == "bar baz"
|
||||
|
||||
def test_trailing_operator_stripped(self) -> None:
|
||||
assert translate_query("foo -", UTC) == "foo"
|
||||
|
||||
def test_date_range_preserved(self) -> None:
|
||||
out = translate_query("created:[2020 TO 2021]", UTC)
|
||||
# Must not corrupt the ISO range
|
||||
assert out == "created:[2020-01-01T00:00:00Z TO 2022-01-01T00:00:00Z]"
|
||||
|
||||
def test_date_scalar_with_or(self) -> None:
|
||||
out = translate_query("created:2020 OR foo", UTC)
|
||||
# The created scalar becomes a range; " OR foo" passes through verbatim.
|
||||
assert out.startswith("created:[")
|
||||
assert "OR foo" in out
|
||||
|
||||
def test_parse_acceptance_spaced_dash(self, index: tantivy.Index) -> None:
|
||||
translated = translate_query("H52.1 - Kurzsichtigkeit", UTC)
|
||||
index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
|
||||
|
||||
def test_parse_acceptance_trailing_op(self, index: tantivy.Index) -> None:
|
||||
translated = translate_query("foo -", UTC)
|
||||
index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
|
||||
|
||||
|
||||
@pytest.mark.search
|
||||
class TestMultiWordDateKeywords:
|
||||
"""scan() must consume multi-word date keywords as a single value."""
|
||||
|
||||
def test_scan_previous_week_as_single_token(self) -> None:
|
||||
# "created:previous week" must produce one FieldValue with value "previous week",
|
||||
# not FieldValue("created","previous") + Passthrough(" week").
|
||||
toks = scan("created:previous week")
|
||||
assert toks == [FieldValue("created", "previous week")]
|
||||
|
||||
def test_scan_this_month_as_single_token(self) -> None:
|
||||
toks = scan("added:this month")
|
||||
assert toks == [FieldValue("added", "this month")]
|
||||
|
||||
def test_scan_previous_month_as_single_token(self) -> None:
|
||||
toks = scan("created:previous month")
|
||||
assert toks == [FieldValue("created", "previous month")]
|
||||
|
||||
def test_scan_this_year_as_single_token(self) -> None:
|
||||
toks = scan("added:this year")
|
||||
assert toks == [FieldValue("added", "this year")]
|
||||
|
||||
def test_scan_previous_year_as_single_token(self) -> None:
|
||||
toks = scan("created:previous year")
|
||||
assert toks == [FieldValue("created", "previous year")]
|
||||
|
||||
def test_scan_previous_quarter_as_single_token(self) -> None:
|
||||
toks = scan("created:previous quarter")
|
||||
assert toks == [FieldValue("created", "previous quarter")]
|
||||
|
||||
def test_quoted_multi_word_keyword_still_works(self) -> None:
|
||||
# The quoted form must continue to work as before.
|
||||
toks = scan('created:"previous week"')
|
||||
assert toks == [FieldValue("created", '"previous week"')]
|
||||
|
||||
def test_non_date_field_not_affected(self) -> None:
|
||||
# "previous" stops at the space for non-date fields; " week" passes through.
|
||||
toks = scan("correspondent:previous week")
|
||||
assert toks == [
|
||||
FieldValue("correspondent", "previous"),
|
||||
Passthrough(" week"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.search
|
||||
class TestKeywordDateResolution:
|
||||
"""Relative date keywords resolve to exact ISO ranges against a frozen clock.
|
||||
|
||||
Frozen at 2026-03-28 12:00 UTC (a Saturday in Q1) so the week, month,
|
||||
quarter and year rollovers are all exercised by a single anchor.
|
||||
"""
|
||||
|
||||
# created is a DateField: bounds are UTC midnight, no timezone offset.
|
||||
@pytest.mark.parametrize(
|
||||
("keyword", "expected"),
|
||||
[
|
||||
pytest.param(
|
||||
"today",
|
||||
"created:[2026-03-28T00:00:00Z TO 2026-03-29T00:00:00Z]",
|
||||
id="today",
|
||||
),
|
||||
pytest.param(
|
||||
"yesterday",
|
||||
"created:[2026-03-27T00:00:00Z TO 2026-03-28T00:00:00Z]",
|
||||
id="yesterday",
|
||||
),
|
||||
pytest.param(
|
||||
"previous week",
|
||||
"created:[2026-03-16T00:00:00Z TO 2026-03-23T00:00:00Z]",
|
||||
id="previous-week",
|
||||
),
|
||||
pytest.param(
|
||||
"this month",
|
||||
"created:[2026-03-01T00:00:00Z TO 2026-04-01T00:00:00Z]",
|
||||
id="this-month",
|
||||
),
|
||||
pytest.param(
|
||||
"previous month",
|
||||
"created:[2026-02-01T00:00:00Z TO 2026-03-01T00:00:00Z]",
|
||||
id="previous-month",
|
||||
),
|
||||
pytest.param(
|
||||
"this year",
|
||||
"created:[2026-01-01T00:00:00Z TO 2027-01-01T00:00:00Z]",
|
||||
id="this-year",
|
||||
),
|
||||
pytest.param(
|
||||
"previous year",
|
||||
"created:[2025-01-01T00:00:00Z TO 2026-01-01T00:00:00Z]",
|
||||
id="previous-year",
|
||||
),
|
||||
pytest.param(
|
||||
"previous quarter",
|
||||
"created:[2025-10-01T00:00:00Z TO 2026-01-01T00:00:00Z]",
|
||||
id="previous-quarter",
|
||||
),
|
||||
],
|
||||
)
|
||||
@time_machine.travel(_FROZEN_NOW, tick=False)
|
||||
def test_date_only_field_keyword_ranges(
|
||||
self,
|
||||
keyword: str,
|
||||
expected: str,
|
||||
) -> None:
|
||||
assert translate_query(f"created:{keyword}", UTC) == expected
|
||||
|
||||
# added is a DateTimeField: local-tz midnight converted to UTC. Tokyo
|
||||
# (+09:00, no DST) shifts each midnight boundary back to 15:00Z the day
|
||||
# before, so this also exercises the local-midnight offset path.
|
||||
@pytest.mark.parametrize(
|
||||
("keyword", "expected"),
|
||||
[
|
||||
pytest.param(
|
||||
"today",
|
||||
"added:[2026-03-27T15:00:00Z TO 2026-03-28T15:00:00Z]",
|
||||
id="today",
|
||||
),
|
||||
pytest.param(
|
||||
"yesterday",
|
||||
"added:[2026-03-26T15:00:00Z TO 2026-03-27T15:00:00Z]",
|
||||
id="yesterday",
|
||||
),
|
||||
pytest.param(
|
||||
"previous week",
|
||||
"added:[2026-03-15T15:00:00Z TO 2026-03-22T15:00:00Z]",
|
||||
id="previous-week",
|
||||
),
|
||||
pytest.param(
|
||||
"this month",
|
||||
"added:[2026-02-28T15:00:00Z TO 2026-03-31T15:00:00Z]",
|
||||
id="this-month",
|
||||
),
|
||||
pytest.param(
|
||||
"previous month",
|
||||
"added:[2026-01-31T15:00:00Z TO 2026-02-28T15:00:00Z]",
|
||||
id="previous-month",
|
||||
),
|
||||
pytest.param(
|
||||
"this year",
|
||||
"added:[2025-12-31T15:00:00Z TO 2026-12-31T15:00:00Z]",
|
||||
id="this-year",
|
||||
),
|
||||
pytest.param(
|
||||
"previous year",
|
||||
"added:[2024-12-31T15:00:00Z TO 2025-12-31T15:00:00Z]",
|
||||
id="previous-year",
|
||||
),
|
||||
pytest.param(
|
||||
"previous quarter",
|
||||
"added:[2025-09-30T15:00:00Z TO 2025-12-31T15:00:00Z]",
|
||||
id="previous-quarter",
|
||||
),
|
||||
],
|
||||
)
|
||||
@time_machine.travel(_FROZEN_NOW, tick=False)
|
||||
def test_datetime_field_keyword_ranges_local_tz(
|
||||
self,
|
||||
keyword: str,
|
||||
expected: str,
|
||||
) -> None:
|
||||
assert translate_query(f"added:{keyword}", ZoneInfo("Asia/Tokyo")) == expected
|
||||
|
||||
|
||||
@pytest.mark.search
|
||||
class TestISODatetimeBounds:
|
||||
"""Full ISO datetime tokens in range bounds must be parsed directly."""
|
||||
|
||||
def test_translate_range_iso_bounds_passthrough(self) -> None:
|
||||
# Already-ISO datetime bounds must pass through as-is (exact instant).
|
||||
result = translate_range(
|
||||
"created",
|
||||
"2020-01-01T00:00:00Z",
|
||||
"2021-01-01T00:00:00Z",
|
||||
UTC,
|
||||
)
|
||||
assert result == "created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z]"
|
||||
|
||||
def test_translate_query_iso_range_preserved(self) -> None:
|
||||
q = "created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z]"
|
||||
assert translate_query(q, UTC) == q
|
||||
|
||||
def test_translate_query_comma_separated_iso_ranges(self) -> None:
|
||||
q = (
|
||||
"created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z],"
|
||||
"added:[2026-05-01T00:00:00Z TO 2026-06-01T00:00:00Z]"
|
||||
)
|
||||
result = translate_query(q, UTC)
|
||||
assert result == (
|
||||
"created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z]"
|
||||
" AND "
|
||||
"added:[2026-05-01T00:00:00Z TO 2026-06-01T00:00:00Z]"
|
||||
)
|
||||
|
||||
def test_invalid_iso_datetime_raises(self) -> None:
|
||||
# A token with "T" that is not valid ISO datetime -> raise.
|
||||
with pytest.raises(InvalidDateQuery) as exc_info:
|
||||
translate_range(
|
||||
"created",
|
||||
"2020-01-01T99:00:00Z",
|
||||
"2021-01-01T00:00:00Z",
|
||||
UTC,
|
||||
)
|
||||
assert exc_info.value.field == "created"
|
||||
assert exc_info.value.value == "2020-01-01T99:00:00Z"
|
||||
|
||||
def test_parse_acceptance_iso_bounds(self, index: tantivy.Index) -> None:
|
||||
q = "created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z]"
|
||||
translated = translate_query(q, UTC)
|
||||
index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
|
||||
|
||||
def test_parse_acceptance_comma_iso_ranges(self, index: tantivy.Index) -> None:
|
||||
q = (
|
||||
"created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z],"
|
||||
"added:[2026-05-01T00:00:00Z TO 2026-06-01T00:00:00Z]"
|
||||
)
|
||||
translated = translate_query(q, UTC)
|
||||
index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
|
||||
@@ -82,6 +82,7 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
|
||||
"llm_api_key": None,
|
||||
"llm_endpoint": None,
|
||||
"llm_output_language": None,
|
||||
"llm_request_timeout": None,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
import unicodedata
|
||||
from typing import TYPE_CHECKING
|
||||
from unittest import mock
|
||||
|
||||
import celery.result
|
||||
import pytest
|
||||
from django.core.files.uploadedfile import SimpleUploadedFile
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from documents.data_models import ConsumableDocument
|
||||
from documents.data_models import DocumentMetadataOverrides
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def consume_file_mock():
|
||||
with mock.patch("documents.tasks.consume_file.apply_async") as m:
|
||||
m.return_value = celery.result.AsyncResult(id="test-task-id")
|
||||
yield m
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def directories(tmp_path, settings, _media_settings):
|
||||
scratch = tmp_path / "scratch"
|
||||
scratch.mkdir()
|
||||
settings.SCRATCH_DIR = scratch
|
||||
return scratch
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestPostDocumentNFCNormalization:
|
||||
def test_nfd_filename_normalized_to_nfc(
|
||||
self,
|
||||
admin_client,
|
||||
consume_file_mock: mock.MagicMock,
|
||||
directories,
|
||||
):
|
||||
"""Uploaded file with NFD filename must have its name stored as NFC."""
|
||||
nfd = unicodedata.normalize("NFD", "Rechnung März.pdf")
|
||||
nfc = unicodedata.normalize("NFC", "Rechnung März.pdf")
|
||||
|
||||
# Verify our test strings actually differ at the byte level
|
||||
assert nfd != nfc
|
||||
|
||||
uploaded = SimpleUploadedFile(
|
||||
nfd,
|
||||
b"%PDF-1.4 test",
|
||||
content_type="application/pdf",
|
||||
)
|
||||
response = admin_client.post(
|
||||
"/api/documents/post_document/",
|
||||
{"document": uploaded},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
|
||||
task_kwargs = consume_file_mock.call_args.kwargs["kwargs"]
|
||||
input_doc: ConsumableDocument = task_kwargs["input_doc"]
|
||||
overrides: DocumentMetadataOverrides = task_kwargs["overrides"]
|
||||
|
||||
# The temp file on disk must have an NFC name
|
||||
assert input_doc.original_file.name == nfc, (
|
||||
f"Expected NFC filename {nfc!r}, got {input_doc.original_file.name!r}"
|
||||
)
|
||||
# The override filename stored for later use must also be NFC
|
||||
assert overrides.filename == nfc, (
|
||||
f"Expected NFC override filename {nfc!r}, got {overrides.filename!r}"
|
||||
)
|
||||
assert unicodedata.is_normalized("NFC", overrides.filename)
|
||||
|
||||
def test_already_nfc_filename_unchanged(
|
||||
self,
|
||||
admin_client,
|
||||
consume_file_mock: mock.MagicMock,
|
||||
directories,
|
||||
):
|
||||
"""Uploaded file with already-NFC filename must pass through unchanged."""
|
||||
nfc = unicodedata.normalize("NFC", "Invoice_2024.pdf")
|
||||
|
||||
uploaded = SimpleUploadedFile(
|
||||
nfc,
|
||||
b"%PDF-1.4 test",
|
||||
content_type="application/pdf",
|
||||
)
|
||||
response = admin_client.post(
|
||||
"/api/documents/post_document/",
|
||||
{"document": uploaded},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
|
||||
task_kwargs = consume_file_mock.call_args.kwargs["kwargs"]
|
||||
overrides: DocumentMetadataOverrides = task_kwargs["overrides"]
|
||||
|
||||
assert overrides.filename == nfc
|
||||
assert unicodedata.is_normalized("NFC", overrides.filename)
|
||||
@@ -725,9 +725,11 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
|
||||
GIVEN:
|
||||
- One document added right now
|
||||
WHEN:
|
||||
- Query with invalid added date
|
||||
- Query with an invalid added date
|
||||
THEN:
|
||||
- 400 Bad Request returned (Tantivy rejects invalid date field syntax)
|
||||
- 400 Bad Request with a message naming the malformed date, so the
|
||||
user knows their date is invalid rather than silently getting zero
|
||||
results
|
||||
"""
|
||||
d1 = Document.objects.create(
|
||||
title="invoice",
|
||||
@@ -740,8 +742,9 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
|
||||
|
||||
response = self.client.get("/api/documents/?query=added:invalid-date")
|
||||
|
||||
# Tantivy rejects unparsable field queries with a 400
|
||||
# An unparsable date is reported as a malformed query, not silently empty.
|
||||
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
|
||||
self.assertIn("invalid-date", str(response.data["query"]))
|
||||
|
||||
@override_settings(
|
||||
TIME_ZONE="UTC",
|
||||
|
||||
@@ -216,6 +216,77 @@ class TestSystemStatus(APITestCase):
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
self.assertEqual(response.data["tasks"]["celery_status"], "OK")
|
||||
|
||||
@mock.patch("celery.app.control.Inspect.ping")
|
||||
def test_system_status_celery_ping_none(self, mock_ping) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- Celery ping returns no worker responses
|
||||
WHEN:
|
||||
- The user requests the system status
|
||||
THEN:
|
||||
- The response contains a warning celery status
|
||||
"""
|
||||
mock_ping.return_value = None
|
||||
self.client.force_login(self.user)
|
||||
response = self.client.get(self.ENDPOINT)
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
self.assertEqual(response.data["tasks"]["celery_status"], "WARNING")
|
||||
self.assertEqual(
|
||||
response.data["tasks"]["celery_error"],
|
||||
"No celery workers responded to ping. This may be temporary.",
|
||||
)
|
||||
|
||||
@mock.patch("celery.app.control.Inspect.ping")
|
||||
def test_system_status_celery_ping_unexpected_responses(self, mock_ping) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- Celery ping returns an unexpected worker response
|
||||
WHEN:
|
||||
- The user requests the system status
|
||||
THEN:
|
||||
- The response contains a warning celery status
|
||||
"""
|
||||
self.client.force_login(self.user)
|
||||
for ping_response in (
|
||||
{"hostname": {"ok": "not-pong"}},
|
||||
{"hostname": {}},
|
||||
{"hostname": "pong"},
|
||||
):
|
||||
with self.subTest(ping_response=ping_response):
|
||||
mock_ping.return_value = ping_response
|
||||
response = self.client.get(self.ENDPOINT)
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
self.assertEqual(response.data["tasks"]["celery_status"], "WARNING")
|
||||
self.assertEqual(response.data["tasks"]["celery_url"], "hostname")
|
||||
self.assertEqual(
|
||||
response.data["tasks"]["celery_error"],
|
||||
"Celery worker responded unexpectedly.",
|
||||
)
|
||||
|
||||
@mock.patch("documents.views.sleep")
|
||||
@mock.patch("celery.app.control.Inspect.ping")
|
||||
def test_system_status_celery_ping_retry_success(
|
||||
self,
|
||||
mock_ping,
|
||||
mock_sleep,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- Celery ping fails once but succeeds on retry
|
||||
WHEN:
|
||||
- The user requests the system status
|
||||
THEN:
|
||||
- The response contains an OK celery status
|
||||
"""
|
||||
mock_ping.side_effect = [None, {"hostname": {"ok": "pong"}}]
|
||||
self.client.force_login(self.user)
|
||||
response = self.client.get(self.ENDPOINT)
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
self.assertEqual(response.data["tasks"]["celery_status"], "OK")
|
||||
self.assertIsNone(response.data["tasks"]["celery_error"])
|
||||
self.assertEqual(mock_ping.call_count, 2)
|
||||
mock_sleep.assert_called_once_with(0.25)
|
||||
|
||||
@mock.patch("documents.search.get_backend")
|
||||
def test_system_status_index_ok(self, mock_get_backend) -> None:
|
||||
"""
|
||||
|
||||
@@ -0,0 +1,187 @@
|
||||
"""
|
||||
Tests for NFC Unicode normalization in generate_filename / FilePathTemplate.render().
|
||||
|
||||
NFC `ü` (UTF-8: c3 bc) and NFD `ü` (UTF-8: 75 cc 88) are visually identical but
|
||||
produce different byte sequences. On Linux (ext4, ZFS) these are distinct filenames.
|
||||
All paths produced by the templating system must be NFC-normalized.
|
||||
"""
|
||||
|
||||
import unicodedata
|
||||
|
||||
import pytest
|
||||
|
||||
from documents.file_handling import generate_filename
|
||||
from documents.models import CustomField
|
||||
from documents.models import CustomFieldInstance
|
||||
from documents.tests.factories import CorrespondentFactory
|
||||
from documents.tests.factories import DocumentFactory
|
||||
from documents.tests.factories import StoragePathFactory
|
||||
from documents.tests.factories import TagFactory
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestGenerateFilenameNFCNormalization:
|
||||
@pytest.mark.parametrize(
|
||||
"raw,display",
|
||||
[
|
||||
(unicodedata.normalize("NFD", "Gemüse"), "Gemüse"),
|
||||
(unicodedata.normalize("NFD", "Café"), "Café"),
|
||||
(unicodedata.normalize("NFD", "naïve"), "naïve"),
|
||||
],
|
||||
)
|
||||
def test_nfd_title_normalized_to_nfc(self, settings, raw, display):
|
||||
"""NFD title must produce NFC path bytes."""
|
||||
settings.FILENAME_FORMAT = "{{ title }}"
|
||||
nfc = unicodedata.normalize("NFC", display)
|
||||
assert raw != nfc # confirm byte-level difference
|
||||
|
||||
doc = DocumentFactory(title=raw, mime_type="application/pdf")
|
||||
result = generate_filename(doc)
|
||||
|
||||
assert str(result) == f"{nfc}.pdf"
|
||||
assert str(result).encode() == f"{nfc}.pdf".encode()
|
||||
|
||||
def test_nfd_correspondent_normalized_to_nfc(self, settings):
|
||||
"""NFD correspondent name must produce NFC path component."""
|
||||
settings.FILENAME_FORMAT = "{{ correspondent }}/{{ title }}"
|
||||
nfd = unicodedata.normalize("NFD", "Müller")
|
||||
nfc = unicodedata.normalize("NFC", "Müller")
|
||||
|
||||
correspondent = CorrespondentFactory(name=nfd)
|
||||
doc = DocumentFactory(
|
||||
title="invoice",
|
||||
correspondent=correspondent,
|
||||
mime_type="application/pdf",
|
||||
)
|
||||
result = generate_filename(doc)
|
||||
|
||||
assert str(result) == f"{nfc}/invoice.pdf"
|
||||
assert str(result).encode() == f"{nfc}/invoice.pdf".encode()
|
||||
|
||||
def test_nfd_storage_path_normalized_to_nfc(self, settings):
|
||||
"""NFD literal in StoragePath.path template must produce NFC path bytes."""
|
||||
settings.FILENAME_FORMAT = None
|
||||
nfd = unicodedata.normalize("NFD", "Büro")
|
||||
nfc = unicodedata.normalize("NFC", "Büro")
|
||||
|
||||
# StoragePath.path is used directly as the format/template string.
|
||||
# Literal NFD characters in the template must survive rendering as NFC.
|
||||
sp = StoragePathFactory(path=f"{nfd}/{{{{ title }}}}")
|
||||
doc = DocumentFactory(title="doc", storage_path=sp, mime_type="application/pdf")
|
||||
result = generate_filename(doc)
|
||||
|
||||
assert str(result).encode() == f"{nfc}/doc.pdf".encode()
|
||||
|
||||
def test_nfd_raw_document_title_normalized_to_nfc(self, settings):
|
||||
"""NFD title accessed via document.title (unsanitized context) must also be NFC."""
|
||||
settings.FILENAME_FORMAT = "{{ document.title }}"
|
||||
nfd = unicodedata.normalize("NFD", "Café")
|
||||
nfc = unicodedata.normalize("NFC", "Café")
|
||||
|
||||
doc = DocumentFactory(title=nfd, mime_type="application/pdf")
|
||||
result = generate_filename(doc)
|
||||
|
||||
assert str(result) == f"{nfc}.pdf"
|
||||
assert str(result).encode() == f"{nfc}.pdf".encode()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestContextBuilderNFCNormalization:
|
||||
"""
|
||||
Defense-in-depth: context builder functions must NFC-normalize string inputs
|
||||
before passing them to sanitize_filename(). Task 1 already normalizes the
|
||||
final rendered path via clean_filepath(), so these tests may already pass;
|
||||
they exist as regression guards for the context-builder layer.
|
||||
"""
|
||||
|
||||
def test_nfd_tag_name_normalized_in_tag_list(self, settings):
|
||||
"""NFD tag name must appear as NFC bytes in the {{ tag_list }} shorthand."""
|
||||
settings.FILENAME_FORMAT = "{{ tag_list }}/{{ title }}"
|
||||
nfd = unicodedata.normalize("NFD", "Büro")
|
||||
nfc = unicodedata.normalize("NFC", "Büro")
|
||||
assert nfd != nfc # confirm they differ at byte level
|
||||
|
||||
tag = TagFactory(name=nfd)
|
||||
doc = DocumentFactory(title="doc", mime_type="application/pdf")
|
||||
doc.tags.set([tag])
|
||||
|
||||
result = generate_filename(doc)
|
||||
|
||||
assert str(result).encode() == f"{nfc}/doc.pdf".encode()
|
||||
|
||||
def test_nfd_original_name_normalized_to_nfc(self, settings):
|
||||
settings.FILENAME_FORMAT = "{{ original_name }}"
|
||||
nfd = unicodedata.normalize("NFD", "Rechnung März")
|
||||
nfc = unicodedata.normalize("NFC", "Rechnung März")
|
||||
|
||||
doc = DocumentFactory(
|
||||
original_filename=f"{nfd}.pdf",
|
||||
mime_type="application/pdf",
|
||||
)
|
||||
result = generate_filename(doc)
|
||||
|
||||
assert str(result).encode() == f"{nfc}.pdf".encode()
|
||||
|
||||
def test_nfd_custom_field_string_value_normalized(self, settings):
|
||||
"""NFD value in a STRING-type custom field must appear as NFC in the context."""
|
||||
settings.FILENAME_FORMAT = (
|
||||
"{{ custom_fields['Location']['value'] }}/{{ title }}"
|
||||
)
|
||||
nfd_value = unicodedata.normalize("NFD", "Düsseldorf")
|
||||
nfc_value = unicodedata.normalize("NFC", "Düsseldorf")
|
||||
assert nfd_value != nfc_value
|
||||
|
||||
doc = DocumentFactory(title="report", mime_type="application/pdf")
|
||||
cf = CustomField.objects.create(
|
||||
name="Location",
|
||||
data_type=CustomField.FieldDataType.STRING,
|
||||
)
|
||||
CustomFieldInstance.objects.create(
|
||||
document=doc,
|
||||
field=cf,
|
||||
value_text=nfd_value,
|
||||
)
|
||||
|
||||
result = generate_filename(doc)
|
||||
|
||||
assert str(result).encode() == f"{nfc_value}/report.pdf".encode()
|
||||
|
||||
def test_nfd_custom_field_name_normalized_as_key(self, settings):
|
||||
"""NFD characters in a custom field name must appear as NFC in the context dict key."""
|
||||
nfd_name = unicodedata.normalize("NFD", "Größe")
|
||||
nfc_name = unicodedata.normalize("NFC", "Größe")
|
||||
assert nfd_name != nfc_name
|
||||
|
||||
settings.FILENAME_FORMAT = f"{{% if custom_fields['{nfc_name}'] %}}{{{{ custom_fields['{nfc_name}']['value'] }}}}/{{{{ title }}}}{{% else %}}{{{{ title }}}}{{% endif %}}"
|
||||
|
||||
doc = DocumentFactory(title="letter", mime_type="application/pdf")
|
||||
cf = CustomField.objects.create(
|
||||
name=nfd_name,
|
||||
data_type=CustomField.FieldDataType.STRING,
|
||||
)
|
||||
CustomFieldInstance.objects.create(
|
||||
document=doc,
|
||||
field=cf,
|
||||
value_text="Berlin",
|
||||
)
|
||||
|
||||
result = generate_filename(doc)
|
||||
|
||||
# If field name key is NFC-normalized, the template condition succeeds
|
||||
# and result is "Berlin/letter.pdf"; otherwise it falls back to "letter.pdf"
|
||||
assert str(result) == "Berlin/letter.pdf"
|
||||
|
||||
def test_nfd_tag_name_list_normalized_to_nfc(self, settings):
|
||||
"""NFD tag names in tag_name_list must appear as NFC bytes when iterated."""
|
||||
settings.FILENAME_FORMAT = (
|
||||
"{% for t in tag_name_list %}{{ t }}{% endfor %}/{{ title }}"
|
||||
)
|
||||
nfd = unicodedata.normalize("NFD", "Büro")
|
||||
nfc = unicodedata.normalize("NFC", "Büro")
|
||||
assert nfd != nfc # confirm byte-level difference
|
||||
|
||||
doc = DocumentFactory(title="doc", mime_type="application/pdf")
|
||||
doc.tags.add(TagFactory(name=nfd))
|
||||
result = generate_filename(doc)
|
||||
|
||||
assert str(result).encode() == f"{nfc}/doc.pdf".encode()
|
||||
@@ -684,6 +684,7 @@ class ConsumerThread(Thread):
|
||||
subdirs_as_tags: bool = False,
|
||||
polling_interval: float = 0,
|
||||
stability_delay: float = 0.1,
|
||||
rescan_interval: float | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.consumption_dir = consumption_dir
|
||||
@@ -693,6 +694,8 @@ class ConsumerThread(Thread):
|
||||
self.polling_interval = polling_interval
|
||||
self.stability_delay = stability_delay
|
||||
self.cmd = Command()
|
||||
if rescan_interval is not None:
|
||||
self.cmd.rescan_interval_s = rescan_interval
|
||||
self.cmd.stop_flag.clear()
|
||||
# Non-daemon ensures finally block runs and connections are closed
|
||||
self.daemon = False
|
||||
@@ -1052,3 +1055,200 @@ class TestCommandWatchEdgeCases:
|
||||
thread.stop_and_wait(timeout=5.0)
|
||||
# Clean up any Tags created by the thread
|
||||
Tag.objects.all().delete()
|
||||
|
||||
|
||||
class TestRescanExistingFiles:
|
||||
"""
|
||||
Unit tests for the rescan safety net.
|
||||
|
||||
Each ``watch()`` recreation silently adopts the current directory contents
|
||||
as its baseline, so a file appearing between one batch and the next
|
||||
watcher's baseline is never reported and would sit in the consume directory
|
||||
forever. ``_rescan_existing_files`` re-injects such files into the
|
||||
stability tracker as a periodic safety net (see GH issue #13011).
|
||||
"""
|
||||
|
||||
@pytest.fixture
|
||||
def pdf_only_filter(self) -> ConsumerFilter:
|
||||
return ConsumerFilter(
|
||||
supported_extensions=frozenset({".pdf"}),
|
||||
ignore_patterns=[],
|
||||
)
|
||||
|
||||
def _rescan(
|
||||
self,
|
||||
directory: Path,
|
||||
consumer_filter: ConsumerFilter,
|
||||
tracker: FileStabilityTracker,
|
||||
queued: set[Path],
|
||||
*,
|
||||
recursive: bool = False,
|
||||
) -> None:
|
||||
Command()._rescan_existing_files(
|
||||
directory=directory,
|
||||
recursive=recursive,
|
||||
consumer_filter=consumer_filter,
|
||||
tracker=tracker,
|
||||
queued=queued,
|
||||
)
|
||||
|
||||
def test_tracks_stranded_file(
|
||||
self,
|
||||
consumption_dir: Path,
|
||||
sample_pdf: Path,
|
||||
pdf_only_filter: ConsumerFilter,
|
||||
) -> None:
|
||||
"""A supported on-disk file the watcher never reported gets tracked."""
|
||||
target = consumption_dir / "stranded.pdf"
|
||||
shutil.copy(sample_pdf, target)
|
||||
tracker = FileStabilityTracker(stability_delay=0.1)
|
||||
|
||||
self._rescan(consumption_dir, pdf_only_filter, tracker, set())
|
||||
|
||||
assert tracker.is_tracking(target) is True
|
||||
assert tracker.pending_count == 1
|
||||
|
||||
def test_skips_already_tracked_file(
|
||||
self,
|
||||
consumption_dir: Path,
|
||||
sample_pdf: Path,
|
||||
pdf_only_filter: ConsumerFilter,
|
||||
) -> None:
|
||||
"""A file already being tracked by the watcher is not double-tracked."""
|
||||
target = consumption_dir / "tracked.pdf"
|
||||
shutil.copy(sample_pdf, target)
|
||||
tracker = FileStabilityTracker(stability_delay=0.1)
|
||||
tracker.track(target, Change.added)
|
||||
|
||||
self._rescan(consumption_dir, pdf_only_filter, tracker, set())
|
||||
|
||||
assert tracker.pending_count == 1
|
||||
|
||||
def test_skips_queued_file(
|
||||
self,
|
||||
consumption_dir: Path,
|
||||
sample_pdf: Path,
|
||||
pdf_only_filter: ConsumerFilter,
|
||||
) -> None:
|
||||
"""A file already queued and awaiting consumption is not re-tracked."""
|
||||
target = consumption_dir / "inflight.pdf"
|
||||
shutil.copy(sample_pdf, target)
|
||||
tracker = FileStabilityTracker(stability_delay=0.1)
|
||||
queued = {target.resolve()}
|
||||
|
||||
self._rescan(consumption_dir, pdf_only_filter, tracker, queued)
|
||||
|
||||
assert tracker.pending_count == 0
|
||||
|
||||
def test_prunes_vanished_queued_paths(
|
||||
self,
|
||||
consumption_dir: Path,
|
||||
pdf_only_filter: ConsumerFilter,
|
||||
) -> None:
|
||||
"""Queued paths no longer on disk are dropped so the name can recur."""
|
||||
gone = (consumption_dir / "gone.pdf").resolve()
|
||||
tracker = FileStabilityTracker(stability_delay=0.1)
|
||||
queued = {gone}
|
||||
|
||||
self._rescan(consumption_dir, pdf_only_filter, tracker, queued)
|
||||
|
||||
assert gone not in queued
|
||||
|
||||
def test_skips_unsupported_extension(
|
||||
self,
|
||||
consumption_dir: Path,
|
||||
pdf_only_filter: ConsumerFilter,
|
||||
) -> None:
|
||||
"""Files filtered out by the consumer filter are not tracked."""
|
||||
(consumption_dir / "notes.xyz").write_bytes(b"content")
|
||||
tracker = FileStabilityTracker(stability_delay=0.1)
|
||||
|
||||
self._rescan(consumption_dir, pdf_only_filter, tracker, set())
|
||||
|
||||
assert tracker.pending_count == 0
|
||||
|
||||
def test_recursive_respects_flag(
|
||||
self,
|
||||
consumption_dir: Path,
|
||||
sample_pdf: Path,
|
||||
pdf_only_filter: ConsumerFilter,
|
||||
) -> None:
|
||||
"""Nested files are only found when recursive scanning is enabled."""
|
||||
subdir = consumption_dir / "nested"
|
||||
subdir.mkdir()
|
||||
target = subdir / "deep.pdf"
|
||||
shutil.copy(sample_pdf, target)
|
||||
|
||||
shallow = FileStabilityTracker(stability_delay=0.1)
|
||||
self._rescan(consumption_dir, pdf_only_filter, shallow, set())
|
||||
assert shallow.pending_count == 0
|
||||
|
||||
deep = FileStabilityTracker(stability_delay=0.1)
|
||||
self._rescan(consumption_dir, pdf_only_filter, deep, set(), recursive=True)
|
||||
assert deep.is_tracking(target) is True
|
||||
|
||||
|
||||
class TestProcessExistingFilesQueued:
|
||||
"""Tests that startup processing reports which paths it queued."""
|
||||
|
||||
@pytest.mark.usefixtures("mock_supported_extensions")
|
||||
def test_returns_queued_paths(
|
||||
self,
|
||||
consumption_dir: Path,
|
||||
sample_pdf: Path,
|
||||
mock_consume_file_delay: MagicMock,
|
||||
settings: SettingsWrapper,
|
||||
) -> None:
|
||||
"""The set returned seeds the rescan's queued set, avoiding re-queue."""
|
||||
target = consumption_dir / "document.pdf"
|
||||
shutil.copy(sample_pdf, target)
|
||||
settings.CONSUMER_IGNORE_PATTERNS = []
|
||||
|
||||
queued = Command()._process_existing_files(
|
||||
directory=consumption_dir,
|
||||
recursive=False,
|
||||
subdirs_as_tags=False,
|
||||
consumer_filter=ConsumerFilter(ignore_patterns=[]),
|
||||
)
|
||||
|
||||
assert target.resolve() in queued
|
||||
|
||||
|
||||
@pytest.mark.management
|
||||
@pytest.mark.django_db
|
||||
class TestCommandRescanRecovery:
|
||||
"""End-to-end test that the rescan recovers files the watcher misses."""
|
||||
|
||||
def test_rescan_consumes_file_the_watcher_never_reports(
|
||||
self,
|
||||
consumption_dir: Path,
|
||||
sample_pdf: Path,
|
||||
mock_consume_file_delay: MagicMock,
|
||||
start_consumer: Callable[..., ConsumerThread],
|
||||
) -> None:
|
||||
"""
|
||||
Isolate the rescan path: a long polling interval guarantees the
|
||||
watcher cannot report the file within the test window, so only the
|
||||
periodic rescan can consume it.
|
||||
"""
|
||||
# poll interval far longer than the test window -> watcher stays silent
|
||||
thread = start_consumer(
|
||||
polling_interval=30.0,
|
||||
stability_delay=0.1,
|
||||
rescan_interval=0.5,
|
||||
)
|
||||
|
||||
# created after startup, so _process_existing_files did not see it
|
||||
target = consumption_dir / "stranded.pdf"
|
||||
shutil.copy(sample_pdf, target)
|
||||
|
||||
wait_for_mock_call(mock_consume_file_delay.apply_async, timeout_s=5.0)
|
||||
|
||||
if thread.exception:
|
||||
raise thread.exception
|
||||
|
||||
mock_consume_file_delay.apply_async.assert_called()
|
||||
call_args = mock_consume_file_delay.apply_async.call_args.kwargs["kwargs"][
|
||||
"input_doc"
|
||||
]
|
||||
assert call_args.original_file.name == "stranded.pdf"
|
||||
|
||||
@@ -12,6 +12,7 @@ from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
|
||||
from documents.sanity_checker import SanityCheckMessages
|
||||
from documents.sanity_checker import check_sanity
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -21,6 +22,26 @@ if TYPE_CHECKING:
|
||||
from documents.tests.conftest import PaperlessDirs
|
||||
|
||||
|
||||
class TestSanityCheckMessages:
|
||||
def test_document_counts_are_unique_per_severity(self) -> None:
|
||||
messages = SanityCheckMessages()
|
||||
|
||||
messages.error(1, "first error")
|
||||
messages.error(1, "second error")
|
||||
messages.warning(1, "first warning")
|
||||
messages.warning(1, "second warning")
|
||||
messages.info(1, "first info")
|
||||
messages.info(1, "second info")
|
||||
messages.warning(None, "global warning")
|
||||
|
||||
assert messages.document_count == 1
|
||||
assert messages.document_error_count == 1
|
||||
assert messages.document_warning_count == 1
|
||||
assert messages.document_info_count == 1
|
||||
assert messages.global_warning_count == 1
|
||||
assert messages.total_issue_count == 5
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestCheckSanityNoDocuments:
|
||||
"""Sanity checks against an empty archive."""
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
from datetime import timedelta
|
||||
from types import SimpleNamespace
|
||||
from unittest import mock
|
||||
|
||||
from django.test import TestCase
|
||||
from django.utils import timezone
|
||||
|
||||
from documents.conditionals import metadata_etag
|
||||
from documents.conditionals import preview_etag
|
||||
@@ -29,10 +31,31 @@ class TestConditionals(DirectoriesMixin, TestCase):
|
||||
)
|
||||
request = SimpleNamespace(query_params={})
|
||||
|
||||
self.assertEqual(metadata_etag(request, root.id), latest.checksum)
|
||||
self.assertEqual(
|
||||
metadata_etag(request, root.id),
|
||||
f"{latest.checksum}:{latest.modified.isoformat()}",
|
||||
)
|
||||
self.assertEqual(preview_etag(request, root.id), latest.archive_checksum)
|
||||
self.assertEqual(thumbnail_etag(request, root.id), latest.checksum)
|
||||
|
||||
def test_metadata_etag_changes_when_document_modified_changes(self) -> None:
|
||||
doc = Document.objects.create(
|
||||
title="doc",
|
||||
checksum="same-checksum",
|
||||
mime_type="application/pdf",
|
||||
)
|
||||
request = SimpleNamespace(query_params={})
|
||||
|
||||
original_etag = metadata_etag(request, doc.id)
|
||||
new_modified = timezone.now() + timedelta(seconds=5)
|
||||
Document.objects.filter(id=doc.id).update(modified=new_modified)
|
||||
|
||||
self.assertNotEqual(metadata_etag(request, doc.id), original_etag)
|
||||
self.assertEqual(
|
||||
metadata_etag(request, doc.id),
|
||||
f"{doc.checksum}:{new_modified.isoformat()}",
|
||||
)
|
||||
|
||||
def test_resolve_effective_doc_returns_none_for_invalid_or_unrelated_version(
|
||||
self,
|
||||
) -> None:
|
||||
|
||||
@@ -30,6 +30,7 @@ from documents.signals.handlers import update_llm_suggestions_cache
|
||||
from documents.tests.utils import DirectoriesMixin
|
||||
from documents.tests.utils import read_streaming_response
|
||||
from paperless.models import ApplicationConfiguration
|
||||
from paperless_ai.exceptions import LLMTimeoutError
|
||||
|
||||
|
||||
class TestViews(DirectoriesMixin, TestCase):
|
||||
@@ -476,6 +477,33 @@ class TestAISuggestions(DirectoriesMixin, TestCase):
|
||||
get_llm_suggestion_cache(self.document.pk, backend="openai-like"),
|
||||
)
|
||||
|
||||
@patch("documents.views.get_ai_document_classification")
|
||||
@override_settings(
|
||||
AI_ENABLED=True,
|
||||
LLM_BACKEND="openai-like",
|
||||
)
|
||||
def test_ai_suggestions_with_llm_timeout(
|
||||
self,
|
||||
mock_get_ai_classification,
|
||||
) -> None:
|
||||
mock_get_ai_classification.side_effect = LLMTimeoutError()
|
||||
|
||||
self.client.force_login(user=self.user)
|
||||
response = self.client.get(
|
||||
f"/api/documents/{self.document.pk}/ai_suggestions/",
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_503_SERVICE_UNAVAILABLE)
|
||||
self.assertEqual(
|
||||
response.json(),
|
||||
{
|
||||
"ai": ["AI backend request timed out."],
|
||||
},
|
||||
)
|
||||
self.assertIsNone(
|
||||
get_llm_suggestion_cache(self.document.pk, backend="openai-like"),
|
||||
)
|
||||
|
||||
def test_invalidate_suggestions_cache(self) -> None:
|
||||
self.client.force_login(user=self.user)
|
||||
suggestions = {
|
||||
|
||||
+43
-5
@@ -12,6 +12,7 @@ from datetime import timedelta
|
||||
from http import HTTPStatus
|
||||
from pathlib import Path
|
||||
from time import mktime
|
||||
from time import sleep
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import Any
|
||||
from typing import Literal
|
||||
@@ -240,6 +241,7 @@ from paperless.serialisers import UserSerializer
|
||||
from paperless.views import StandardPagination
|
||||
from paperless_ai.ai_classifier import get_ai_document_classification
|
||||
from paperless_ai.chat import stream_chat_with_documents
|
||||
from paperless_ai.exceptions import LLMTimeoutError
|
||||
from paperless_ai.matching import extract_unmatched_names
|
||||
from paperless_ai.matching import match_correspondents_by_name
|
||||
from paperless_ai.matching import match_document_types_by_name
|
||||
@@ -1509,6 +1511,17 @@ class DocumentViewSet(
|
||||
exc_info=True,
|
||||
)
|
||||
raise ValidationError({"ai": [_("Invalid AI configuration.")]}) from exc
|
||||
except LLMTimeoutError as exc:
|
||||
logger.exception(
|
||||
"AI backend timed out while generating suggestions for document %s: %s",
|
||||
doc.pk,
|
||||
exc,
|
||||
exc_info=True,
|
||||
)
|
||||
return Response(
|
||||
{"ai": [_("AI backend request timed out.")]},
|
||||
status=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
)
|
||||
|
||||
matched_tags = match_tags_by_name(
|
||||
llm_suggestions.get("tags", []),
|
||||
@@ -2276,6 +2289,7 @@ class UnifiedSearchViewSet(DocumentViewSet):
|
||||
return super().list(request)
|
||||
|
||||
from documents.search import SearchHit
|
||||
from documents.search import SearchQueryError
|
||||
from documents.search import TantivyBackend
|
||||
from documents.search import TantivyRelevanceList
|
||||
from documents.search import get_backend
|
||||
@@ -2468,6 +2482,11 @@ class UnifiedSearchViewSet(DocumentViewSet):
|
||||
return HttpResponseForbidden(_("Insufficient permissions."))
|
||||
except ValidationError:
|
||||
raise
|
||||
except SearchQueryError as e:
|
||||
# User-fixable query error (e.g. an unparsable date): surface the
|
||||
# specific message so the user can correct it, rather than a generic
|
||||
# 400 or silently empty results.
|
||||
raise ValidationError({"query": [str(e)]}) from e
|
||||
except Exception as e:
|
||||
logger.warning(f"An error occurred listing search results: {e!s}")
|
||||
return HttpResponseBadRequest(
|
||||
@@ -3126,6 +3145,7 @@ class PostDocumentView(GenericAPIView[Any]):
|
||||
serializer.is_valid(raise_exception=True)
|
||||
|
||||
doc_name, doc_data = serializer.validated_data.get("document")
|
||||
doc_name = normalize("NFC", doc_name)
|
||||
correspondent_id = serializer.validated_data.get("correspondent")
|
||||
document_type_id = serializer.validated_data.get("document_type")
|
||||
storage_path_id = serializer.validated_data.get("storage_path")
|
||||
@@ -4989,11 +5009,29 @@ class SystemStatusView(PassUserMixin):
|
||||
celery_error = None
|
||||
celery_url = None
|
||||
try:
|
||||
celery_ping = celery_app.control.inspect().ping()
|
||||
celery_url = next(iter(celery_ping.keys()))
|
||||
first_worker_ping = celery_ping[celery_url]
|
||||
if first_worker_ping["ok"] == "pong":
|
||||
celery_active = "OK"
|
||||
celery_ping = None
|
||||
for ping_attempt in range(3):
|
||||
celery_ping = celery_app.control.inspect().ping()
|
||||
if celery_ping:
|
||||
break
|
||||
if ping_attempt < 2:
|
||||
sleep(0.25)
|
||||
|
||||
if not celery_ping:
|
||||
celery_active = "WARNING"
|
||||
celery_error = (
|
||||
"No celery workers responded to ping. This may be temporary."
|
||||
)
|
||||
else:
|
||||
celery_url, first_worker_ping = next(iter(celery_ping.items()))
|
||||
if (
|
||||
isinstance(first_worker_ping, dict)
|
||||
and first_worker_ping.get("ok") == "pong"
|
||||
):
|
||||
celery_active = "OK"
|
||||
else:
|
||||
celery_active = "WARNING"
|
||||
celery_error = "Celery worker responded unexpectedly."
|
||||
except Exception as e:
|
||||
celery_active = "ERROR"
|
||||
logger.exception(
|
||||
|
||||
@@ -2,7 +2,7 @@ msgid ""
|
||||
msgstr ""
|
||||
"Project-Id-Version: paperless-ngx\n"
|
||||
"Report-Msgid-Bugs-To: \n"
|
||||
"POT-Creation-Date: 2026-06-02 15:33+0000\n"
|
||||
"POT-Creation-Date: 2026-06-23 14:33+0000\n"
|
||||
"PO-Revision-Date: 2022-02-17 04:17\n"
|
||||
"Last-Translator: \n"
|
||||
"Language-Team: English\n"
|
||||
@@ -21,39 +21,39 @@ msgstr ""
|
||||
msgid "Documents"
|
||||
msgstr ""
|
||||
|
||||
#: documents/filters.py:463
|
||||
#: documents/filters.py:464
|
||||
msgid "Value must be valid JSON."
|
||||
msgstr ""
|
||||
|
||||
#: documents/filters.py:482
|
||||
#: documents/filters.py:483
|
||||
msgid "Invalid custom field query expression"
|
||||
msgstr ""
|
||||
|
||||
#: documents/filters.py:492
|
||||
#: documents/filters.py:493
|
||||
msgid "Invalid expression list. Must be nonempty."
|
||||
msgstr ""
|
||||
|
||||
#: documents/filters.py:513
|
||||
#: documents/filters.py:514
|
||||
msgid "Invalid logical operator {op!r}"
|
||||
msgstr ""
|
||||
|
||||
#: documents/filters.py:527
|
||||
#: documents/filters.py:528
|
||||
msgid "Maximum number of query conditions exceeded."
|
||||
msgstr ""
|
||||
|
||||
#: documents/filters.py:591
|
||||
#: documents/filters.py:592
|
||||
msgid "{name!r} is not a valid custom field."
|
||||
msgstr ""
|
||||
|
||||
#: documents/filters.py:628
|
||||
#: documents/filters.py:629
|
||||
msgid "{data_type} does not support query expr {expr!r}."
|
||||
msgstr ""
|
||||
|
||||
#: documents/filters.py:743 documents/models.py:136
|
||||
#: documents/filters.py:744 documents/models.py:136
|
||||
msgid "Maximum nesting depth exceeded."
|
||||
msgstr ""
|
||||
|
||||
#: documents/filters.py:990
|
||||
#: documents/filters.py:1052
|
||||
msgid "Custom field not found"
|
||||
msgstr ""
|
||||
|
||||
@@ -1351,49 +1351,49 @@ msgstr ""
|
||||
msgid "workflow runs"
|
||||
msgstr ""
|
||||
|
||||
#: documents/serialisers.py:463 documents/serialisers.py:815
|
||||
#: documents/serialisers.py:2681 documents/views.py:295 documents/views.py:2464
|
||||
#: documents/serialisers.py:503 documents/serialisers.py:855
|
||||
#: documents/serialisers.py:2744 documents/views.py:297 documents/views.py:2482
|
||||
#: paperless_mail/serialisers.py:155
|
||||
msgid "Insufficient permissions."
|
||||
msgstr ""
|
||||
|
||||
#: documents/serialisers.py:651
|
||||
#: documents/serialisers.py:691
|
||||
msgid "Invalid color."
|
||||
msgstr ""
|
||||
|
||||
#: documents/serialisers.py:2175
|
||||
#: documents/serialisers.py:2216
|
||||
#, python-format
|
||||
msgid "File type %(type)s not supported"
|
||||
msgstr ""
|
||||
|
||||
#: documents/serialisers.py:2219
|
||||
#: documents/serialisers.py:2260
|
||||
#, python-format
|
||||
msgid "Custom field id must be an integer: %(id)s"
|
||||
msgstr ""
|
||||
|
||||
#: documents/serialisers.py:2226
|
||||
#: documents/serialisers.py:2267
|
||||
#, python-format
|
||||
msgid "Custom field with id %(id)s does not exist"
|
||||
msgstr ""
|
||||
|
||||
#: documents/serialisers.py:2243 documents/serialisers.py:2253
|
||||
#: documents/serialisers.py:2284 documents/serialisers.py:2294
|
||||
msgid ""
|
||||
"Custom fields must be a list of integers or an object mapping ids to values."
|
||||
msgstr ""
|
||||
|
||||
#: documents/serialisers.py:2248
|
||||
#: documents/serialisers.py:2289
|
||||
msgid "Some custom fields don't exist or were specified twice."
|
||||
msgstr ""
|
||||
|
||||
#: documents/serialisers.py:2395
|
||||
#: documents/serialisers.py:2436
|
||||
msgid "Invalid variable detected."
|
||||
msgstr ""
|
||||
|
||||
#: documents/serialisers.py:2737
|
||||
#: documents/serialisers.py:2800
|
||||
msgid "Duplicate document identifiers are not allowed."
|
||||
msgstr ""
|
||||
|
||||
#: documents/serialisers.py:2767 documents/views.py:4341
|
||||
#: documents/serialisers.py:2830 documents/views.py:4429
|
||||
#, python-format
|
||||
msgid "Documents not found: %(ids)s"
|
||||
msgstr ""
|
||||
@@ -1661,32 +1661,36 @@ msgstr ""
|
||||
msgid "Unable to parse URI {value}"
|
||||
msgstr ""
|
||||
|
||||
#: documents/views.py:288 documents/views.py:2461
|
||||
#: documents/views.py:290 documents/views.py:2479
|
||||
msgid "Invalid more_like_id"
|
||||
msgstr ""
|
||||
|
||||
#: documents/views.py:1507
|
||||
#: documents/views.py:1513
|
||||
msgid "Invalid AI configuration."
|
||||
msgstr ""
|
||||
|
||||
#: documents/views.py:2286 documents/views.py:2602
|
||||
#: documents/views.py:1522
|
||||
msgid "AI backend request timed out."
|
||||
msgstr ""
|
||||
|
||||
#: documents/views.py:2304 documents/views.py:2625
|
||||
msgid "Specify only one of text, title_search, query, or more_like_id."
|
||||
msgstr ""
|
||||
|
||||
#: documents/views.py:4353
|
||||
#: documents/views.py:4441
|
||||
#, python-format
|
||||
msgid "Insufficient permissions to share document %(id)s."
|
||||
msgstr ""
|
||||
|
||||
#: documents/views.py:4399
|
||||
#: documents/views.py:4487
|
||||
msgid "Bundle is already being processed."
|
||||
msgstr ""
|
||||
|
||||
#: documents/views.py:4459
|
||||
#: documents/views.py:4547
|
||||
msgid "The share link bundle is still being prepared. Please try again later."
|
||||
msgstr ""
|
||||
|
||||
#: documents/views.py:4469
|
||||
#: documents/views.py:4557
|
||||
msgid "The share link bundle is unavailable."
|
||||
msgstr ""
|
||||
|
||||
@@ -1931,154 +1935,162 @@ msgid "Sets the LLM endpoint, optional"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/models.py:363
|
||||
msgid "Sets the LLM output language"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/models.py:370
|
||||
msgid "Sets the LLM timeout in seconds"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/models.py:376
|
||||
msgid "paperless application settings"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:537
|
||||
#: paperless/settings/__init__.py:545
|
||||
msgid "English (US)"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:538
|
||||
#: paperless/settings/__init__.py:546
|
||||
msgid "Arabic"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:539
|
||||
#: paperless/settings/__init__.py:547
|
||||
msgid "Afrikaans"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:540
|
||||
#: paperless/settings/__init__.py:548
|
||||
msgid "Belarusian"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:541
|
||||
#: paperless/settings/__init__.py:549
|
||||
msgid "Bulgarian"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:542
|
||||
#: paperless/settings/__init__.py:550
|
||||
msgid "Catalan"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:543
|
||||
#: paperless/settings/__init__.py:551
|
||||
msgid "Czech"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:544
|
||||
#: paperless/settings/__init__.py:552
|
||||
msgid "Danish"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:545
|
||||
#: paperless/settings/__init__.py:553
|
||||
msgid "German"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:546
|
||||
#: paperless/settings/__init__.py:554
|
||||
msgid "Greek"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:547
|
||||
#: paperless/settings/__init__.py:555
|
||||
msgid "English (GB)"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:548
|
||||
#: paperless/settings/__init__.py:556
|
||||
msgid "Spanish"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:549
|
||||
#: paperless/settings/__init__.py:557
|
||||
msgid "Persian"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:550
|
||||
#: paperless/settings/__init__.py:558
|
||||
msgid "Finnish"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:551
|
||||
#: paperless/settings/__init__.py:559
|
||||
msgid "French"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:552
|
||||
#: paperless/settings/__init__.py:560
|
||||
msgid "Hungarian"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:553
|
||||
#: paperless/settings/__init__.py:561
|
||||
msgid "Indonesian"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:554
|
||||
#: paperless/settings/__init__.py:562
|
||||
msgid "Italian"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:555
|
||||
#: paperless/settings/__init__.py:563
|
||||
msgid "Japanese"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:556
|
||||
#: paperless/settings/__init__.py:564
|
||||
msgid "Korean"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:557
|
||||
#: paperless/settings/__init__.py:565
|
||||
msgid "Luxembourgish"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:558
|
||||
#: paperless/settings/__init__.py:566
|
||||
msgid "Norwegian"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:559
|
||||
#: paperless/settings/__init__.py:567
|
||||
msgid "Dutch"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:560
|
||||
#: paperless/settings/__init__.py:568
|
||||
msgid "Polish"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:561
|
||||
#: paperless/settings/__init__.py:569
|
||||
msgid "Portuguese (Brazil)"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:562
|
||||
#: paperless/settings/__init__.py:570
|
||||
msgid "Portuguese"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:563
|
||||
#: paperless/settings/__init__.py:571
|
||||
msgid "Romanian"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:564
|
||||
#: paperless/settings/__init__.py:572
|
||||
msgid "Russian"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:565
|
||||
#: paperless/settings/__init__.py:573
|
||||
msgid "Slovak"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:566
|
||||
#: paperless/settings/__init__.py:574
|
||||
msgid "Slovenian"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:567
|
||||
#: paperless/settings/__init__.py:575
|
||||
msgid "Serbian"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:568
|
||||
#: paperless/settings/__init__.py:576
|
||||
msgid "Swedish"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:569
|
||||
#: paperless/settings/__init__.py:577
|
||||
msgid "Turkish"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:570
|
||||
#: paperless/settings/__init__.py:578
|
||||
msgid "Ukrainian"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:571
|
||||
#: paperless/settings/__init__.py:579
|
||||
msgid "Vietnamese"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:572
|
||||
#: paperless/settings/__init__.py:580
|
||||
msgid "Chinese Simplified"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:573
|
||||
#: paperless/settings/__init__.py:581
|
||||
msgid "Chinese Traditional"
|
||||
msgstr ""
|
||||
|
||||
|
||||
@@ -197,6 +197,7 @@ class AIConfig(BaseConfig):
|
||||
llm_embedding_endpoint: str = dataclasses.field(init=False)
|
||||
llm_embedding_chunk_size: int = dataclasses.field(init=False)
|
||||
llm_context_size: int = dataclasses.field(init=False)
|
||||
llm_request_timeout: int = dataclasses.field(init=False)
|
||||
llm_backend: str = dataclasses.field(init=False)
|
||||
llm_model: str = dataclasses.field(init=False)
|
||||
llm_api_key: str = dataclasses.field(init=False)
|
||||
@@ -221,6 +222,9 @@ class AIConfig(BaseConfig):
|
||||
app_config.llm_embedding_chunk_size or settings.LLM_EMBEDDING_CHUNK_SIZE
|
||||
)
|
||||
self.llm_context_size = app_config.llm_context_size or settings.LLM_CONTEXT_SIZE
|
||||
self.llm_request_timeout = (
|
||||
app_config.llm_request_timeout or settings.LLM_REQUEST_TIMEOUT
|
||||
)
|
||||
self.llm_backend = app_config.llm_backend or settings.LLM_BACKEND
|
||||
self.llm_model = app_config.llm_model or settings.LLM_MODEL
|
||||
self.llm_api_key = app_config.llm_api_key or settings.LLM_API_KEY
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
# Generated by Django 5.2.14 on 2026-06-14 14:22
|
||||
|
||||
import django.core.validators
|
||||
from django.db import migrations
|
||||
from django.db import models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("paperless", "0012_applicationconfiguration_llm_output_language"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name="applicationconfiguration",
|
||||
name="llm_request_timeout",
|
||||
field=models.PositiveSmallIntegerField(
|
||||
null=True,
|
||||
validators=[django.core.validators.MinValueValidator(1)],
|
||||
verbose_name="Sets the LLM request timeout in seconds",
|
||||
),
|
||||
),
|
||||
]
|
||||
@@ -366,6 +366,12 @@ class ApplicationConfiguration(AbstractSingletonModel):
|
||||
max_length=32,
|
||||
)
|
||||
|
||||
llm_request_timeout = models.PositiveSmallIntegerField(
|
||||
verbose_name=_("Sets the LLM timeout in seconds"),
|
||||
null=True,
|
||||
validators=[MinValueValidator(1)],
|
||||
)
|
||||
|
||||
class Meta:
|
||||
verbose_name = _("paperless application settings")
|
||||
permissions = [
|
||||
|
||||
@@ -20,6 +20,7 @@ from PIL import Image
|
||||
from PIL import ImageDraw
|
||||
from PIL import ImageFont
|
||||
|
||||
from paperless.parsers.utils import read_file_handle_unicode_errors
|
||||
from paperless.version import __full_version_str__
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -183,7 +184,7 @@ class TextDocumentParser:
|
||||
documents.parsers.ParseError
|
||||
If the file cannot be read.
|
||||
"""
|
||||
self._text = self._read_text(document_path)
|
||||
self._text = read_file_handle_unicode_errors(document_path, log=logger)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Result accessors
|
||||
@@ -295,30 +296,3 @@ class TextDocumentParser:
|
||||
Always ``[]`` — plain text files carry no structured metadata.
|
||||
"""
|
||||
return []
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Private helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _read_text(self, filepath: Path) -> str:
|
||||
"""Read file content, replacing invalid UTF-8 bytes rather than failing.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
filepath:
|
||||
Path to the file to read.
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
File content as a string.
|
||||
"""
|
||||
try:
|
||||
return filepath.read_text(encoding="utf-8")
|
||||
except UnicodeDecodeError as exc:
|
||||
logger.warning(
|
||||
"Unicode error reading %s, replacing bad bytes: %s",
|
||||
filepath,
|
||||
exc,
|
||||
)
|
||||
return filepath.read_bytes().decode("utf-8", errors="replace")
|
||||
|
||||
@@ -8,6 +8,7 @@ share implementation.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import codecs
|
||||
import logging
|
||||
import re
|
||||
import tempfile
|
||||
@@ -114,7 +115,7 @@ def read_file_handle_unicode_errors(
|
||||
filepath: Path,
|
||||
log: logging.Logger | None = None,
|
||||
) -> str:
|
||||
"""Read a file as UTF-8 text, replacing invalid bytes rather than raising.
|
||||
"""Read a file as text, detecting encoding via BOM and stripping NUL bytes.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
@@ -127,15 +128,27 @@ def read_file_handle_unicode_errors(
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
File content as a string, with any invalid UTF-8 sequences replaced
|
||||
by the Unicode replacement character.
|
||||
File content as a string, with NUL bytes removed so the result is
|
||||
safe to store in PostgreSQL text fields.
|
||||
"""
|
||||
_log = log or logger
|
||||
raw = filepath.read_bytes()
|
||||
|
||||
if raw.startswith((codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE)):
|
||||
encoding = "utf-16"
|
||||
elif raw.startswith(codecs.BOM_UTF8):
|
||||
encoding = "utf-8-sig"
|
||||
else:
|
||||
encoding = "utf-8"
|
||||
|
||||
try:
|
||||
return filepath.read_text(encoding="utf-8")
|
||||
text = raw.decode(encoding)
|
||||
except UnicodeDecodeError as e:
|
||||
_log.warning("Unicode error during text reading, continuing: %s", e)
|
||||
return filepath.read_bytes().decode("utf-8", errors="replace")
|
||||
text = raw.decode("utf-8", errors="replace")
|
||||
|
||||
# PostgreSQL rejects NUL (0x00) bytes in text fields
|
||||
return text.replace("\x00", "")
|
||||
|
||||
|
||||
def get_page_count_for_pdf(
|
||||
|
||||
@@ -98,6 +98,13 @@ MODEL_FILE = get_path_from_env(
|
||||
)
|
||||
LLM_INDEX_DIR = DATA_DIR / "llm_index"
|
||||
LLM_INDEX_LOCK = LLM_INDEX_DIR / "index.lock"
|
||||
# Cross-process read/write lock guarding the LLM index compaction/migration
|
||||
# file swap. Readers hold it shared; the swap takes it exclusively so it never
|
||||
# runs while a reader connection is open. Must be a SQLite (.db) file.
|
||||
LLM_INDEX_RWLOCK = LLM_INDEX_DIR / "llmindex.rwlock.db"
|
||||
# Seconds the compaction swap waits for active readers to drain before skipping
|
||||
# this cycle (it is a maintenance operation; the next run retries).
|
||||
LLM_INDEX_COMPACTION_LOCK_TIMEOUT = 30
|
||||
|
||||
LOGGING_DIR = get_path_from_env("PAPERLESS_LOGGING_DIR", DATA_DIR / "log")
|
||||
|
||||
@@ -1199,6 +1206,9 @@ if LLM_EMBEDDING_CHUNK_SIZE < 1:
|
||||
LLM_CONTEXT_SIZE = get_int_from_env("PAPERLESS_AI_LLM_CONTEXT_SIZE", 8192)
|
||||
if LLM_CONTEXT_SIZE < 1:
|
||||
raise ImproperlyConfigured("PAPERLESS_AI_LLM_CONTEXT_SIZE must be >= 1")
|
||||
LLM_REQUEST_TIMEOUT = get_int_from_env("PAPERLESS_AI_LLM_REQUEST_TIMEOUT", 120)
|
||||
if LLM_REQUEST_TIMEOUT < 1:
|
||||
raise ImproperlyConfigured("PAPERLESS_AI_LLM_REQUEST_TIMEOUT must be >= 1")
|
||||
LLM_BACKEND = get_choice_from_env(
|
||||
"PAPERLESS_AI_LLM_BACKEND",
|
||||
{"ollama", "openai-like"},
|
||||
|
||||
@@ -2,13 +2,50 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import codecs
|
||||
from pathlib import Path
|
||||
|
||||
from paperless.parsers.utils import is_tagged_pdf
|
||||
from paperless.parsers.utils import read_file_handle_unicode_errors
|
||||
|
||||
SAMPLES = Path(__file__).parent / "samples" / "tesseract"
|
||||
|
||||
|
||||
class TestReadFileHandleUnicodeErrors:
|
||||
def test_plain_utf8(self, tmp_path: Path) -> None:
|
||||
f = tmp_path / "plain.txt"
|
||||
f.write_bytes(b"hello world")
|
||||
assert read_file_handle_unicode_errors(f) == "hello world"
|
||||
|
||||
def test_utf8_bom(self, tmp_path: Path) -> None:
|
||||
f = tmp_path / "bom.txt"
|
||||
f.write_bytes(codecs.BOM_UTF8 + b"hello")
|
||||
assert read_file_handle_unicode_errors(f) == "hello"
|
||||
|
||||
def test_utf16_le(self, tmp_path: Path) -> None:
|
||||
f = tmp_path / "utf16le.txt"
|
||||
f.write_bytes(codecs.BOM_UTF16_LE + "hello".encode("utf-16-le"))
|
||||
assert read_file_handle_unicode_errors(f) == "hello"
|
||||
|
||||
def test_utf16_be(self, tmp_path: Path) -> None:
|
||||
f = tmp_path / "utf16be.txt"
|
||||
f.write_bytes(codecs.BOM_UTF16_BE + "hello".encode("utf-16-be"))
|
||||
assert read_file_handle_unicode_errors(f) == "hello"
|
||||
|
||||
def test_nul_bytes_stripped(self, tmp_path: Path) -> None:
|
||||
f = tmp_path / "null-bytes.txt"
|
||||
f.write_bytes(b"foo\x00bar")
|
||||
assert read_file_handle_unicode_errors(f) == "foobar"
|
||||
|
||||
def test_invalid_utf8_replaced(self, tmp_path: Path) -> None:
|
||||
f = tmp_path / "bad.txt"
|
||||
f.write_bytes(b"ok\x80\x81bad")
|
||||
result = read_file_handle_unicode_errors(f)
|
||||
assert "ok" in result
|
||||
assert "bad" in result
|
||||
assert "\x00" not in result
|
||||
|
||||
|
||||
class TestIsTaggedPdf:
|
||||
def test_tagged_pdf_returns_true(self) -> None:
|
||||
assert is_tagged_pdf(SAMPLES / "simple-digital.pdf") is True
|
||||
|
||||
@@ -87,7 +87,7 @@ def build_localization_prompt(suggestions: dict, output_language: str) -> str:
|
||||
Return the same JSON schema with all fields present.
|
||||
|
||||
Suggestions:
|
||||
{json.dumps(suggestions)}
|
||||
{json.dumps(suggestions, ensure_ascii=False)}
|
||||
""".strip()
|
||||
|
||||
|
||||
|
||||
+49
-42
@@ -9,6 +9,7 @@ from paperless_ai.db import db_connection_released
|
||||
from paperless_ai.indexing import _document_id_filters
|
||||
from paperless_ai.indexing import get_rag_prompt_helper
|
||||
from paperless_ai.indexing import load_or_build_index
|
||||
from paperless_ai.indexing import read_store
|
||||
|
||||
logger = logging.getLogger("paperless_ai.chat")
|
||||
|
||||
@@ -97,53 +98,59 @@ def _stream_chat_with_documents(query_str: str, documents: list[Document]):
|
||||
from llama_index.core.retrievers import VectorIndexRetriever
|
||||
|
||||
config = AIConfig()
|
||||
index = load_or_build_index(config)
|
||||
filters = _document_id_filters(str(doc.pk) for doc in documents)
|
||||
|
||||
retriever = VectorIndexRetriever(
|
||||
index=index,
|
||||
similarity_top_k=CHAT_RETRIEVER_TOP_K,
|
||||
filters=filters,
|
||||
)
|
||||
# Hold the shared read lock for the whole operation: the query engine
|
||||
# retrieves from the vector store again during synthesis, so the connection
|
||||
# must stay open (and the swap must not run) until the stream finishes.
|
||||
with read_store() as store:
|
||||
index = load_or_build_index(config, store)
|
||||
retriever = VectorIndexRetriever(
|
||||
index=index,
|
||||
similarity_top_k=CHAT_RETRIEVER_TOP_K,
|
||||
filters=filters,
|
||||
)
|
||||
|
||||
# Slow query-embedding + vector search; no Django ORM access happens during
|
||||
# it, so release the pooled DB connection for its duration. See #12976.
|
||||
with db_connection_released():
|
||||
top_nodes = retriever.retrieve(query_str)
|
||||
if not top_nodes:
|
||||
logger.warning("No nodes found for the given documents.")
|
||||
yield CHAT_NO_CONTENT_MESSAGE
|
||||
return
|
||||
# Slow query-embedding + vector search; no Django ORM access happens
|
||||
# during it, so release the pooled DB connection for its duration. See
|
||||
# #12976.
|
||||
with db_connection_released():
|
||||
top_nodes = retriever.retrieve(query_str)
|
||||
if not top_nodes:
|
||||
logger.warning("No nodes found for the given documents.")
|
||||
yield CHAT_NO_CONTENT_MESSAGE
|
||||
return
|
||||
|
||||
client = AIClient()
|
||||
client = AIClient()
|
||||
|
||||
references = _get_document_references(documents, top_nodes)
|
||||
references = _get_document_references(documents, top_nodes)
|
||||
|
||||
prompt_template = PromptTemplate(template=CHAT_PROMPT_TMPL)
|
||||
response_synthesizer = get_response_synthesizer(
|
||||
llm=client.llm,
|
||||
prompt_helper=get_rag_prompt_helper(
|
||||
chunk_size=config.llm_embedding_chunk_size,
|
||||
context_size=config.llm_context_size,
|
||||
),
|
||||
text_qa_template=prompt_template,
|
||||
streaming=True,
|
||||
)
|
||||
query_engine = RetrieverQueryEngine.from_args(
|
||||
retriever=retriever,
|
||||
llm=client.llm,
|
||||
response_synthesizer=response_synthesizer,
|
||||
streaming=True,
|
||||
)
|
||||
prompt_template = PromptTemplate(template=CHAT_PROMPT_TMPL)
|
||||
response_synthesizer = get_response_synthesizer(
|
||||
llm=client.llm,
|
||||
prompt_helper=get_rag_prompt_helper(
|
||||
chunk_size=config.llm_embedding_chunk_size,
|
||||
context_size=config.llm_context_size,
|
||||
),
|
||||
text_qa_template=prompt_template,
|
||||
streaming=True,
|
||||
)
|
||||
query_engine = RetrieverQueryEngine.from_args(
|
||||
retriever=retriever,
|
||||
llm=client.llm,
|
||||
response_synthesizer=response_synthesizer,
|
||||
streaming=True,
|
||||
)
|
||||
|
||||
logger.debug("Document chat query: %s", query_str)
|
||||
# Release the pooled DB connection for the slow streaming LLM response so it
|
||||
# is not pinned for the whole stream; see paperless_ai.db and #12976.
|
||||
with db_connection_released():
|
||||
response_stream = query_engine.query(query_str)
|
||||
for chunk in response_stream.response_gen:
|
||||
yield chunk
|
||||
sys.stdout.flush()
|
||||
logger.debug("Document chat query: %s", query_str)
|
||||
# Release the pooled DB connection for the slow streaming LLM response
|
||||
# so it is not pinned for the whole stream; see paperless_ai.db and
|
||||
# #12976.
|
||||
with db_connection_released():
|
||||
response_stream = query_engine.query(query_str)
|
||||
for chunk in response_stream.response_gen:
|
||||
yield chunk
|
||||
sys.stdout.flush()
|
||||
|
||||
if references:
|
||||
yield _format_chat_metadata_trailer(references)
|
||||
if references:
|
||||
yield _format_chat_metadata_trailer(references)
|
||||
|
||||
+49
-28
@@ -1,11 +1,14 @@
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import httpx
|
||||
|
||||
from paperless.models import LLMBackend
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from llama_index.core.llms import ChatMessage
|
||||
from llama_index.llms.ollama import Ollama
|
||||
from llama_index.llms.openai_like import OpenAILike
|
||||
|
||||
@@ -16,6 +19,7 @@ from paperless.network import create_pinned_async_httpx_client
|
||||
from paperless.network import create_pinned_httpx_client
|
||||
from paperless.network import validate_outbound_http_url
|
||||
from paperless_ai.base_model import DocumentClassifierSchema
|
||||
from paperless_ai.exceptions import LLMTimeoutError
|
||||
|
||||
logger = logging.getLogger("paperless_ai.client")
|
||||
|
||||
@@ -61,16 +65,16 @@ class AIClient:
|
||||
model=self.settings.llm_model or "llama3.1",
|
||||
base_url=endpoint,
|
||||
context_window=self.settings.llm_context_size,
|
||||
request_timeout=120,
|
||||
request_timeout=self.settings.llm_request_timeout,
|
||||
system_prompt=LLM_SYSTEM_PROMPT,
|
||||
client=Client(
|
||||
host=endpoint,
|
||||
timeout=120,
|
||||
timeout=self.settings.llm_request_timeout,
|
||||
transport=transport,
|
||||
),
|
||||
async_client=AsyncClient(
|
||||
host=endpoint,
|
||||
timeout=120,
|
||||
timeout=self.settings.llm_request_timeout,
|
||||
transport=async_transport,
|
||||
),
|
||||
)
|
||||
@@ -84,15 +88,18 @@ class AIClient:
|
||||
http_client = create_pinned_httpx_client(
|
||||
endpoint,
|
||||
allow_internal=self.settings.llm_allow_internal_endpoints,
|
||||
timeout=self.settings.llm_request_timeout,
|
||||
)
|
||||
async_http_client = create_pinned_async_httpx_client(
|
||||
endpoint,
|
||||
allow_internal=self.settings.llm_allow_internal_endpoints,
|
||||
timeout=self.settings.llm_request_timeout,
|
||||
)
|
||||
return OpenAILike(
|
||||
model=self.settings.llm_model or "gpt-3.5-turbo",
|
||||
api_base=endpoint,
|
||||
api_key=self.settings.llm_api_key,
|
||||
timeout=self.settings.llm_request_timeout,
|
||||
is_chat_model=True,
|
||||
is_function_calling_model=True,
|
||||
system_prompt=LLM_SYSTEM_PROMPT,
|
||||
@@ -113,11 +120,12 @@ class AIClient:
|
||||
|
||||
user_msg = ChatMessage(role="user", content=prompt)
|
||||
if self.settings.llm_backend == LLMBackend.OLLAMA:
|
||||
result = self.llm.chat(
|
||||
[user_msg],
|
||||
format=DocumentClassifierSchema.model_json_schema(),
|
||||
think=False,
|
||||
)
|
||||
with self._normalize_timeouts():
|
||||
result = self.llm.chat(
|
||||
[user_msg],
|
||||
format=DocumentClassifierSchema.model_json_schema(),
|
||||
think=False,
|
||||
)
|
||||
logger.debug("LLM query result: %s", result)
|
||||
parsed = DocumentClassifierSchema(**json.loads(result.message.content))
|
||||
return parsed.model_dump()
|
||||
@@ -125,26 +133,39 @@ class AIClient:
|
||||
from llama_index.core.program.function_program import get_function_tool
|
||||
|
||||
tool = get_function_tool(DocumentClassifierSchema)
|
||||
result = self.llm.chat_with_tools(
|
||||
tools=[tool],
|
||||
user_msg=user_msg,
|
||||
chat_history=[],
|
||||
allow_parallel_tool_calls=True,
|
||||
)
|
||||
tool_calls = self.llm.get_tool_calls_from_response(
|
||||
result,
|
||||
error_on_no_tool_call=True,
|
||||
)
|
||||
with self._normalize_timeouts():
|
||||
result = self.llm.chat_with_tools(
|
||||
tools=[tool],
|
||||
user_msg=user_msg,
|
||||
chat_history=[],
|
||||
allow_parallel_tool_calls=True,
|
||||
tool_required=True,
|
||||
)
|
||||
tool_calls = self.llm.get_tool_calls_from_response(
|
||||
result,
|
||||
error_on_no_tool_call=True,
|
||||
)
|
||||
logger.debug("LLM query result: %s", tool_calls)
|
||||
parsed = DocumentClassifierSchema(**tool_calls[0].tool_kwargs)
|
||||
return parsed.model_dump()
|
||||
|
||||
def run_chat(self, messages: list["ChatMessage"]) -> str:
|
||||
logger.debug(
|
||||
"Running chat query against %s with model %s",
|
||||
self.settings.llm_backend,
|
||||
self.settings.llm_model,
|
||||
)
|
||||
result = self.llm.chat(messages)
|
||||
logger.debug("Chat result: %s", result)
|
||||
return result
|
||||
@contextmanager
|
||||
def _normalize_timeouts(self) -> Iterator[None]:
|
||||
try:
|
||||
yield
|
||||
except httpx.TimeoutException as exc:
|
||||
raise LLMTimeoutError from exc
|
||||
except Exception as exc:
|
||||
if self._is_openai_timeout(exc):
|
||||
raise LLMTimeoutError from exc
|
||||
raise
|
||||
|
||||
def _is_openai_timeout(self, exc: Exception) -> bool:
|
||||
if self.settings.llm_backend != LLMBackend.OPENAI_LIKE:
|
||||
return False
|
||||
|
||||
# Keep OpenAI imports out of module import paths and only load the SDK
|
||||
# when translating an error from an OpenAI-backed request.
|
||||
from openai import APITimeoutError
|
||||
|
||||
return isinstance(exc, APITimeoutError)
|
||||
|
||||
@@ -32,15 +32,18 @@ def get_embedding_model(config: AIConfig) -> "BaseEmbedding":
|
||||
http_client = create_pinned_httpx_client(
|
||||
endpoint,
|
||||
allow_internal=config.llm_allow_internal_endpoints,
|
||||
timeout=config.llm_request_timeout,
|
||||
)
|
||||
async_http_client = create_pinned_async_httpx_client(
|
||||
endpoint,
|
||||
allow_internal=config.llm_allow_internal_endpoints,
|
||||
timeout=config.llm_request_timeout,
|
||||
)
|
||||
return OpenAILikeEmbedding(
|
||||
model_name=config.llm_embedding_model or "text-embedding-3-small",
|
||||
api_key=config.llm_api_key,
|
||||
api_base=endpoint,
|
||||
timeout=config.llm_request_timeout,
|
||||
http_client=http_client,
|
||||
async_http_client=async_http_client,
|
||||
)
|
||||
@@ -73,12 +76,14 @@ def get_embedding_model(config: AIConfig) -> "BaseEmbedding":
|
||||
)
|
||||
embedding._client = Client(
|
||||
host=endpoint,
|
||||
timeout=config.llm_request_timeout,
|
||||
transport=PinnedHostHTTPTransport(
|
||||
allow_internal=config.llm_allow_internal_endpoints,
|
||||
),
|
||||
)
|
||||
embedding._async_client = AsyncClient(
|
||||
host=endpoint,
|
||||
timeout=config.llm_request_timeout,
|
||||
transport=PinnedHostAsyncHTTPTransport(
|
||||
allow_internal=config.llm_allow_internal_endpoints,
|
||||
),
|
||||
@@ -99,9 +104,13 @@ _DEFAULT_MODEL_NAMES = {
|
||||
|
||||
def get_configured_model_name(config: AIConfig) -> str:
|
||||
"""Return the canonical name of the currently configured embedding model."""
|
||||
default = _DEFAULT_MODEL_NAMES.get(
|
||||
config.llm_embedding_backend,
|
||||
"sentence-transformers/all-MiniLM-L6-v2",
|
||||
# dict.get(key, default) overload resolution fails for TextChoices keys in some
|
||||
# type checkers; use `or` fallback to avoid the ambiguity.
|
||||
default = (
|
||||
_DEFAULT_MODEL_NAMES.get(
|
||||
config.llm_embedding_backend,
|
||||
)
|
||||
or "sentence-transformers/all-MiniLM-L6-v2"
|
||||
)
|
||||
return config.llm_embedding_model or default
|
||||
|
||||
@@ -112,15 +121,11 @@ def _normalize_llm_index_text(text: str) -> str:
|
||||
|
||||
|
||||
def build_llm_index_text(doc: Document) -> str:
|
||||
# TODO: Filename, Storage Path, and Archive Serial Number are short structured
|
||||
# values that could move to node.metadata (excluded from embeddings, visible to
|
||||
# LLM via metadata prepend) — same pattern as title/tags/correspondent. Notes
|
||||
# and Custom Fields should stay here: Notes can be long free text, Custom Fields
|
||||
# are dynamic in count and best kept in the embedding.
|
||||
# Short structured fields (filename, storage path, ASN, title, tags, ...) live
|
||||
# in node.metadata: excluded from embeddings, shown to the LLM via metadata
|
||||
# prepend. Notes and Custom Fields stay in the body: Notes can be long free
|
||||
# text, Custom Fields are dynamic in count and best kept in the embedding.
|
||||
lines = [
|
||||
f"Filename: {doc.filename}",
|
||||
f"Storage Path: {doc.storage_path.name if doc.storage_path else ''}",
|
||||
f"Archive Serial Number: {doc.archive_serial_number or ''}",
|
||||
f"Notes: {','.join([str(c.note) for c in Note.objects.filter(document=doc)])}",
|
||||
]
|
||||
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
class LLMTimeoutError(Exception):
|
||||
pass
|
||||
+162
-47
@@ -7,6 +7,8 @@ from typing import TYPE_CHECKING
|
||||
from django.conf import settings
|
||||
from django.utils import timezone
|
||||
from filelock import FileLock
|
||||
from filelock import ReadWriteLock
|
||||
from filelock import Timeout
|
||||
|
||||
from documents.models import Document
|
||||
from documents.models import PaperlessTask
|
||||
@@ -21,13 +23,11 @@ from paperless_ai.embedding import get_embedding_model
|
||||
if TYPE_CHECKING:
|
||||
from llama_index.core.schema import BaseNode
|
||||
|
||||
from paperless_ai.vector_store import PaperlessLanceVectorStore
|
||||
from paperless_ai.vector_store import PaperlessSqliteVecVectorStore
|
||||
|
||||
|
||||
logger = logging.getLogger("paperless_ai.indexing")
|
||||
|
||||
LLM_INDEX_TABLE = "documents"
|
||||
|
||||
RAG_NUM_OUTPUT = 512
|
||||
RAG_CHUNK_OVERLAP = 200
|
||||
|
||||
@@ -63,36 +63,108 @@ def queue_llm_index_update_if_needed(*, rebuild: bool, reason: str) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
def get_vector_store() -> "PaperlessLanceVectorStore":
|
||||
from paperless_ai.vector_store import PaperlessLanceVectorStore
|
||||
def get_vector_store() -> "PaperlessSqliteVecVectorStore":
|
||||
from paperless_ai.vector_store import PaperlessSqliteVecVectorStore
|
||||
|
||||
settings.LLM_INDEX_DIR.mkdir(parents=True, exist_ok=True)
|
||||
return PaperlessLanceVectorStore(
|
||||
return PaperlessSqliteVecVectorStore(
|
||||
uri=str(settings.LLM_INDEX_DIR),
|
||||
table_name=LLM_INDEX_TABLE,
|
||||
)
|
||||
|
||||
|
||||
# --- LLM index locking ---------------------------------------------------
|
||||
#
|
||||
# Two locks guard the index; they answer different questions and are NOT
|
||||
# interchangeable:
|
||||
#
|
||||
# * settings.LLM_INDEX_LOCK (FileLock, exclusive) -- serializes WRITERS against
|
||||
# each other, so only one rebuild/upsert/delete/compaction runs at a time.
|
||||
# Taken by write_store(). Readers never take it, so it never blocks reads.
|
||||
#
|
||||
# * settings.LLM_INDEX_RWLOCK (ReadWriteLock) -- coordinates readers against the
|
||||
# compaction/migration file swap. read_store() takes it SHARED (readers run
|
||||
# concurrently); _exclude_readers() takes it EXCLUSIVE, only for the swap, so
|
||||
# the database file is never replaced while a reader connection is open (that
|
||||
# would alias the old WAL onto the new file and corrupt it).
|
||||
#
|
||||
# | vs another writer | vs a reader
|
||||
# -----------------+-------------------+----------------------------
|
||||
# normal write | LLM_INDEX_LOCK | nothing (WAL gives MVCC)
|
||||
# compaction/swap | LLM_INDEX_LOCK | LLM_INDEX_RWLOCK (exclusive)
|
||||
# reader | nothing (WAL) | LLM_INDEX_RWLOCK (shared)
|
||||
#
|
||||
# They can't be merged into one ReadWriteLock: a normal write must exclude other
|
||||
# writers WITHOUT blocking readers (WAL already gives reader/writer concurrency),
|
||||
# and ReadWriteLock has no "exclusive vs writers, shared vs readers" mode. Only
|
||||
# the swap needs to exclude readers.
|
||||
def _index_rwlock() -> ReadWriteLock:
|
||||
"""Return a fresh read/write lock instance for the index swap.
|
||||
|
||||
``is_singleton=False`` so reads and the swap always coordinate through
|
||||
SQLite (the actual cross-process case) rather than hitting the in-process
|
||||
reentrant-upgrade guard; callers must ``close()`` it (the context managers
|
||||
below do).
|
||||
"""
|
||||
settings.LLM_INDEX_DIR.mkdir(parents=True, exist_ok=True)
|
||||
return ReadWriteLock(str(settings.LLM_INDEX_RWLOCK), is_singleton=False)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def read_store():
|
||||
"""Acquire the shared read lock and yield the vector store for a read.
|
||||
|
||||
The shared lock is held for the whole lifetime of the connection (and
|
||||
closed on exit) so the compaction/migration swap, which takes the exclusive
|
||||
lock, never runs while this connection is open. Concurrent readers do not
|
||||
block each other; only the swap does.
|
||||
"""
|
||||
lock = _index_rwlock()
|
||||
try:
|
||||
with lock.read_lock(), get_vector_store() as store:
|
||||
yield store
|
||||
finally:
|
||||
lock.close()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _exclude_readers():
|
||||
"""Acquire exclusive index access, blocking until readers have drained.
|
||||
|
||||
The exclusive counterpart to ``read_store()``: a compaction or migration
|
||||
must not run while any reader connection is open. Raises
|
||||
:class:`filelock.Timeout` if active readers do not drain within
|
||||
``LLM_INDEX_COMPACTION_LOCK_TIMEOUT``; callers skip the operation on timeout.
|
||||
"""
|
||||
lock = _index_rwlock()
|
||||
try:
|
||||
with lock.write_lock(timeout=settings.LLM_INDEX_COMPACTION_LOCK_TIMEOUT):
|
||||
yield
|
||||
finally:
|
||||
lock.close()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def write_store(embed_model_name: str | None = None):
|
||||
"""Acquire the write lock and yield the vector store.
|
||||
|
||||
All mutating operations (upsert, delete, rebuild, compact) must go through
|
||||
this context manager to serialise concurrent Celery writers.
|
||||
Read paths use ``get_vector_store()`` directly — no lock needed.
|
||||
Read paths use ``read_store()`` so they hold the shared read lock.
|
||||
|
||||
Pass ``embed_model_name`` whenever the operation may create the table so
|
||||
the model name is recorded in the schema metadata for future mismatch checks.
|
||||
"""
|
||||
from paperless_ai.vector_store import PaperlessLanceVectorStore
|
||||
from paperless_ai.vector_store import PaperlessSqliteVecVectorStore
|
||||
|
||||
settings.LLM_INDEX_DIR.mkdir(parents=True, exist_ok=True)
|
||||
with FileLock(settings.LLM_INDEX_LOCK):
|
||||
yield PaperlessLanceVectorStore(
|
||||
with (
|
||||
FileLock(settings.LLM_INDEX_LOCK),
|
||||
PaperlessSqliteVecVectorStore(
|
||||
uri=str(settings.LLM_INDEX_DIR),
|
||||
table_name=LLM_INDEX_TABLE,
|
||||
embed_model_name=embed_model_name,
|
||||
)
|
||||
) as store,
|
||||
):
|
||||
yield store
|
||||
|
||||
|
||||
def build_document_node(
|
||||
@@ -114,6 +186,9 @@ def build_document_node(
|
||||
"document_type": document.document_type.name
|
||||
if document.document_type
|
||||
else None,
|
||||
"filename": document.filename,
|
||||
"storage_path": document.storage_path.name if document.storage_path else None,
|
||||
"archive_serial_number": document.archive_serial_number,
|
||||
"created": document.created.isoformat() if document.created else None,
|
||||
"added": document.added.isoformat() if document.added else None,
|
||||
"modified": document.modified.isoformat(),
|
||||
@@ -140,23 +215,27 @@ def build_document_node(
|
||||
return parser.get_nodes_from_documents([doc])
|
||||
|
||||
|
||||
def load_or_build_index(config: AIConfig):
|
||||
"""Return a VectorStoreIndex backed by the vector store."""
|
||||
def load_or_build_index(config: AIConfig, store: "PaperlessSqliteVecVectorStore"):
|
||||
"""Return a VectorStoreIndex backed by ``store``.
|
||||
|
||||
``store`` is supplied by the caller's ``read_store()`` context so the shared
|
||||
read lock and the connection stay alive for the whole retrieval.
|
||||
"""
|
||||
import llama_index.core.settings as llama_settings
|
||||
from llama_index.core import VectorStoreIndex
|
||||
|
||||
embed_model = get_embedding_model(config)
|
||||
llama_settings.Settings.embed_model = embed_model
|
||||
vector_store = get_vector_store()
|
||||
return VectorStoreIndex.from_vector_store(
|
||||
vector_store=vector_store,
|
||||
vector_store=store,
|
||||
embed_model=embed_model,
|
||||
)
|
||||
|
||||
|
||||
def llm_index_exists() -> bool:
|
||||
"""True when the index table exists on disk."""
|
||||
return get_vector_store().table_exists()
|
||||
with read_store() as store:
|
||||
return store.table_exists()
|
||||
|
||||
|
||||
def get_rag_chunk_size() -> int:
|
||||
@@ -224,6 +303,21 @@ def update_llm_index(
|
||||
rebuild=False,
|
||||
) -> str:
|
||||
"""Rebuild or incrementally update the LLM index."""
|
||||
with write_store() as store:
|
||||
try:
|
||||
with _exclude_readers():
|
||||
needs_reembed = store.check_and_run_migrations()
|
||||
except Timeout:
|
||||
logger.info(
|
||||
"Skipping LLM index migration check: index readers are active; "
|
||||
"will retry next run.",
|
||||
)
|
||||
needs_reembed = False
|
||||
if needs_reembed:
|
||||
logger.warning(
|
||||
"LLM index migration requires re-embedding; forcing rebuild.",
|
||||
)
|
||||
rebuild = True
|
||||
documents = Document.objects.all()
|
||||
no_documents = not documents.exists()
|
||||
|
||||
@@ -235,13 +329,12 @@ def update_llm_index(
|
||||
config = AIConfig()
|
||||
model_name = get_configured_model_name(config)
|
||||
|
||||
if (
|
||||
not rebuild
|
||||
and llm_index_exists()
|
||||
and get_vector_store().config_mismatch(model_name)
|
||||
):
|
||||
logger.warning("Embedding model changed; forcing LLM index rebuild.")
|
||||
rebuild = True
|
||||
if not rebuild and llm_index_exists():
|
||||
with read_store() as store:
|
||||
config_mismatch = store.config_mismatch(model_name)
|
||||
if config_mismatch:
|
||||
logger.warning("Embedding model changed; forcing LLM index rebuild.")
|
||||
rebuild = True
|
||||
|
||||
if no_documents:
|
||||
logger.warning("No documents found to index.")
|
||||
@@ -251,7 +344,6 @@ def update_llm_index(
|
||||
|
||||
with write_store(embed_model_name=model_name) as store:
|
||||
if rebuild or not store.table_exists():
|
||||
(settings.LLM_INDEX_DIR / "meta.json").unlink(missing_ok=True)
|
||||
logger.info("Rebuilding LLM index.")
|
||||
store.drop_table()
|
||||
for document in iter_wrapper(documents):
|
||||
@@ -276,9 +368,14 @@ def update_llm_index(
|
||||
else "No changes detected in LLM index."
|
||||
)
|
||||
|
||||
store.ensure_document_id_scalar_index()
|
||||
store.maybe_create_ann_index()
|
||||
store.compact(retention_seconds=60 * 60) # 1 hour: safe for in-flight readers
|
||||
try:
|
||||
with _exclude_readers():
|
||||
store.compact()
|
||||
except Timeout:
|
||||
logger.info(
|
||||
"Skipping LLM index compaction: index readers are active; "
|
||||
"will retry next run.",
|
||||
)
|
||||
return msg
|
||||
|
||||
|
||||
@@ -294,13 +391,19 @@ def llm_index_add_or_update_document(document: Document):
|
||||
|
||||
with write_store(embed_model_name=get_configured_model_name(config)) as store:
|
||||
store.upsert_document(str(document.id), new_nodes)
|
||||
store.ensure_document_id_scalar_index()
|
||||
|
||||
|
||||
def llm_index_compact() -> None:
|
||||
"""Compact the index immediately, clearing all MVCC version history."""
|
||||
"""Compact the index immediately, rebuilding the table to reclaim space."""
|
||||
with write_store() as store:
|
||||
store.compact(retention_seconds=0)
|
||||
try:
|
||||
with _exclude_readers():
|
||||
store.compact(force=True)
|
||||
except Timeout:
|
||||
logger.info(
|
||||
"Skipping LLM index compaction: index readers are active; "
|
||||
"will retry next run.",
|
||||
)
|
||||
|
||||
|
||||
def llm_index_remove_document(document: Document):
|
||||
@@ -340,6 +443,18 @@ def truncate_content(
|
||||
return " ".join(truncated_chunks)
|
||||
|
||||
|
||||
def truncate_embedding_query(content: str, *, chunk_size: int) -> str:
|
||||
from llama_index.core.text_splitter import TokenTextSplitter
|
||||
|
||||
splitter = TokenTextSplitter(
|
||||
separator=" ",
|
||||
chunk_size=chunk_size,
|
||||
chunk_overlap=0,
|
||||
)
|
||||
content_chunks = splitter.split_text(content)
|
||||
return content_chunks[0] if content_chunks else ""
|
||||
|
||||
|
||||
def normalize_document_ids(document_ids: Iterable[int | str] | None) -> set[str] | None:
|
||||
if document_ids is None:
|
||||
return None
|
||||
@@ -367,30 +482,30 @@ def query_similar_documents(
|
||||
|
||||
from llama_index.core.retrievers import VectorIndexRetriever
|
||||
|
||||
index = load_or_build_index(config)
|
||||
|
||||
filters = (
|
||||
_document_id_filters(allowed_document_ids)
|
||||
if allowed_document_ids is not None
|
||||
else None
|
||||
)
|
||||
|
||||
retriever = VectorIndexRetriever(
|
||||
index=index,
|
||||
similarity_top_k=top_k,
|
||||
filters=filters,
|
||||
)
|
||||
|
||||
query_text = truncate_content(
|
||||
query_text = truncate_embedding_query(
|
||||
(document.title or "") + "\n" + (document.content or ""),
|
||||
chunk_size=config.llm_embedding_chunk_size,
|
||||
context_size=config.llm_context_size,
|
||||
)
|
||||
# The retrieve() call generates a query embedding (a slow external request)
|
||||
# and searches the vector store; no Django ORM access happens during it, so
|
||||
# release the pooled DB connection for its duration. See #12976.
|
||||
with db_connection_released():
|
||||
results = retriever.retrieve(query_text)
|
||||
# Hold the shared read lock for the whole retrieval so the connection is
|
||||
# never open across a compaction swap. The retrieve() call generates a
|
||||
# query embedding (a slow external request) and searches the vector store;
|
||||
# no Django ORM access happens during it, so release the pooled DB
|
||||
# connection for its duration. See #12976.
|
||||
with read_store() as store:
|
||||
index = load_or_build_index(config, store)
|
||||
retriever = VectorIndexRetriever(
|
||||
index=index,
|
||||
similarity_top_k=top_k,
|
||||
filters=filters,
|
||||
)
|
||||
with db_connection_released():
|
||||
results = retriever.retrieve(query_text)
|
||||
|
||||
retrieved_document_ids: list[int] = []
|
||||
for node in results:
|
||||
|
||||
@@ -10,6 +10,7 @@ from pytest_django.fixtures import SettingsWrapper
|
||||
def temp_llm_index_dir(tmp_path: Path, settings: SettingsWrapper) -> Path:
|
||||
settings.LLM_INDEX_DIR = tmp_path
|
||||
settings.LLM_INDEX_LOCK = tmp_path / "index.lock"
|
||||
settings.LLM_INDEX_RWLOCK = tmp_path / "llmindex.rwlock.db"
|
||||
return tmp_path
|
||||
|
||||
|
||||
|
||||
@@ -239,6 +239,23 @@ def test_get_language_name_falls_back_to_language_code():
|
||||
assert get_language_name("zz-zz") == "zz-zz"
|
||||
|
||||
|
||||
def test_build_localization_prompt_preserves_unicode_characters():
|
||||
prompt = build_localization_prompt(
|
||||
{
|
||||
"title": "Gebührenbescheid",
|
||||
"tags": [],
|
||||
"correspondents": [],
|
||||
"document_types": [],
|
||||
"storage_paths": [],
|
||||
"dates": [],
|
||||
},
|
||||
output_language="de-de",
|
||||
)
|
||||
|
||||
assert "Gebührenbescheid" in prompt
|
||||
assert "\\u00fc" not in prompt
|
||||
|
||||
|
||||
@patch("paperless_ai.ai_classifier.query_similar_documents")
|
||||
def test_get_context_for_document(
|
||||
mock_query_similar_documents,
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
from unittest.mock import patch
|
||||
@@ -7,6 +6,7 @@ import pytest
|
||||
import pytest_mock
|
||||
from django.test import override_settings
|
||||
from django.utils import timezone
|
||||
from llama_index.core.schema import MetadataMode
|
||||
|
||||
from documents.models import Document
|
||||
from documents.models import PaperlessTask
|
||||
@@ -17,6 +17,7 @@ from documents.tests.factories import PaperlessTaskFactory
|
||||
from paperless.models import ApplicationConfiguration
|
||||
from paperless_ai import indexing
|
||||
from paperless_ai.tests.conftest import FakeEmbedding
|
||||
from paperless_ai.vector_store import PaperlessSqliteVecVectorStore
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -33,12 +34,22 @@ def test_build_document_node(real_document: Document) -> None:
|
||||
nodes = indexing.build_document_node(real_document)
|
||||
assert len(nodes) > 0
|
||||
assert nodes[0].metadata["document_id"] == str(real_document.id)
|
||||
assert nodes[0].metadata["filename"] == real_document.filename
|
||||
assert nodes[0].metadata["storage_path"] == (
|
||||
real_document.storage_path.name if real_document.storage_path else None
|
||||
)
|
||||
assert (
|
||||
nodes[0].metadata["archive_serial_number"]
|
||||
== real_document.archive_serial_number
|
||||
)
|
||||
assert "filename" in nodes[0].excluded_embed_metadata_keys
|
||||
assert "filename" not in nodes[0].excluded_llm_metadata_keys
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_build_document_node_sets_ref_doc_id(real_document: Document) -> None:
|
||||
"""Every node produced by build_document_node must carry the paperless document id
|
||||
as its ref_doc_id so that the LanceDB adapter's delete(str(doc.id)) works correctly."""
|
||||
as its ref_doc_id so that the vector store's delete(str(doc.id)) works correctly."""
|
||||
nodes = indexing.build_document_node(real_document)
|
||||
assert len(nodes) > 0, "Expected at least one node"
|
||||
for node in nodes:
|
||||
@@ -58,8 +69,6 @@ def test_build_document_node_excludes_metadata_from_embedding(
|
||||
double the token count and exceed embedding models with small context
|
||||
windows (e.g. nomic-embed-text via Ollama defaults to num_ctx=2048).
|
||||
"""
|
||||
from llama_index.core.schema import MetadataMode
|
||||
|
||||
nodes = indexing.build_document_node(real_document)
|
||||
for node in nodes:
|
||||
embed_text = node.get_content(metadata_mode=MetadataMode.EMBED)
|
||||
@@ -91,8 +100,6 @@ def test_build_document_node_excludes_document_id_from_llm_context(
|
||||
real_document: Document,
|
||||
) -> None:
|
||||
"""document_id is an internal key and must not appear in LLM context text."""
|
||||
from llama_index.core.schema import MetadataMode
|
||||
|
||||
nodes = indexing.build_document_node(real_document)
|
||||
assert len(nodes) > 0
|
||||
for node in nodes:
|
||||
@@ -130,6 +137,16 @@ def test_get_rag_prompt_helper_uses_context_setting() -> None:
|
||||
assert prompt_helper.context_window == 4096
|
||||
|
||||
|
||||
def test_truncate_embedding_query_returns_single_chunk() -> None:
|
||||
content = " ".join(f"word{i}" for i in range(200))
|
||||
|
||||
result = indexing.truncate_embedding_query(content, chunk_size=32)
|
||||
|
||||
assert result
|
||||
assert result != content
|
||||
assert "word199" not in result
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_update_llm_index(
|
||||
temp_llm_index_dir: Path,
|
||||
@@ -154,29 +171,6 @@ def test_update_llm_index(
|
||||
build_document_node.assert_called_once_with(real_document, chunk_size=512)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_update_llm_index_cleans_stale_meta_on_rebuild(
|
||||
temp_llm_index_dir: Path,
|
||||
real_document: Document,
|
||||
mock_embed_model: FakeEmbedding,
|
||||
) -> None:
|
||||
# A meta.json left over from the FAISS era (or written by older code) must be
|
||||
# deleted on rebuild so stale artifacts don't accumulate on disk.
|
||||
stale_meta = temp_llm_index_dir / "meta.json"
|
||||
stale_meta.write_text(json.dumps({"embedding_model": "old", "dim": 1}))
|
||||
|
||||
with patch("documents.models.Document.objects.all") as mock_all:
|
||||
mock_queryset = MagicMock()
|
||||
mock_queryset.exists.return_value = True
|
||||
mock_queryset.__iter__.return_value = iter([real_document])
|
||||
mock_all.return_value = mock_queryset
|
||||
indexing.update_llm_index(rebuild=True)
|
||||
|
||||
assert not stale_meta.exists(), (
|
||||
"update_llm_index(rebuild=True) must remove stale meta.json"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_update_llm_index_rebuilds_on_model_name_change(
|
||||
temp_llm_index_dir: Path,
|
||||
@@ -207,10 +201,10 @@ def test_update_llm_index_rebuilds_on_model_name_change(
|
||||
):
|
||||
indexing.update_llm_index(rebuild=False)
|
||||
|
||||
store = indexing.get_vector_store()
|
||||
# Schema metadata only updates when the table is dropped and recreated, never on
|
||||
# incremental writes -- so "model-b" here proves a full rebuild happened.
|
||||
assert store.stored_model_name() == "model-b"
|
||||
with indexing.get_vector_store() as store:
|
||||
# Schema metadata only updates when the table is dropped and recreated, never
|
||||
# on incremental writes -- so "model-b" here proves a full rebuild happened.
|
||||
assert store.stored_model_name() == "model-b"
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@@ -254,10 +248,10 @@ def test_update_llm_index_partial_update(
|
||||
|
||||
indexing.update_llm_index(rebuild=False)
|
||||
|
||||
store = indexing.get_vector_store()
|
||||
assert store.table_exists(), (
|
||||
"Expected the LanceDB table to exist after incremental update"
|
||||
)
|
||||
with indexing.get_vector_store() as store:
|
||||
assert store.table_exists(), (
|
||||
"Expected the vector store table to exist after incremental update"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@@ -269,10 +263,10 @@ def test_add_or_update_document_updates_existing_entry(
|
||||
indexing.update_llm_index(rebuild=True)
|
||||
indexing.llm_index_add_or_update_document(real_document)
|
||||
|
||||
store = indexing.get_vector_store()
|
||||
assert store.table_exists(), (
|
||||
"Expected the LanceDB table to exist after add-or-update"
|
||||
)
|
||||
with indexing.get_vector_store() as store:
|
||||
assert store.table_exists(), (
|
||||
"Expected the vector store table to exist after add-or-update"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@@ -409,6 +403,42 @@ def test_query_similar_documents(
|
||||
assert result == mock_filtered_docs
|
||||
|
||||
|
||||
@override_settings(
|
||||
LLM_EMBEDDING_BACKEND="huggingface",
|
||||
LLM_EMBEDDING_CHUNK_SIZE=32,
|
||||
LLM_BACKEND="ollama",
|
||||
)
|
||||
def test_query_similar_documents_truncates_query_to_embedding_chunk_size(
|
||||
temp_llm_index_dir: Path,
|
||||
real_document: Document,
|
||||
) -> None:
|
||||
real_document.content = " ".join(f"word{i}" for i in range(200))
|
||||
with (
|
||||
patch("paperless_ai.indexing.load_or_build_index") as mock_load_or_build_index,
|
||||
patch(
|
||||
"paperless_ai.indexing.llm_index_exists",
|
||||
) as mock_vector_store_exists,
|
||||
patch("llama_index.core.retrievers.VectorIndexRetriever") as mock_retriever_cls,
|
||||
patch("paperless_ai.indexing.Document.objects.filter") as mock_filter,
|
||||
patch("paperless_ai.indexing.truncate_content") as mock_truncate_content,
|
||||
):
|
||||
mock_vector_store_exists.return_value = True
|
||||
mock_load_or_build_index.return_value = MagicMock()
|
||||
mock_truncate_content.return_value = "wrong helper"
|
||||
|
||||
mock_retriever = MagicMock()
|
||||
mock_retriever.retrieve.return_value = []
|
||||
mock_retriever_cls.return_value = mock_retriever
|
||||
mock_filter.return_value = []
|
||||
|
||||
indexing.query_similar_documents(real_document, top_k=3)
|
||||
|
||||
mock_truncate_content.assert_not_called()
|
||||
query_text = mock_retriever.retrieve.call_args.args[0]
|
||||
assert query_text
|
||||
assert "word199" not in query_text
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_query_similar_documents_triggers_update_when_index_missing(
|
||||
temp_llm_index_dir: Path,
|
||||
@@ -461,7 +491,7 @@ def test_query_similar_documents_empty_allow_list_fails_closed(
|
||||
|
||||
|
||||
class TestUpdateLlmIndexEmptyDocumentSet:
|
||||
"""update_llm_index must clear the LanceDB table when all documents are deleted.
|
||||
"""update_llm_index must clear the vector store table when all documents are deleted.
|
||||
|
||||
Without this, the stale vectors are never cleared and subsequent similarity
|
||||
searches return phantom hits for document IDs that no longer exist in the DB.
|
||||
@@ -489,10 +519,11 @@ class TestUpdateLlmIndexEmptyDocumentSet:
|
||||
)
|
||||
indexing.update_llm_index(rebuild=True)
|
||||
|
||||
store = indexing.get_vector_store()
|
||||
assert store.table_exists(), (
|
||||
"Precondition failed: expected the LanceDB table to exist before deletion"
|
||||
)
|
||||
with indexing.get_vector_store() as store:
|
||||
assert store.table_exists(), (
|
||||
"Precondition failed: expected the vector store table to exist "
|
||||
"before deletion"
|
||||
)
|
||||
|
||||
# Step 2: delete all documents
|
||||
Document.objects.all().delete()
|
||||
@@ -503,10 +534,11 @@ class TestUpdateLlmIndexEmptyDocumentSet:
|
||||
indexing.update_llm_index(rebuild=True)
|
||||
|
||||
# Step 4: the table must be absent (no rows) — phantom vectors gone
|
||||
store2 = indexing.get_vector_store()
|
||||
assert not store2.table_exists(), (
|
||||
"Expected the LanceDB table to be absent after rebuilding with no documents"
|
||||
)
|
||||
with indexing.get_vector_store() as store2:
|
||||
assert not store2.table_exists(), (
|
||||
"Expected the vector store table to be absent after rebuilding "
|
||||
"with no documents"
|
||||
)
|
||||
|
||||
|
||||
class TestDocumentUpdatedSignalTriggersLlmReindex:
|
||||
@@ -578,11 +610,11 @@ class TestLlmIndexAddOrUpdateDocumentEmptyContent:
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_llm_index_compact_uses_zero_retention(
|
||||
def test_llm_index_compact_uses_force(
|
||||
temp_llm_index_dir: Path,
|
||||
mocker: pytest_mock.MockerFixture,
|
||||
) -> None:
|
||||
"""compact must use retention_seconds=0 to clear all MVCC history immediately."""
|
||||
"""compact must use force=True to rebuild the table and reclaim space immediately."""
|
||||
mock_store = mocker.MagicMock()
|
||||
mocker.patch(
|
||||
"paperless_ai.indexing.write_store",
|
||||
@@ -594,7 +626,7 @@ def test_llm_index_compact_uses_zero_retention(
|
||||
|
||||
indexing.llm_index_compact()
|
||||
|
||||
mock_store.compact.assert_called_once_with(retention_seconds=0)
|
||||
mock_store.compact.assert_called_once_with(force=True)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@@ -678,16 +710,14 @@ class TestLlmIndexLocking:
|
||||
|
||||
@pytest.mark.django_db
|
||||
@pytest.mark.django_db
|
||||
class TestLanceDbIndexing:
|
||||
class TestVectorStoreIndexing:
|
||||
def test_get_vector_store_roundtrip(
|
||||
self,
|
||||
temp_llm_index_dir: Path,
|
||||
mock_embed_model: FakeEmbedding,
|
||||
) -> None:
|
||||
from paperless_ai.vector_store import PaperlessLanceVectorStore
|
||||
|
||||
store = indexing.get_vector_store()
|
||||
assert isinstance(store, PaperlessLanceVectorStore)
|
||||
with indexing.get_vector_store() as store:
|
||||
assert isinstance(store, PaperlessSqliteVecVectorStore)
|
||||
|
||||
def test_add_then_remove_document(
|
||||
self,
|
||||
@@ -696,12 +726,13 @@ class TestLanceDbIndexing:
|
||||
real_document: Document,
|
||||
) -> None:
|
||||
indexing.llm_index_add_or_update_document(real_document)
|
||||
store = indexing.get_vector_store()
|
||||
table = store.client.open_table(indexing.LLM_INDEX_TABLE)
|
||||
assert table.count_rows() >= 1
|
||||
with indexing.get_vector_store() as store:
|
||||
assert store.table_exists()
|
||||
count_sql = "SELECT count(*) FROM documents"
|
||||
assert store.client.execute(count_sql).fetchone()[0] >= 1
|
||||
|
||||
indexing.llm_index_remove_document(real_document)
|
||||
assert store.client.open_table(indexing.LLM_INDEX_TABLE).count_rows() == 0
|
||||
indexing.llm_index_remove_document(real_document)
|
||||
assert store.client.execute(count_sql).fetchone()[0] == 0
|
||||
|
||||
def test_update_shrinks_chunks_without_orphans(
|
||||
self,
|
||||
@@ -712,16 +743,17 @@ class TestLanceDbIndexing:
|
||||
real_document.content = "word " * 4000 # many chunks
|
||||
real_document.save()
|
||||
indexing.llm_index_add_or_update_document(real_document)
|
||||
store = indexing.get_vector_store()
|
||||
big = store.client.open_table(indexing.LLM_INDEX_TABLE).count_rows()
|
||||
count_sql = "SELECT count(*) FROM documents"
|
||||
with indexing.get_vector_store() as store:
|
||||
big = store.client.execute(count_sql).fetchone()[0]
|
||||
|
||||
real_document.content = "short" # one chunk
|
||||
real_document.save()
|
||||
indexing.llm_index_add_or_update_document(real_document)
|
||||
real_document.content = "short" # one chunk
|
||||
real_document.save()
|
||||
indexing.llm_index_add_or_update_document(real_document)
|
||||
|
||||
rows = store.client.open_table(indexing.LLM_INDEX_TABLE).count_rows()
|
||||
assert rows < big
|
||||
assert rows >= 1
|
||||
rows = store.client.execute(count_sql).fetchone()[0]
|
||||
assert rows < big
|
||||
assert rows >= 1
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
|
||||
@@ -3,9 +3,13 @@ from unittest.mock import MagicMock
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from llama_index.core import settings as llama_settings
|
||||
from llama_index.core.embeddings.mock_embed_model import MockEmbedding
|
||||
from llama_index.core.schema import TextNode
|
||||
|
||||
from documents.tests.factories import DocumentFactory
|
||||
from paperless_ai import chat
|
||||
from paperless_ai import indexing
|
||||
from paperless_ai.chat import CHAT_ERROR_MESSAGE
|
||||
from paperless_ai.chat import CHAT_METADATA_DELIMITER
|
||||
from paperless_ai.chat import stream_chat_with_documents
|
||||
@@ -13,9 +17,6 @@ from paperless_ai.chat import stream_chat_with_documents
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def patch_embed_model():
|
||||
from llama_index.core import settings as llama_settings
|
||||
from llama_index.core.embeddings.mock_embed_model import MockEmbedding
|
||||
|
||||
# Use a real BaseEmbedding subclass to satisfy llama-index 0.14 validation
|
||||
llama_settings.Settings.embed_model = MockEmbedding(embed_dim=1536)
|
||||
yield
|
||||
@@ -241,8 +242,6 @@ class TestStreamChatRetrieval:
|
||||
temp_llm_index_dir,
|
||||
mock_embed_model,
|
||||
) -> None:
|
||||
from documents.tests.factories import DocumentFactory
|
||||
|
||||
doc = DocumentFactory.create(content="hello world")
|
||||
# Nothing indexed for this document yet.
|
||||
out = list(chat.stream_chat_with_documents("question?", [doc]))
|
||||
@@ -258,9 +257,6 @@ class TestStreamChatRetrieval:
|
||||
requested documents only — content from other indexed documents must
|
||||
not be surfaced.
|
||||
"""
|
||||
from documents.tests.factories import DocumentFactory
|
||||
from paperless_ai import indexing
|
||||
|
||||
included = DocumentFactory.create(content="included document content")
|
||||
excluded = DocumentFactory.create(content="excluded document content")
|
||||
indexing.llm_index_add_or_update_document(included)
|
||||
|
||||
@@ -3,12 +3,14 @@ from unittest.mock import ANY
|
||||
from unittest.mock import MagicMock
|
||||
from unittest.mock import patch
|
||||
|
||||
import httpx
|
||||
import openai
|
||||
import pytest
|
||||
from llama_index.core.llms import ChatMessage
|
||||
from llama_index.core.llms.llm import ToolSelection
|
||||
|
||||
from paperless_ai.client import LLM_SYSTEM_PROMPT
|
||||
from paperless_ai.client import AIClient
|
||||
from paperless_ai.exceptions import LLMTimeoutError
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -17,6 +19,7 @@ def mock_ai_config():
|
||||
mock_config = MagicMock()
|
||||
mock_config.llm_allow_internal_endpoints = True
|
||||
mock_config.llm_context_size = 8192
|
||||
mock_config.llm_request_timeout = 120
|
||||
MockAIConfig.return_value = mock_config
|
||||
yield mock_config
|
||||
|
||||
@@ -64,6 +67,7 @@ def test_get_llm_openai(mock_ai_config, mock_openai_llm):
|
||||
model="test_model",
|
||||
api_base="http://test-url",
|
||||
api_key="test_api_key",
|
||||
timeout=120,
|
||||
is_chat_model=True,
|
||||
is_function_calling_model=True,
|
||||
system_prompt=LLM_SYSTEM_PROMPT,
|
||||
@@ -151,17 +155,38 @@ def test_run_llm_query_openai_uses_tools(mock_ai_config, mock_openai_llm):
|
||||
mock_llm_instance.chat_with_tools.assert_called_once()
|
||||
|
||||
|
||||
def test_run_chat(mock_ai_config, mock_ollama_llm):
|
||||
def test_run_llm_query_openai_timeout_raises_local_error(
|
||||
mock_ai_config,
|
||||
mock_openai_llm,
|
||||
):
|
||||
mock_ai_config.llm_backend = "openai-like"
|
||||
mock_ai_config.llm_model = "test_model"
|
||||
mock_ai_config.llm_api_key = "test_api_key"
|
||||
mock_ai_config.llm_endpoint = "http://test-url"
|
||||
|
||||
request = httpx.Request("POST", "http://test-url/v1/chat/completions")
|
||||
mock_openai_llm.return_value.chat_with_tools.side_effect = openai.APITimeoutError(
|
||||
request,
|
||||
)
|
||||
|
||||
client = AIClient()
|
||||
|
||||
with pytest.raises(LLMTimeoutError):
|
||||
client.run_llm_query("test_prompt")
|
||||
|
||||
|
||||
def test_run_llm_query_httpx_timeout_raises_local_error(
|
||||
mock_ai_config,
|
||||
mock_ollama_llm,
|
||||
):
|
||||
mock_ai_config.llm_backend = "ollama"
|
||||
mock_ai_config.llm_model = "test_model"
|
||||
mock_ai_config.llm_endpoint = "http://test-url"
|
||||
|
||||
mock_llm_instance = mock_ollama_llm.return_value
|
||||
mock_llm_instance.chat.return_value = "test_chat_result"
|
||||
mock_llm_instance.chat.side_effect = httpx.ReadTimeout("timed out")
|
||||
|
||||
client = AIClient()
|
||||
messages = [ChatMessage(role="user", content="Hello")]
|
||||
result = client.run_chat(messages)
|
||||
|
||||
mock_llm_instance.chat.assert_called_once_with(messages)
|
||||
assert result == "test_chat_result"
|
||||
with pytest.raises(LLMTimeoutError):
|
||||
client.run_llm_query("test_prompt")
|
||||
|
||||
@@ -19,6 +19,7 @@ def mock_ai_config():
|
||||
MockAIConfig.return_value.llm_embedding_endpoint = None
|
||||
MockAIConfig.return_value.llm_allow_internal_endpoints = True
|
||||
MockAIConfig.return_value.llm_context_size = 8192
|
||||
MockAIConfig.return_value.llm_request_timeout = 120
|
||||
yield MockAIConfig
|
||||
|
||||
|
||||
@@ -71,6 +72,7 @@ def test_get_embedding_model_openai(mock_ai_config):
|
||||
model_name="text-embedding-3-small",
|
||||
api_key="test_api_key",
|
||||
api_base="http://test-url",
|
||||
timeout=120,
|
||||
http_client=ANY,
|
||||
async_http_client=ANY,
|
||||
)
|
||||
@@ -92,6 +94,7 @@ def test_get_embedding_model_openai_prefers_embedding_endpoint(mock_ai_config):
|
||||
model_name="text-embedding-3-small",
|
||||
api_key="test_api_key",
|
||||
api_base="http://embedding-url",
|
||||
timeout=120,
|
||||
http_client=ANY,
|
||||
async_http_client=ANY,
|
||||
)
|
||||
@@ -224,15 +227,17 @@ def test_build_llm_index_text(mock_document):
|
||||
|
||||
result = build_llm_index_text(mock_document)
|
||||
|
||||
# Structured fields live in node.metadata for LLM context — not body text
|
||||
# Structured fields live in node.metadata for LLM context -- not body text
|
||||
assert "Title: Test Title" not in result
|
||||
assert "Created: 2023-01-01" not in result
|
||||
assert "Tags: Tag1, Tag2" not in result
|
||||
assert "Document Type: Invoice" not in result
|
||||
assert "Correspondent: Test Correspondent" not in result
|
||||
assert "Filename:" not in result
|
||||
assert "Storage Path:" not in result
|
||||
assert "Archive Serial Number:" not in result
|
||||
|
||||
# Fields without a metadata equivalent stay in body text
|
||||
assert "Filename: test_file.pdf" in result
|
||||
assert "Notes: Note1,Note2" in result
|
||||
assert "Content:\n\nThis is the document content." in result
|
||||
assert "Custom Field - Field1: Value1\nCustom Field - Field2: Value2" in result
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
import logging
|
||||
import sqlite3
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from django.conf import settings
|
||||
from filelock import ReadWriteLock
|
||||
from llama_index.core.schema import TextNode
|
||||
from pytest_django.fixtures import SettingsWrapper
|
||||
|
||||
from paperless_ai import indexing
|
||||
from paperless_ai.vector_store import PaperlessSqliteVecVectorStore
|
||||
|
||||
DIM = 8
|
||||
|
||||
|
||||
def _node(node_id: str, document_id: str, *, seed: float = 0.0) -> TextNode:
|
||||
node = TextNode(
|
||||
id_=node_id,
|
||||
text="chunk",
|
||||
metadata={"document_id": document_id, "modified": "2026-06-01T00:00:00"},
|
||||
)
|
||||
node.relationships = {}
|
||||
node.embedding = [seed + i / 100 for i in range(DIM)]
|
||||
return node
|
||||
|
||||
|
||||
def _seed_bloated_index(index_dir: Path) -> None:
|
||||
"""Create an index whose cumulative inserts far exceed live rows."""
|
||||
store = PaperlessSqliteVecVectorStore(uri=str(index_dir))
|
||||
store.add([_node(f"d{j}", str(j), seed=float(j)) for j in range(20)])
|
||||
for cycle in range(6):
|
||||
for j in range(20):
|
||||
store.upsert_document(
|
||||
str(j),
|
||||
[_node(f"d{j}-c{cycle}", str(j), seed=float(j))],
|
||||
)
|
||||
store.client.close()
|
||||
|
||||
|
||||
def _bloat_ratio(index_dir: Path) -> float:
|
||||
store = PaperlessSqliteVecVectorStore(uri=str(index_dir))
|
||||
live = store.client.execute("SELECT count(*) FROM documents").fetchone()[0]
|
||||
row = store.client.execute(
|
||||
"SELECT value FROM index_meta WHERE key = 'total_inserts'",
|
||||
).fetchone()
|
||||
total = int(row["value"]) if row else live
|
||||
store.client.close()
|
||||
return total / max(live, 1)
|
||||
|
||||
|
||||
def _integrity_ok(index_dir: Path) -> bool:
|
||||
store = PaperlessSqliteVecVectorStore(uri=str(index_dir))
|
||||
result = store.client.execute("PRAGMA integrity_check").fetchone()[0]
|
||||
rows = store.client.execute("SELECT count(*) FROM documents").fetchone()[0]
|
||||
store.client.close()
|
||||
return result == "ok" and rows == 20
|
||||
|
||||
|
||||
def _reader_lock() -> ReadWriteLock:
|
||||
# A distinct instance simulates a reader in another process: it coordinates
|
||||
# with the production lock purely through SQLite, never reentrant upgrade.
|
||||
return ReadWriteLock(str(settings.LLM_INDEX_RWLOCK), is_singleton=False)
|
||||
|
||||
|
||||
class TestCompactionLock:
|
||||
def test_compaction_skips_when_a_reader_holds_the_lock(
|
||||
self,
|
||||
temp_llm_index_dir: Path,
|
||||
settings: SettingsWrapper,
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
_seed_bloated_index(temp_llm_index_dir)
|
||||
settings.LLM_INDEX_COMPACTION_LOCK_TIMEOUT = 0.3
|
||||
|
||||
lock = _reader_lock()
|
||||
with lock.read_lock(), caplog.at_level(logging.INFO):
|
||||
indexing.llm_index_compact() # must not raise
|
||||
lock.close()
|
||||
|
||||
# Swap was skipped: bloat remains, nothing corrupted, data intact.
|
||||
assert _integrity_ok(temp_llm_index_dir)
|
||||
assert _bloat_ratio(temp_llm_index_dir) > 2
|
||||
assert "Skipping LLM index compaction" in caplog.text
|
||||
|
||||
def test_compaction_runs_when_no_reader_holds_the_lock(
|
||||
self,
|
||||
temp_llm_index_dir: Path,
|
||||
) -> None:
|
||||
_seed_bloated_index(temp_llm_index_dir)
|
||||
assert _bloat_ratio(temp_llm_index_dir) > 2
|
||||
|
||||
indexing.llm_index_compact()
|
||||
|
||||
assert _bloat_ratio(temp_llm_index_dir) == pytest.approx(1.0)
|
||||
assert _integrity_ok(temp_llm_index_dir)
|
||||
|
||||
def test_normal_write_is_not_gated_by_the_compaction_lock(
|
||||
self,
|
||||
temp_llm_index_dir: Path,
|
||||
) -> None:
|
||||
"""A held exclusive lock must not block ordinary writes (WAL handles them)."""
|
||||
_seed_bloated_index(temp_llm_index_dir)
|
||||
done = threading.Event()
|
||||
|
||||
def remove() -> None:
|
||||
indexing.llm_index_remove_document(MagicMock(id=999))
|
||||
done.set()
|
||||
|
||||
holder = _reader_lock()
|
||||
with holder.write_lock():
|
||||
t = threading.Thread(target=remove)
|
||||
t.start()
|
||||
finished = done.wait(timeout=5)
|
||||
t.join(timeout=2)
|
||||
holder.close()
|
||||
assert finished, "a normal write blocked on the compaction lock"
|
||||
|
||||
|
||||
class TestReadStore:
|
||||
def test_closes_connection_on_exit(self, temp_llm_index_dir: Path) -> None:
|
||||
with indexing.read_store() as store:
|
||||
conn = store.client
|
||||
assert conn.execute("SELECT 1").fetchone()[0] == 1
|
||||
with pytest.raises(sqlite3.ProgrammingError):
|
||||
conn.execute("SELECT 1")
|
||||
|
||||
def test_concurrent_readers_do_not_block(self, temp_llm_index_dir: Path) -> None:
|
||||
_seed_bloated_index(temp_llm_index_dir)
|
||||
with indexing.read_store() as a, indexing.read_store() as b:
|
||||
assert a.table_exists()
|
||||
assert b.table_exists()
|
||||
@@ -12,7 +12,7 @@ class TestLazyAiImports:
|
||||
"os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'paperless.settings')\n"
|
||||
"django.setup()\n"
|
||||
"import documents.tasks # noqa: F401\n"
|
||||
"leaked = [m for m in ('lancedb', 'pyarrow', 'llama_index') "
|
||||
"leaked = [m for m in ('lancedb', 'pyarrow', 'llama_index', 'sqlite_vec') "
|
||||
"if m in sys.modules]\n"
|
||||
"assert not leaked, f'AI libraries leaked into the light path: {leaked}'\n"
|
||||
)
|
||||
|
||||
@@ -1,417 +1,623 @@
|
||||
import sqlite3
|
||||
from collections.abc import Generator
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from llama_index.core.schema import NodeRelationship
|
||||
from llama_index.core.schema import RelatedNodeInfo
|
||||
from llama_index.core.schema import TextNode
|
||||
from llama_index.core.vector_stores.types import FilterOperator
|
||||
from llama_index.core.vector_stores.types import MetadataFilter
|
||||
from llama_index.core.vector_stores.types import MetadataFilters
|
||||
from llama_index.core.vector_stores.types import VectorStoreQuery
|
||||
|
||||
from paperless_ai.vector_store import PaperlessLanceVectorStore
|
||||
from paperless_ai.vector_store import DB_FILENAME
|
||||
from paperless_ai.vector_store import DEFAULT_TABLE_NAME
|
||||
from paperless_ai.vector_store import MIGRATIONS
|
||||
from paperless_ai.vector_store import SCHEMA_VERSION
|
||||
from paperless_ai.vector_store import Migration
|
||||
from paperless_ai.vector_store import PaperlessSqliteVecVectorStore
|
||||
from paperless_ai.vector_store import _build_where
|
||||
|
||||
DIM = 8
|
||||
DIM = 16
|
||||
|
||||
|
||||
def _node(node_id: str, document_id: str, text: str, vec: float) -> TextNode:
|
||||
node = TextNode(id_=node_id, text=text, metadata={"document_id": document_id})
|
||||
node.set_content(text)
|
||||
node.embedding = [vec] * DIM
|
||||
# Use relationships so ref_doc_id resolves correctly (it's a read-only property)
|
||||
node.relationships = {
|
||||
NodeRelationship.SOURCE: RelatedNodeInfo(node_id=document_id),
|
||||
}
|
||||
def make_node(
|
||||
node_id: str,
|
||||
document_id: str,
|
||||
*,
|
||||
modified: str = "2026-06-10T00:00:00",
|
||||
seed: float = 0.0,
|
||||
text: str = "some text",
|
||||
) -> TextNode:
|
||||
node = TextNode(
|
||||
id_=node_id,
|
||||
text=text,
|
||||
metadata={"document_id": document_id, "modified": modified},
|
||||
)
|
||||
node.relationships = {}
|
||||
node.embedding = [seed + i / 100 for i in range(DIM)]
|
||||
return node
|
||||
|
||||
|
||||
class TestPaperlessLanceVectorStoreCrud:
|
||||
@pytest.fixture
|
||||
def store(self, tmp_path: Path) -> PaperlessLanceVectorStore:
|
||||
return PaperlessLanceVectorStore(uri=str(tmp_path / "idx"))
|
||||
@pytest.fixture
|
||||
def store(tmp_path: Path) -> Generator[PaperlessSqliteVecVectorStore, None, None]:
|
||||
with PaperlessSqliteVecVectorStore(uri=str(tmp_path)) as store:
|
||||
yield store
|
||||
|
||||
def test_add_then_query_returns_node(
|
||||
self,
|
||||
store: PaperlessLanceVectorStore,
|
||||
) -> None:
|
||||
store.add([_node("1-0", "1", "alpha", 0.1), _node("2-0", "2", "beta", 0.9)])
|
||||
|
||||
result = store.query(
|
||||
VectorStoreQuery(query_embedding=[0.1] * DIM, similarity_top_k=1),
|
||||
)
|
||||
def _query(
|
||||
store: PaperlessSqliteVecVectorStore,
|
||||
embedding: list[float],
|
||||
top_k: int = 5,
|
||||
filters=None,
|
||||
):
|
||||
return store.query(
|
||||
VectorStoreQuery(
|
||||
query_embedding=embedding,
|
||||
similarity_top_k=top_k,
|
||||
filters=filters,
|
||||
),
|
||||
)
|
||||
|
||||
assert len(result.nodes) == 1
|
||||
|
||||
def _eq_filter(key: str, value: str):
|
||||
return MetadataFilters(
|
||||
filters=[MetadataFilter(key=key, operator=FilterOperator.EQ, value=value)],
|
||||
)
|
||||
|
||||
|
||||
def _in_filter(document_ids: list[str]):
|
||||
return MetadataFilters(
|
||||
filters=[
|
||||
MetadataFilter(
|
||||
key="document_id",
|
||||
operator=FilterOperator.IN,
|
||||
value=document_ids,
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
class TestCrud:
|
||||
def test_add_then_query_returns_node(self, store) -> None:
|
||||
node = make_node("n1", "1")
|
||||
assert store.add([node]) == ["n1"]
|
||||
result = _query(store, node.embedding, top_k=1)
|
||||
assert result.ids == ["n1"]
|
||||
assert result.nodes[0].metadata["document_id"] == "1"
|
||||
# cosine distance of the identical vector is 0 -> similarity 1
|
||||
assert result.similarities[0] == pytest.approx(1.0)
|
||||
|
||||
def test_query_empty_table_returns_empty_no_raise(
|
||||
self,
|
||||
store: PaperlessLanceVectorStore,
|
||||
) -> None:
|
||||
result = store.query(
|
||||
VectorStoreQuery(query_embedding=[0.1] * DIM, similarity_top_k=5),
|
||||
)
|
||||
assert result.nodes == []
|
||||
assert result.ids == []
|
||||
def test_query_empty_store_returns_empty_no_raise(self, store) -> None:
|
||||
result = _query(store, [0.0] * DIM)
|
||||
assert result.ids == [] and result.nodes == [] and result.similarities == []
|
||||
|
||||
def test_delete_removes_all_chunks_of_document(
|
||||
self,
|
||||
store: PaperlessLanceVectorStore,
|
||||
) -> None:
|
||||
store.add([_node("1-0", "1", "a", 0.1), _node("1-1", "1", "b", 0.2)])
|
||||
store.add([_node("2-0", "2", "c", 0.9)])
|
||||
def test_add_empty_list_is_noop(self, store) -> None:
|
||||
assert store.add([]) == []
|
||||
assert not store.table_exists()
|
||||
|
||||
def test_delete_removes_all_chunks_of_document(self, store) -> None:
|
||||
store.add([make_node("a1", "1"), make_node("a2", "1"), make_node("b1", "2")])
|
||||
store.delete("1")
|
||||
result = _query(store, [0.0] * DIM, top_k=10)
|
||||
assert result.ids == ["b1"]
|
||||
|
||||
assert store.client.open_table("documents").count_rows() == 1
|
||||
def test_query_with_in_filter_scopes_results(self, store) -> None:
|
||||
store.add(
|
||||
[
|
||||
make_node("a1", "1", seed=0.0),
|
||||
make_node("b1", "2", seed=1.0),
|
||||
make_node("c1", "3", seed=2.0),
|
||||
],
|
||||
)
|
||||
result = _query(store, [0.0] * DIM, top_k=10, filters=_in_filter(["2", "3"]))
|
||||
assert sorted(result.ids) == ["b1", "c1"]
|
||||
|
||||
def test_query_with_in_filter_scopes_results(
|
||||
self,
|
||||
store: PaperlessLanceVectorStore,
|
||||
) -> None:
|
||||
store.add([_node("1-0", "1", "a", 0.1), _node("2-0", "2", "b", 0.1)])
|
||||
def test_query_respects_top_k_with_filter(self, store) -> None:
|
||||
# k semantics: global top-k even with IN filters (document_id is a
|
||||
# metadata column, not a partition key -- see design doc).
|
||||
store.add(
|
||||
[make_node(f"n{i}", str(i % 4), seed=float(i)) for i in range(12)],
|
||||
)
|
||||
result = _query(
|
||||
store,
|
||||
[0.0] * DIM,
|
||||
top_k=3,
|
||||
filters=_in_filter(["0", "1", "2", "3"]),
|
||||
)
|
||||
assert len(result.ids) == 3
|
||||
assert result.similarities == sorted(result.similarities, reverse=True)
|
||||
|
||||
result = store.query(
|
||||
VectorStoreQuery(
|
||||
query_embedding=[0.1] * DIM,
|
||||
similarity_top_k=5,
|
||||
filters=MetadataFilters(
|
||||
filters=[
|
||||
MetadataFilter(
|
||||
key="document_id",
|
||||
operator=FilterOperator.IN,
|
||||
value=["2"],
|
||||
),
|
||||
],
|
||||
def test_get_nodes_filter_and_empty_paths(self, store) -> None:
|
||||
assert store.get_nodes(filters=_in_filter(["1"])) == [] # no table yet
|
||||
store.add([make_node("a1", "1"), make_node("b1", "2")])
|
||||
nodes = store.get_nodes(filters=_in_filter(["1"]))
|
||||
assert [n.node_id for n in nodes] == ["a1"]
|
||||
assert nodes[0].embedding is not None
|
||||
assert store.get_nodes(filters=_in_filter(["999"])) == []
|
||||
|
||||
def test_query_with_eq_filter_scopes_results(self, store) -> None:
|
||||
store.add(
|
||||
[
|
||||
make_node("a1", "1", seed=0.0),
|
||||
make_node("b1", "2", seed=1.0),
|
||||
make_node("c1", "3", seed=2.0),
|
||||
],
|
||||
)
|
||||
result = _query(
|
||||
store,
|
||||
[0.0] * DIM,
|
||||
top_k=10,
|
||||
filters=_eq_filter("document_id", "2"),
|
||||
)
|
||||
assert result.ids == ["b1"]
|
||||
|
||||
def test_get_nodes_node_ids_not_implemented(self, store) -> None:
|
||||
with pytest.raises(NotImplementedError):
|
||||
store.get_nodes(node_ids=["x"])
|
||||
|
||||
def test_fresh_instance_sees_existing_table(self, store, tmp_path: Path) -> None:
|
||||
store.add([make_node("a1", "1")])
|
||||
with PaperlessSqliteVecVectorStore(uri=str(tmp_path)) as reopened:
|
||||
assert reopened.table_exists()
|
||||
assert reopened.vector_dim() == DIM
|
||||
assert _query(reopened, [0.0] * DIM, top_k=1).ids == ["a1"]
|
||||
|
||||
def test_table_exists_and_drop(self, store) -> None:
|
||||
assert not store.table_exists()
|
||||
store.add([make_node("a1", "1")])
|
||||
assert store.table_exists()
|
||||
store.drop_table()
|
||||
assert not store.table_exists()
|
||||
assert store.vector_dim() is None
|
||||
|
||||
|
||||
class TestBuildWhere:
|
||||
def test_fails_closed_when_no_filter_is_translatable(self) -> None:
|
||||
# A nested MetadataFilters is not a MetadataFilter, so it is skipped.
|
||||
# With no translatable clauses, the function must fail closed rather
|
||||
# than emit "()" (invalid SQL) and never widen document access.
|
||||
nested = MetadataFilters(
|
||||
filters=[
|
||||
MetadataFilter(
|
||||
key="document_id",
|
||||
operator=FilterOperator.EQ,
|
||||
value="1",
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
where, params = _build_where(MetadataFilters(filters=[nested]))
|
||||
assert where == "1 = 0"
|
||||
assert params == []
|
||||
|
||||
assert [n.metadata["document_id"] for n in result.nodes] == ["2"]
|
||||
def test_query_with_untranslatable_filter_returns_no_rows(self, store) -> None:
|
||||
store.add([make_node("a1", "1"), make_node("b1", "2")])
|
||||
nested = MetadataFilters(
|
||||
filters=[
|
||||
MetadataFilter(
|
||||
key="document_id",
|
||||
operator=FilterOperator.EQ,
|
||||
value="1",
|
||||
),
|
||||
],
|
||||
)
|
||||
filters = MetadataFilters(filters=[nested])
|
||||
# Must not raise (no "WHERE ()") and must return nothing (fail closed).
|
||||
assert _query(store, [0.0] * DIM, top_k=5, filters=filters).ids == []
|
||||
assert store.get_nodes(filters=filters) == []
|
||||
|
||||
def test_get_nodes_filter_returns_empty_cleanly(
|
||||
|
||||
class TestUpsert:
|
||||
def test_upsert_replaces_and_prunes_stale_chunks(self, store) -> None:
|
||||
store.add(
|
||||
[make_node("d1c1", "1"), make_node("d1c2", "1"), make_node("d2c1", "2")],
|
||||
)
|
||||
store.upsert_document("1", [make_node("d1new", "1")])
|
||||
result = _query(store, [0.0] * DIM, top_k=10)
|
||||
assert sorted(result.ids) == ["d1new", "d2c1"]
|
||||
|
||||
def test_upsert_creates_table_when_missing(self, store) -> None:
|
||||
store.upsert_document("1", [make_node("a1", "1")])
|
||||
assert _query(store, [0.0] * DIM, top_k=1).ids == ["a1"]
|
||||
|
||||
def test_upsert_empty_nodes_removes_document(self, store) -> None:
|
||||
store.add([make_node("a1", "1"), make_node("b1", "2")])
|
||||
store.upsert_document("1", [])
|
||||
assert _query(store, [0.0] * DIM, top_k=10).ids == ["b1"]
|
||||
|
||||
def test_upsert_is_atomic_for_concurrent_readers(
|
||||
self,
|
||||
store: PaperlessLanceVectorStore,
|
||||
store,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
store.add([_node("1-0", "1", "a", 0.1)])
|
||||
nodes = store.get_nodes(
|
||||
filters=MetadataFilters(
|
||||
filters=[
|
||||
MetadataFilter(
|
||||
key="document_id",
|
||||
operator=FilterOperator.IN,
|
||||
value=["999"],
|
||||
),
|
||||
],
|
||||
),
|
||||
)
|
||||
assert nodes == []
|
||||
"""A second connection must never observe document 1 half-replaced."""
|
||||
store.add([make_node("a1", "1"), make_node("a2", "1")])
|
||||
with PaperlessSqliteVecVectorStore(uri=str(tmp_path)) as reader:
|
||||
store.upsert_document("1", [make_node("a3", "1")])
|
||||
ids = [n.node_id for n in reader.get_nodes(filters=_in_filter(["1"]))]
|
||||
assert ids == ["a3"]
|
||||
|
||||
def test_get_nodes_returns_empty_when_no_table(
|
||||
self,
|
||||
store: PaperlessLanceVectorStore,
|
||||
) -> None:
|
||||
result = store.get_nodes(
|
||||
filters=MetadataFilters(
|
||||
filters=[
|
||||
MetadataFilter(
|
||||
key="document_id",
|
||||
operator=FilterOperator.IN,
|
||||
value=["1"],
|
||||
),
|
||||
],
|
||||
),
|
||||
)
|
||||
assert result == []
|
||||
|
||||
def test_fresh_instance_filters_existing_table(
|
||||
class TestMetadataCoercion:
|
||||
def test_none_metadata_values_become_empty_strings(self, store) -> None:
|
||||
node = make_node("a1", "1")
|
||||
node.metadata["modified"] = None
|
||||
store.add([node]) # must not raise (vec0 rejects NULL metadata)
|
||||
assert store.get_modified_times() == {"1": ""}
|
||||
|
||||
|
||||
class TestModelNameTracking:
|
||||
def test_stored_model_name_none_without_table(self, tmp_path: Path) -> None:
|
||||
with PaperlessSqliteVecVectorStore(
|
||||
uri=str(tmp_path),
|
||||
embed_model_name="model-a",
|
||||
) as store:
|
||||
assert store.stored_model_name() is None
|
||||
|
||||
def test_model_name_stored_after_add_and_persists(self, tmp_path: Path) -> None:
|
||||
with PaperlessSqliteVecVectorStore(
|
||||
uri=str(tmp_path),
|
||||
embed_model_name="model-a",
|
||||
) as store:
|
||||
store.add([make_node("a1", "1")])
|
||||
assert store.stored_model_name() == "model-a"
|
||||
with PaperlessSqliteVecVectorStore(uri=str(tmp_path)) as reopened:
|
||||
assert reopened.stored_model_name() == "model-a"
|
||||
|
||||
def test_config_mismatch_semantics(self, tmp_path: Path) -> None:
|
||||
with PaperlessSqliteVecVectorStore(
|
||||
uri=str(tmp_path),
|
||||
embed_model_name="model-a",
|
||||
) as store:
|
||||
assert not store.config_mismatch("anything") # no table yet
|
||||
store.add([make_node("a1", "1")])
|
||||
assert not store.config_mismatch("model-a")
|
||||
assert store.config_mismatch("model-b")
|
||||
|
||||
def test_config_mismatch_false_when_table_predates_tracking(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
uri = str(tmp_path / "idx")
|
||||
PaperlessLanceVectorStore(uri=uri).add(
|
||||
[_node("1-0", "1", "a", 0.1), _node("2-0", "2", "b", 0.1)],
|
||||
)
|
||||
|
||||
reopened = PaperlessLanceVectorStore(uri=uri)
|
||||
result = reopened.query(
|
||||
VectorStoreQuery(
|
||||
query_embedding=[0.1] * DIM,
|
||||
similarity_top_k=5,
|
||||
filters=MetadataFilters(
|
||||
filters=[
|
||||
MetadataFilter(
|
||||
key="document_id",
|
||||
operator=FilterOperator.IN,
|
||||
value=["1"],
|
||||
),
|
||||
],
|
||||
),
|
||||
),
|
||||
)
|
||||
assert [n.metadata["document_id"] for n in result.nodes] == ["1"]
|
||||
|
||||
def test_table_exists_and_drop(
|
||||
self,
|
||||
store: PaperlessLanceVectorStore,
|
||||
) -> None:
|
||||
assert store.table_exists() is False
|
||||
store.add([_node("1-0", "1", "a", 0.1)])
|
||||
assert store.table_exists() is True
|
||||
assert store.vector_dim() == DIM
|
||||
store.drop_table()
|
||||
assert store.table_exists() is False
|
||||
|
||||
def test_build_where_or_condition(self) -> None:
|
||||
from llama_index.core.vector_stores.types import FilterCondition
|
||||
|
||||
from paperless_ai.vector_store import _build_where
|
||||
|
||||
where = _build_where(
|
||||
MetadataFilters(
|
||||
filters=[
|
||||
MetadataFilter(
|
||||
key="document_id",
|
||||
operator=FilterOperator.EQ,
|
||||
value="1",
|
||||
),
|
||||
MetadataFilter(
|
||||
key="document_id",
|
||||
operator=FilterOperator.EQ,
|
||||
value="2",
|
||||
),
|
||||
],
|
||||
condition=FilterCondition.OR,
|
||||
),
|
||||
)
|
||||
assert where == "document_id = '1' OR document_id = '2'"
|
||||
|
||||
|
||||
class TestPaperlessLanceVectorStoreUpsert:
|
||||
@pytest.fixture
|
||||
def store(self, tmp_path: Path) -> PaperlessLanceVectorStore:
|
||||
s = PaperlessLanceVectorStore(uri=str(tmp_path / "idx"))
|
||||
s.add(
|
||||
[
|
||||
_node("1-0", "1", "old0", 0.1),
|
||||
_node("1-1", "1", "old1", 0.2),
|
||||
_node("1-2", "1", "old2", 0.3),
|
||||
_node("2-0", "2", "keep", 0.9),
|
||||
],
|
||||
)
|
||||
return s
|
||||
|
||||
def test_upsert_prunes_stale_chunks_and_keeps_others(
|
||||
self,
|
||||
store: PaperlessLanceVectorStore,
|
||||
) -> None:
|
||||
store.upsert_document(
|
||||
"1",
|
||||
[_node("1-0", "1", "new0", 0.1), _node("1-1", "1", "new1", 0.2)],
|
||||
)
|
||||
|
||||
table = store.client.open_table("documents")
|
||||
doc1 = sorted(
|
||||
r["id"] for r in table.search().where("document_id = '1'").to_list()
|
||||
)
|
||||
assert doc1 == ["1-0", "1-1"] # 1-2 pruned
|
||||
assert table.count_rows() == 3 # 2 new doc1 + 1 doc2
|
||||
|
||||
def test_upsert_is_single_commit(
|
||||
self,
|
||||
store: PaperlessLanceVectorStore,
|
||||
) -> None:
|
||||
table = store.client.open_table("documents")
|
||||
before = table.version
|
||||
store.upsert_document("1", [_node("1-0", "1", "new0", 0.1)])
|
||||
assert store.client.open_table("documents").version == before + 1
|
||||
|
||||
def test_upsert_empty_nodes_removes_document(
|
||||
self,
|
||||
store: PaperlessLanceVectorStore,
|
||||
) -> None:
|
||||
store.upsert_document("1", [])
|
||||
|
||||
table = store.client.open_table("documents")
|
||||
remaining = sorted(r["document_id"] for r in table.search().to_list())
|
||||
assert "1" not in remaining
|
||||
assert "2" in remaining
|
||||
|
||||
|
||||
class TestPaperlessLanceVectorStoreMaintenance:
|
||||
@pytest.fixture
|
||||
def store(self, tmp_path: Path) -> PaperlessLanceVectorStore:
|
||||
return PaperlessLanceVectorStore(uri=str(tmp_path / "idx"))
|
||||
|
||||
def test_maybe_create_ann_index_noop_below_threshold(
|
||||
self,
|
||||
store: PaperlessLanceVectorStore,
|
||||
) -> None:
|
||||
store.add([_node("1-0", "1", "a", 0.1)])
|
||||
# Threshold far above row count -> no index attempted, no error.
|
||||
store.maybe_create_ann_index(min_rows=1000)
|
||||
# Still queryable.
|
||||
result = store.query(
|
||||
VectorStoreQuery(query_embedding=[0.1] * DIM, similarity_top_k=1),
|
||||
)
|
||||
assert len(result.nodes) == 1
|
||||
|
||||
def test_maybe_create_ann_index_non_divisible_dim_falls_back(
|
||||
self,
|
||||
store: PaperlessLanceVectorStore,
|
||||
) -> None:
|
||||
# DIM=8 is not divisible by the PQ default sub-vectors; must not raise
|
||||
# and must leave the table queryable (IVF_FLAT fallback or skipped).
|
||||
for i in range(40):
|
||||
store.add([_node(f"1-{i}", "1", f"t{i}", float(i))])
|
||||
store.maybe_create_ann_index(min_rows=10)
|
||||
result = store.query(
|
||||
VectorStoreQuery(query_embedding=[1.0] * DIM, similarity_top_k=3),
|
||||
)
|
||||
assert len(result.nodes) == 3
|
||||
|
||||
def test_compact_reduces_to_single_version(
|
||||
self,
|
||||
store: PaperlessLanceVectorStore,
|
||||
) -> None:
|
||||
for i in range(5):
|
||||
store.add([_node(f"1-{i}", "1", f"t{i}", float(i))])
|
||||
assert len(store.client.open_table("documents").list_versions()) > 1
|
||||
store.compact(retention_seconds=0)
|
||||
assert len(store.client.open_table("documents").list_versions()) == 1
|
||||
|
||||
def test_upsert_after_optimize_with_scalar_index(
|
||||
self,
|
||||
store: PaperlessLanceVectorStore,
|
||||
) -> None:
|
||||
store.add(
|
||||
[
|
||||
_node("1-0", "1", "old0", 0.1),
|
||||
_node("1-1", "1", "old1", 0.2),
|
||||
_node("1-2", "1", "old2", 0.3),
|
||||
_node("2-0", "2", "keep", 0.9),
|
||||
],
|
||||
)
|
||||
store.ensure_document_id_scalar_index()
|
||||
store.compact(retention_seconds=0)
|
||||
|
||||
store.upsert_document("1", [_node("1-0", "1", "new0", 0.1)])
|
||||
|
||||
table = store.client.open_table("documents")
|
||||
doc1 = sorted(
|
||||
r["id"] for r in table.search().where("document_id = '1'").to_list()
|
||||
)
|
||||
assert doc1 == ["1-0"]
|
||||
assert table.count_rows() == 2
|
||||
|
||||
def test_ensure_scalar_index_is_idempotent(
|
||||
self,
|
||||
store: PaperlessLanceVectorStore,
|
||||
) -> None:
|
||||
store.add([_node("1-0", "1", "text", 0.5)])
|
||||
store.ensure_document_id_scalar_index()
|
||||
# Second call must not raise and must not replace the existing index.
|
||||
store.ensure_document_id_scalar_index()
|
||||
assert store._has_index_on("document_id")
|
||||
|
||||
def test_ensure_scalar_index_noop_on_empty_store(
|
||||
self,
|
||||
store: PaperlessLanceVectorStore,
|
||||
) -> None:
|
||||
store.ensure_document_id_scalar_index() # no table yet — must not raise
|
||||
|
||||
|
||||
class TestConfigMismatch:
|
||||
@pytest.fixture
|
||||
def uri(self, tmp_path: Path) -> str:
|
||||
return str(tmp_path / "idx")
|
||||
|
||||
def test_stored_model_name_returns_none_when_no_table(self, uri: str) -> None:
|
||||
store = PaperlessLanceVectorStore(uri=uri)
|
||||
assert store.stored_model_name() is None
|
||||
|
||||
def test_model_name_stored_in_schema_after_add(self, uri: str) -> None:
|
||||
store = PaperlessLanceVectorStore(uri=uri, embed_model_name="all-MiniLM-L6-v2")
|
||||
store.add([_node("1-0", "1", "text", 0.1)])
|
||||
assert store.stored_model_name() == "all-MiniLM-L6-v2"
|
||||
|
||||
def test_model_name_stored_in_schema_after_upsert(self, uri: str) -> None:
|
||||
store = PaperlessLanceVectorStore(uri=uri, embed_model_name="nomic-embed")
|
||||
store.upsert_document("1", [_node("1-0", "1", "text", 0.1)])
|
||||
assert store.stored_model_name() == "nomic-embed"
|
||||
|
||||
def test_model_name_persists_after_reopen(self, uri: str) -> None:
|
||||
PaperlessLanceVectorStore(uri=uri, embed_model_name="all-MiniLM-L6-v2").add(
|
||||
[_node("1-0", "1", "text", 0.1)],
|
||||
)
|
||||
reopened = PaperlessLanceVectorStore(uri=uri)
|
||||
assert reopened.stored_model_name() == "all-MiniLM-L6-v2"
|
||||
|
||||
def test_config_mismatch_returns_false_when_no_table(self, uri: str) -> None:
|
||||
store = PaperlessLanceVectorStore(uri=uri)
|
||||
assert store.config_mismatch("any-model") is False
|
||||
|
||||
def test_config_mismatch_returns_false_when_model_matches(self, uri: str) -> None:
|
||||
store = PaperlessLanceVectorStore(uri=uri, embed_model_name="all-MiniLM-L6-v2")
|
||||
store.add([_node("1-0", "1", "text", 0.1)])
|
||||
assert store.config_mismatch("all-MiniLM-L6-v2") is False
|
||||
|
||||
def test_config_mismatch_returns_true_when_model_differs(self, uri: str) -> None:
|
||||
store = PaperlessLanceVectorStore(uri=uri, embed_model_name="old-model")
|
||||
store.add([_node("1-0", "1", "text", 0.1)])
|
||||
assert store.config_mismatch("new-model") is True
|
||||
|
||||
def test_config_mismatch_returns_false_when_no_metadata_stored(
|
||||
self,
|
||||
uri: str,
|
||||
) -> None:
|
||||
# Tables created before model-name tracking was added have no schema metadata.
|
||||
# Conservative default: assume compatible rather than force a rebuild.
|
||||
store = PaperlessLanceVectorStore(uri=uri)
|
||||
store.add([_node("1-0", "1", "text", 0.1)])
|
||||
assert store.config_mismatch("any-model") is False
|
||||
with PaperlessSqliteVecVectorStore(uri=str(tmp_path)) as store: # no model name
|
||||
store.add([make_node("a1", "1")])
|
||||
assert not store.config_mismatch("model-a")
|
||||
|
||||
|
||||
class TestGetModifiedTimes:
|
||||
@pytest.fixture
|
||||
def store(self, tmp_path: Path) -> PaperlessLanceVectorStore:
|
||||
return PaperlessLanceVectorStore(uri=str(tmp_path / "idx"))
|
||||
|
||||
def _node_with_modified(
|
||||
self,
|
||||
node_id: str,
|
||||
doc_id: str,
|
||||
modified: str,
|
||||
) -> TextNode:
|
||||
node = TextNode(
|
||||
id_=node_id,
|
||||
text="text",
|
||||
metadata={"document_id": doc_id, "modified": modified},
|
||||
)
|
||||
node.embedding = [0.1] * DIM
|
||||
node.relationships = {
|
||||
NodeRelationship.SOURCE: RelatedNodeInfo(node_id=doc_id),
|
||||
}
|
||||
return node
|
||||
|
||||
def test_empty_store_returns_empty_dict(
|
||||
self,
|
||||
store: PaperlessLanceVectorStore,
|
||||
) -> None:
|
||||
def test_empty_store_returns_empty_dict(self, store) -> None:
|
||||
assert store.get_modified_times() == {}
|
||||
|
||||
def test_returns_one_entry_per_document(
|
||||
self,
|
||||
store: PaperlessLanceVectorStore,
|
||||
) -> None:
|
||||
def test_returns_one_entry_per_document(self, store) -> None:
|
||||
store.add(
|
||||
[
|
||||
self._node_with_modified("1-0", "1", "2024-01-01T00:00:00"),
|
||||
self._node_with_modified("1-1", "1", "2024-01-01T00:00:00"),
|
||||
self._node_with_modified("2-0", "2", "2024-06-01T00:00:00"),
|
||||
make_node("a1", "1", modified="2026-01-01T00:00:00"),
|
||||
make_node("a2", "1", modified="2026-01-01T00:00:00"),
|
||||
make_node("b1", "2", modified="2026-02-02T00:00:00"),
|
||||
],
|
||||
)
|
||||
result = store.get_modified_times()
|
||||
assert result == {
|
||||
"1": "2024-01-01T00:00:00",
|
||||
"2": "2024-06-01T00:00:00",
|
||||
assert store.get_modified_times() == {
|
||||
"1": "2026-01-01T00:00:00",
|
||||
"2": "2026-02-02T00:00:00",
|
||||
}
|
||||
|
||||
|
||||
class TestCompact:
|
||||
def _bloat_ratio(self, store) -> float:
|
||||
live = store.client.execute(
|
||||
"SELECT count(*) FROM documents",
|
||||
).fetchone()[0]
|
||||
# vec0 0.1.9 does not accumulate deleted rows in the _rowids shadow
|
||||
# table, so we track cumulative inserts in index_meta instead.
|
||||
row = store.client.execute(
|
||||
"SELECT value FROM index_meta WHERE key = 'total_inserts'",
|
||||
).fetchone()
|
||||
total = int(row["value"]) if row else live
|
||||
return total / max(live, 1)
|
||||
|
||||
def _churn(self, store, cycles: int) -> None:
|
||||
for i in range(cycles):
|
||||
store.upsert_document(
|
||||
"1",
|
||||
[make_node(f"gen{i}-{j}", "1", seed=float(j)) for j in range(20)],
|
||||
)
|
||||
|
||||
def test_compact_noop_below_threshold(self, store) -> None:
|
||||
store.add([make_node("a1", "1")])
|
||||
store.compact()
|
||||
assert _query(store, [0.0] * DIM, top_k=1).ids == ["a1"]
|
||||
|
||||
def test_force_compact_preserves_rows_and_metadata(self, store) -> None:
|
||||
store.add([make_node("a1", "1"), make_node("b1", "2", seed=3.0)])
|
||||
self._churn(store, 5)
|
||||
before = {
|
||||
n.node_id: n.metadata
|
||||
for n in store.get_nodes(filters=_in_filter(["1", "2"]))
|
||||
}
|
||||
store.compact(force=True)
|
||||
after = {
|
||||
n.node_id: n.metadata
|
||||
for n in store.get_nodes(filters=_in_filter(["1", "2"]))
|
||||
}
|
||||
assert after == before
|
||||
assert self._bloat_ratio(store) == pytest.approx(1.0)
|
||||
# store remains fully usable after the rebuild; use a seed far from all
|
||||
# existing nodes (gen4-0..gen4-19 have seeds 0..19) so cosine KNN is
|
||||
# unambiguous at top_k=1.
|
||||
store.upsert_document("3", [make_node("c1", "3", seed=100.0)])
|
||||
assert "c1" in _query(store, [100.0] * DIM, top_k=1).ids
|
||||
|
||||
def test_auto_compact_triggers_on_churn(self, store) -> None:
|
||||
store.add([make_node(f"s{j}", "1", seed=float(j)) for j in range(20)])
|
||||
self._churn(store, 5)
|
||||
assert self._bloat_ratio(store) > 2
|
||||
store.compact()
|
||||
assert self._bloat_ratio(store) == pytest.approx(1.0)
|
||||
|
||||
def test_compact_on_missing_table_is_noop(self, store) -> None:
|
||||
store.compact()
|
||||
store.compact(force=True)
|
||||
|
||||
def test_failed_compact_removes_temp_wal_and_shm(
|
||||
self,
|
||||
store,
|
||||
tmp_path: Path,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
"""A compact() that raises mid-rebuild must leave no .compact* files.
|
||||
|
||||
Normally the sole connection's close() checkpoints the temp WAL away,
|
||||
but a concurrent reader keeps -wal/-shm alive, so the cleanup must
|
||||
unlink them explicitly (as the structural-migration path does).
|
||||
"""
|
||||
store.add([make_node("a1", "1")])
|
||||
compact_path = str(tmp_path / DB_FILENAME) + ".compact"
|
||||
held: list[sqlite3.Connection] = []
|
||||
|
||||
def boom(conn: sqlite3.Connection, dim: int) -> None:
|
||||
# Hold an extra connection so close() of the rebuild connection is
|
||||
# not the last one -> the temp -wal/-shm survive the checkpoint.
|
||||
extra = sqlite3.connect(compact_path)
|
||||
extra.execute("SELECT 1").fetchall()
|
||||
held.append(extra)
|
||||
raise RuntimeError("boom")
|
||||
|
||||
monkeypatch.setattr(
|
||||
PaperlessSqliteVecVectorStore,
|
||||
"_create_vec_table",
|
||||
staticmethod(boom),
|
||||
)
|
||||
try:
|
||||
with pytest.raises(RuntimeError):
|
||||
store.compact(force=True)
|
||||
assert sorted(p.name for p in tmp_path.glob("*.compact*")) == []
|
||||
finally:
|
||||
for c in held:
|
||||
c.close()
|
||||
|
||||
def test_force_compact_streams_rows_across_batches(
|
||||
self,
|
||||
store,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
"""Rebuild must preserve every row when rows span multiple batches.
|
||||
|
||||
A tiny batch size forces several fetchmany()/executemany() cycles so a
|
||||
regression in the streaming loop (dropped tail, off-by-one) surfaces.
|
||||
"""
|
||||
monkeypatch.setattr("paperless_ai.vector_store.COMPACT_BATCH_SIZE", 3)
|
||||
store.add([make_node(f"n{i}", "1", seed=float(i)) for i in range(10)])
|
||||
store.compact(force=True)
|
||||
ids = {n.node_id for n in store.get_nodes(filters=_in_filter(["1"]))}
|
||||
assert ids == {f"n{i}" for i in range(10)}
|
||||
assert self._bloat_ratio(store) == pytest.approx(1.0)
|
||||
|
||||
|
||||
class TestDbFile:
|
||||
def test_single_db_file_in_index_dir(self, store, tmp_path: Path) -> None:
|
||||
store.add([make_node("a1", "1")])
|
||||
assert (tmp_path / DB_FILENAME).exists()
|
||||
|
||||
def test_wal_mode_enabled(self, store) -> None:
|
||||
assert (
|
||||
store.client.execute("PRAGMA journal_mode").fetchone()[0].lower() == "wal"
|
||||
)
|
||||
|
||||
|
||||
class TestMigrations:
|
||||
"""Tests for the schema migration machinery."""
|
||||
|
||||
def _schema_version(self, store: PaperlessSqliteVecVectorStore) -> int | None:
|
||||
row = store.client.execute(
|
||||
"SELECT value FROM index_meta WHERE key = 'schema_version'",
|
||||
).fetchone()
|
||||
return int(row[0]) if row else None
|
||||
|
||||
def test_new_table_records_schema_version(self, store) -> None:
|
||||
store.add([make_node("a1", "1")])
|
||||
assert self._schema_version(store) == SCHEMA_VERSION
|
||||
|
||||
def test_check_migrations_no_table_returns_false(self, store) -> None:
|
||||
assert store.check_and_run_migrations() is False
|
||||
|
||||
def test_check_migrations_current_version_returns_false(self, store) -> None:
|
||||
store.add([make_node("a1", "1")])
|
||||
assert store.check_and_run_migrations() is False
|
||||
|
||||
def test_reembed_migration_returns_true(self, store, tmp_path: Path) -> None:
|
||||
store.add([make_node("a1", "1")])
|
||||
migration = Migration(
|
||||
from_version=1,
|
||||
to_version=2,
|
||||
kind="re-embed",
|
||||
description="test re-embed",
|
||||
)
|
||||
MIGRATIONS.append(migration)
|
||||
try:
|
||||
from paperless_ai import vector_store as vs_mod
|
||||
|
||||
original = vs_mod.SCHEMA_VERSION
|
||||
vs_mod.SCHEMA_VERSION = 2
|
||||
result = store.check_and_run_migrations()
|
||||
finally:
|
||||
MIGRATIONS.remove(migration)
|
||||
vs_mod.SCHEMA_VERSION = original
|
||||
assert result is True
|
||||
|
||||
def test_structural_migration_copies_rows_and_updates_version(
|
||||
self,
|
||||
store,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
store.add([make_node("a1", "1"), make_node("b1", "2")])
|
||||
|
||||
def apply(
|
||||
src: sqlite3.Connection,
|
||||
dst: sqlite3.Connection,
|
||||
dim: int,
|
||||
) -> None:
|
||||
dst.execute( # nosemgrep
|
||||
f"CREATE VIRTUAL TABLE {DEFAULT_TABLE_NAME} USING vec0("
|
||||
"id TEXT PRIMARY KEY, document_id TEXT, modified TEXT,"
|
||||
f" +node_content TEXT, embedding float[{dim}] distance_metric=cosine"
|
||||
")",
|
||||
)
|
||||
dst.execute(
|
||||
"INSERT INTO index_meta (key, value) VALUES ('dim', ?) "
|
||||
"ON CONFLICT(key) DO UPDATE SET value = excluded.value",
|
||||
(str(dim),),
|
||||
)
|
||||
rows = src.execute(
|
||||
"SELECT id, document_id, modified, node_content, embedding "
|
||||
f"FROM {DEFAULT_TABLE_NAME}",
|
||||
).fetchall()
|
||||
dst.execute("BEGIN IMMEDIATE")
|
||||
dst.executemany(
|
||||
f"INSERT INTO {DEFAULT_TABLE_NAME} "
|
||||
"(id, document_id, modified, node_content, embedding) "
|
||||
"VALUES (?, ?, ?, ?, ?)",
|
||||
[
|
||||
(
|
||||
r["id"],
|
||||
r["document_id"],
|
||||
r["modified"],
|
||||
r["node_content"],
|
||||
bytes(r["embedding"]),
|
||||
)
|
||||
for r in rows
|
||||
],
|
||||
)
|
||||
dst.execute(
|
||||
"INSERT INTO index_meta (key, value) VALUES ('total_inserts', ?) "
|
||||
"ON CONFLICT(key) DO UPDATE SET value = excluded.value",
|
||||
(str(len(rows)),),
|
||||
)
|
||||
dst.execute("COMMIT")
|
||||
|
||||
migration = Migration(
|
||||
from_version=1,
|
||||
to_version=2,
|
||||
kind="structural",
|
||||
description="test structural",
|
||||
apply=apply,
|
||||
)
|
||||
MIGRATIONS.append(migration)
|
||||
try:
|
||||
from paperless_ai import vector_store as vs_mod
|
||||
|
||||
original = vs_mod.SCHEMA_VERSION
|
||||
vs_mod.SCHEMA_VERSION = 2
|
||||
result = store.check_and_run_migrations()
|
||||
finally:
|
||||
MIGRATIONS.remove(migration)
|
||||
vs_mod.SCHEMA_VERSION = original
|
||||
|
||||
assert result is False
|
||||
assert self._schema_version(store) == 2
|
||||
ids = {n.node_id for n in store.get_nodes()}
|
||||
assert ids == {"a1", "b1"}
|
||||
|
||||
def test_compact_preserves_schema_version(self, store) -> None:
|
||||
store.add([make_node("a1", "1")])
|
||||
assert self._schema_version(store) == SCHEMA_VERSION
|
||||
store.compact(force=True)
|
||||
assert self._schema_version(store) == SCHEMA_VERSION
|
||||
|
||||
def test_stop_at_reembed_boundary(self, store) -> None:
|
||||
# Registry: structural v2, re-embed v3, structural v4.
|
||||
# Only v2 should apply; the re-embed boundary must stop execution
|
||||
# before v4 runs, and the stored version must stay at 2.
|
||||
store.add([make_node("a1", "1"), make_node("b1", "2")])
|
||||
|
||||
def copy_apply(
|
||||
src: sqlite3.Connection,
|
||||
dst: sqlite3.Connection,
|
||||
dim: int,
|
||||
) -> None:
|
||||
dst.execute( # nosemgrep
|
||||
f"CREATE VIRTUAL TABLE {DEFAULT_TABLE_NAME} USING vec0("
|
||||
"id TEXT PRIMARY KEY, document_id TEXT, modified TEXT,"
|
||||
f" +node_content TEXT, embedding float[{dim}] distance_metric=cosine"
|
||||
")",
|
||||
)
|
||||
dst.execute(
|
||||
"INSERT INTO index_meta (key, value) VALUES ('dim', ?) "
|
||||
"ON CONFLICT(key) DO UPDATE SET value = excluded.value",
|
||||
(str(dim),),
|
||||
)
|
||||
rows = src.execute(
|
||||
"SELECT id, document_id, modified, node_content, embedding "
|
||||
f"FROM {DEFAULT_TABLE_NAME}",
|
||||
).fetchall()
|
||||
dst.execute("BEGIN IMMEDIATE")
|
||||
dst.executemany(
|
||||
f"INSERT INTO {DEFAULT_TABLE_NAME} "
|
||||
"(id, document_id, modified, node_content, embedding) "
|
||||
"VALUES (?, ?, ?, ?, ?)",
|
||||
[
|
||||
(
|
||||
r["id"],
|
||||
r["document_id"],
|
||||
r["modified"],
|
||||
r["node_content"],
|
||||
bytes(r["embedding"]),
|
||||
)
|
||||
for r in rows
|
||||
],
|
||||
)
|
||||
dst.execute("COMMIT")
|
||||
|
||||
migrations = [
|
||||
Migration(
|
||||
from_version=1,
|
||||
to_version=2,
|
||||
kind="structural",
|
||||
description="v2 structural",
|
||||
apply=copy_apply,
|
||||
),
|
||||
Migration(
|
||||
from_version=2,
|
||||
to_version=3,
|
||||
kind="re-embed",
|
||||
description="v3 re-embed boundary",
|
||||
),
|
||||
Migration(
|
||||
from_version=3,
|
||||
to_version=4,
|
||||
kind="structural",
|
||||
description="v4 structural - must not run",
|
||||
apply=copy_apply,
|
||||
),
|
||||
]
|
||||
MIGRATIONS.extend(migrations)
|
||||
try:
|
||||
from paperless_ai import vector_store as vs_mod
|
||||
|
||||
original = vs_mod.SCHEMA_VERSION
|
||||
vs_mod.SCHEMA_VERSION = 4
|
||||
result = store.check_and_run_migrations()
|
||||
finally:
|
||||
for m in migrations:
|
||||
MIGRATIONS.remove(m)
|
||||
vs_mod.SCHEMA_VERSION = original
|
||||
|
||||
assert result is True
|
||||
assert self._schema_version(store) == 2
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user