mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-10-02 06:10:31 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0535aec0c6 | ||
|
|
f1ae9c00ac | ||
|
|
bc2ff953ed | ||
|
|
e17689a91b | ||
|
|
764d1a492d | ||
|
|
9d50a3b261 | ||
|
|
da3de299fc | ||
|
|
3a022e2dc4 | ||
|
|
58bda48682 | ||
|
|
b2a2e15ead | ||
|
|
e865185425 | ||
|
|
d02c19a09e | ||
|
|
4a106454ce | ||
|
|
4254100fae | ||
|
|
74f727b1f3 | ||
|
|
fb6fad952e | ||
|
|
f9b4e0c8e0 | ||
|
|
e5041097af | ||
|
|
61f36eb253 | ||
|
|
e50bcc2efb | ||
|
|
88000aab45 | ||
|
|
01b4ab392c | ||
|
|
52aa8758e3 | ||
|
|
4b7cc1c0c3 | ||
|
|
f02d7455be | ||
|
|
a61c35fcd8 | ||
|
|
64552d5d5b | ||
|
|
a59edf3e5b | ||
|
|
6a5d06ee26 | ||
|
|
126ec414a8 | ||
|
|
773744d04b | ||
|
|
7b98085b31 | ||
|
|
b3bf220f56 | ||
|
|
d4429c3cc7 | ||
|
|
4f777d7438 | ||
|
|
2a44d8b5ba | ||
|
|
abf5050ea7 | ||
|
|
091ddf7c45 | ||
|
|
c9f7f2cfbe | ||
|
|
b457610ffb | ||
|
|
969c2ea0e2 | ||
|
|
31b806a285 | ||
|
|
99851b418c | ||
|
|
e34eda07bb | ||
|
|
793459b416 | ||
|
|
04297fd02c | ||
|
|
1b277dd8e1 | ||
|
|
3c20abeb4c | ||
|
|
b11f1f8459 | ||
|
|
7424e7ce0b | ||
|
|
a53a3d3769 | ||
|
|
15b73b890c | ||
|
|
02d355061f | ||
|
|
d8b5b4d447 | ||
|
|
03ac4aed7e | ||
|
|
90d23bad9c | ||
|
|
e4367b5648 | ||
|
|
cb85441c2f | ||
|
|
cceaa559d4 | ||
|
|
a748d4c64f | ||
|
|
452ed005bd | ||
|
|
40058ff7d5 | ||
|
|
f502cd5e34 | ||
|
|
99ce6b5db3 | ||
|
|
2d955e9697 | ||
|
|
f440e8d33c | ||
|
|
4c264651e8 | ||
|
|
48d97b78bb | ||
|
|
659a0cb2ef | ||
|
|
d02d1e1711 | ||
|
|
3b41810e7b | ||
|
|
12314fcaa8 |
@@ -15,6 +15,8 @@
|
||||
# Test related
|
||||
**/.pytest_cache
|
||||
**/tests
|
||||
src/paperless_testing
|
||||
src/conftest.py
|
||||
**/*.spec.ts
|
||||
**/htmlcov
|
||||
# Local folders
|
||||
|
||||
@@ -59,7 +59,6 @@ updates:
|
||||
- "drf-*"
|
||||
- "djangorestframework"
|
||||
- "whitenoise"
|
||||
- "bleach"
|
||||
- "jinja2"
|
||||
# Async, Task Queuing & Caching
|
||||
async-tasks:
|
||||
|
||||
@@ -102,7 +102,7 @@ jobs:
|
||||
with:
|
||||
python-version: "${{ matrix.python-version }}"
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
|
||||
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
|
||||
with:
|
||||
version: ${{ env.DEFAULT_UV_VERSION }}
|
||||
enable-cache: true
|
||||
@@ -139,13 +139,13 @@ jobs:
|
||||
pytest
|
||||
- name: Upload test results to Codecov
|
||||
if: always()
|
||||
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
|
||||
uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1
|
||||
with:
|
||||
flags: backend-python-${{ matrix.python-version }}
|
||||
files: junit.xml
|
||||
report_type: test_results
|
||||
- name: Upload coverage to Codecov
|
||||
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
|
||||
uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1
|
||||
with:
|
||||
flags: backend-python-${{ matrix.python-version }}
|
||||
files: coverage.xml
|
||||
@@ -176,7 +176,7 @@ jobs:
|
||||
with:
|
||||
python-version: "${{ env.DEFAULT_PYTHON }}"
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
|
||||
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
|
||||
with:
|
||||
version: ${{ env.DEFAULT_UV_VERSION }}
|
||||
enable-cache: true
|
||||
|
||||
@@ -106,7 +106,7 @@ jobs:
|
||||
echo "repository=${repo_name}"
|
||||
echo "name=${repo_name}" >> $GITHUB_OUTPUT
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0
|
||||
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
|
||||
with:
|
||||
@@ -132,7 +132,7 @@ jobs:
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
- name: Build and push by digest
|
||||
id: build
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile
|
||||
@@ -182,7 +182,7 @@ jobs:
|
||||
echo "Downloaded digests:"
|
||||
ls -la /tmp/digests/
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0
|
||||
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
|
||||
with:
|
||||
|
||||
@@ -78,7 +78,7 @@ jobs:
|
||||
with:
|
||||
python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
|
||||
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
|
||||
with:
|
||||
version: ${{ env.DEFAULT_UV_VERSION }}
|
||||
enable-cache: true
|
||||
|
||||
@@ -174,13 +174,13 @@ jobs:
|
||||
run: cd src-ui && pnpm run test --max-workers=2 --shard=${{ matrix.shard-index }}/${{ matrix.shard-count }}
|
||||
- name: Upload test results to Codecov
|
||||
if: always()
|
||||
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
|
||||
uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1
|
||||
with:
|
||||
flags: frontend-node-${{ matrix.node-version }}
|
||||
directory: src-ui/
|
||||
report_type: test_results
|
||||
- name: Upload coverage to Codecov
|
||||
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
|
||||
uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1
|
||||
with:
|
||||
flags: frontend-node-${{ matrix.node-version }}
|
||||
directory: src-ui/coverage/
|
||||
@@ -216,7 +216,7 @@ jobs:
|
||||
with:
|
||||
python-version: '3.12'
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
|
||||
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
|
||||
with:
|
||||
version: '0.12.x'
|
||||
enable-cache: false
|
||||
|
||||
@@ -59,7 +59,7 @@ jobs:
|
||||
with:
|
||||
python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
|
||||
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
|
||||
with:
|
||||
version: ${{ env.DEFAULT_UV_VERSION }}
|
||||
enable-cache: false
|
||||
@@ -212,7 +212,7 @@ jobs:
|
||||
with:
|
||||
python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
|
||||
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
|
||||
with:
|
||||
version: ${{ env.DEFAULT_UV_VERSION }}
|
||||
enable-cache: false
|
||||
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
- name: Run Semgrep
|
||||
run: semgrep scan --config auto --sarif-output results.sarif
|
||||
- name: Upload results to GitHub code scanning
|
||||
uses: github/codeql-action/upload-sarif@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4.38.0
|
||||
uses: github/codeql-action/upload-sarif@1c5b675653bb5c22dbe9b12b556ec555138e09fd # v4.38.1
|
||||
if: always()
|
||||
with:
|
||||
sarif_file: results.sarif
|
||||
|
||||
@@ -29,7 +29,7 @@ jobs:
|
||||
steps:
|
||||
- name: Clean temporary images
|
||||
if: "${{ env.TOKEN != '' }}"
|
||||
uses: stumpylog/image-cleaner-action/ephemeral@4fe057d991d63b8f6d5d22c40f17c1bca2226537 # v0.12.0
|
||||
uses: stumpylog/image-cleaner-action/ephemeral@21f875bab2376314e0525c614e923c669434f0e4 # v0.13.0
|
||||
with:
|
||||
token: "${{ env.TOKEN }}"
|
||||
owner: "${{ github.repository_owner }}"
|
||||
@@ -56,7 +56,7 @@ jobs:
|
||||
steps:
|
||||
- name: Clean untagged images
|
||||
if: "${{ env.TOKEN != '' }}"
|
||||
uses: stumpylog/image-cleaner-action/untagged@4fe057d991d63b8f6d5d22c40f17c1bca2226537 # v0.12.0
|
||||
uses: stumpylog/image-cleaner-action/untagged@21f875bab2376314e0525c614e923c669434f0e4 # v0.13.0
|
||||
with:
|
||||
token: "${{ env.TOKEN }}"
|
||||
owner: "${{ github.repository_owner }}"
|
||||
|
||||
@@ -39,7 +39,7 @@ jobs:
|
||||
persist-credentials: false
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4.38.0
|
||||
uses: github/codeql-action/init@1c5b675653bb5c22dbe9b12b556ec555138e09fd # v4.38.1
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
# If you wish to specify custom queries, you can do so here or in a config file.
|
||||
@@ -47,4 +47,4 @@ jobs:
|
||||
# Prefix the list here with "+" to use these queries and those in the config file.
|
||||
# queries: ./path/to/local/query, your-org/your-repo/queries@main
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4.38.0
|
||||
uses: github/codeql-action/analyze@1c5b675653bb5c22dbe9b12b556ec555138e09fd # v4.38.1
|
||||
|
||||
@@ -22,7 +22,7 @@ jobs:
|
||||
token: ${{ secrets.PNGX_BOT_PAT }}
|
||||
persist-credentials: false
|
||||
- name: crowdin action
|
||||
uses: crowdin/github-action@0d5670f539973aea2f01abce61a8989934df0025 # v3.0.2
|
||||
uses: crowdin/github-action@df474cdfb9f41d6ae777118749477c2cdf7cacc8 # v3.2.0
|
||||
with:
|
||||
upload_translations: false
|
||||
download_translations: true
|
||||
|
||||
@@ -29,7 +29,7 @@ jobs:
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -qq --no-install-recommends gettext
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
|
||||
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
|
||||
with:
|
||||
version: ${{ env.DEFAULT_UV_VERSION }}
|
||||
enable-cache: true
|
||||
|
||||
+706
-456
File diff suppressed because it is too large
Load Diff
@@ -38,7 +38,7 @@ repos:
|
||||
- json
|
||||
# See https://github.com/prettier/prettier/issues/15742 for the fork reason
|
||||
- repo: https://github.com/rbubley/mirrors-prettier
|
||||
rev: 'v3.9.6'
|
||||
rev: 'v3.9.8'
|
||||
hooks:
|
||||
- id: prettier
|
||||
types_or:
|
||||
@@ -46,16 +46,16 @@ repos:
|
||||
- ts
|
||||
- markdown
|
||||
additional_dependencies:
|
||||
- prettier@3.9.6
|
||||
- prettier@3.9.8
|
||||
- 'prettier-plugin-organize-imports@4.3.0'
|
||||
# Python hooks
|
||||
- repo: https://github.com/astral-sh/ruff-pre-commit
|
||||
rev: v0.16.5
|
||||
rev: v0.16.8
|
||||
hooks:
|
||||
- id: ruff-check
|
||||
- id: ruff-format
|
||||
- repo: https://github.com/tox-dev/pyproject-fmt
|
||||
rev: "v2.28.1"
|
||||
rev: "v2.29.4"
|
||||
hooks:
|
||||
- id: pyproject-fmt
|
||||
additional_dependencies: [tomli]
|
||||
|
||||
+4804
-12941
File diff suppressed because one or more lines are too long
+4
-2
@@ -30,7 +30,7 @@ RUN set -eux \
|
||||
# Purpose: Installs s6-overlay and rootfs
|
||||
# Comments:
|
||||
# - Don't leave anything extra in here either
|
||||
FROM ghcr.io/astral-sh/uv:0.12.16-python3.14-trixie-slim AS s6-overlay-base
|
||||
FROM ghcr.io/astral-sh/uv:0.12.20-python3.14-trixie-slim AS s6-overlay-base
|
||||
|
||||
WORKDIR /usr/src/s6
|
||||
|
||||
@@ -171,7 +171,9 @@ RUN set -eux \
|
||||
&& cp /etc/ImageMagick-6/paperless-policy.xml /etc/ImageMagick-6/policy.xml \
|
||||
&& echo "Cleaning up image layer" \
|
||||
&& rm --force --verbose *.deb \
|
||||
&& rm --recursive --force --verbose /var/lib/apt/lists/*
|
||||
&& rm --recursive --force --verbose /var/lib/apt/lists/* \
|
||||
&& echo "Configuring interactive shells to source the s6 container environment" \
|
||||
&& echo '. /etc/profile.d/contenv.sh' >> /etc/bash.bashrc
|
||||
|
||||
WORKDIR /usr/src/paperless/src/
|
||||
|
||||
|
||||
@@ -24,7 +24,7 @@ services:
|
||||
network_mode: host
|
||||
restart: unless-stopped
|
||||
greenmail:
|
||||
image: docker.io/greenmail/standalone:2.1.13
|
||||
image: docker.io/greenmail/standalone:2.1.14
|
||||
hostname: greenmail
|
||||
container_name: greenmail
|
||||
environment:
|
||||
|
||||
Executable
+18
@@ -0,0 +1,18 @@
|
||||
#!/bin/sh
|
||||
# Source s6 container environment for interactive shells.
|
||||
# Ensures variables resolved from *_FILE secret injection are visible
|
||||
# when using 'docker exec bash'. Does not affect s6 services (those
|
||||
# use with-contenv directly). Has no effect in non-container contexts
|
||||
# because the directory will not exist.
|
||||
# Note: sh/dash shells opened via 'docker exec sh' are not covered;
|
||||
# only bash-based sessions benefit from this file.
|
||||
_pngx_contenv="/run/s6/container_environment"
|
||||
if [ -d "${_pngx_contenv}" ]; then
|
||||
for _pngx_f in "${_pngx_contenv}"/*; do
|
||||
[ -f "${_pngx_f}" ] || continue
|
||||
_pngx_name=$(basename "${_pngx_f}")
|
||||
_pngx_val=$(cat "${_pngx_f}")
|
||||
export "${_pngx_name}=${_pngx_val}"
|
||||
done
|
||||
fi
|
||||
unset _pngx_contenv _pngx_f _pngx_name _pngx_val
|
||||
+14
-9
@@ -136,13 +136,15 @@ for suggested generation and embedding models.
|
||||
### AI-assisted suggestions
|
||||
|
||||
With AI enabled, Paperless-ngx can suggest a title, tags, correspondent, document type,
|
||||
storage path and dates by sending the document to the LLM. This is **opt-in per request**
|
||||
and surfaces through the "Suggest" control on the document detail page, alongside the
|
||||
classic classifier-based suggestions — it does not disable them. Suggestions are requested
|
||||
automatically when you open a document that carries an inbox tag unless "Automatically request
|
||||
suggestions for inbox documents" under Settings > Documents is disabled. Suggestion output
|
||||
language can be steered with
|
||||
[`PAPERLESS_AI_LLM_OUTPUT_LANGUAGE`](configuration.md#PAPERLESS_AI_LLM_OUTPUT_LANGUAGE)
|
||||
storage path and dates by sending the document to the LLM using "Suggest" button on the document
|
||||
detail page. You can choose which type of suggestions are requested by default under Settings >
|
||||
Documents, either ML (classifier-based) suggestions, AI suggestions, or both. When both are requested
|
||||
the results are combined.
|
||||
|
||||
Suggestions are requested automatically when you open a document that carries an inbox tag
|
||||
unless "Automatically request suggestions for inbox documents" under Settings > Documents is disabled.
|
||||
|
||||
Suggestion output language can be steered with [`PAPERLESS_AI_LLM_OUTPUT_LANGUAGE`](configuration.md#PAPERLESS_AI_LLM_OUTPUT_LANGUAGE)
|
||||
(otherwise it follows the user's UI language).
|
||||
|
||||
### The LLM index (RAG) and similar documents
|
||||
@@ -153,8 +155,11 @@ in similar existing documents, and the document chat can retrieve relevant conte
|
||||
|
||||
Enable it by setting
|
||||
[`PAPERLESS_AI_LLM_EMBEDDING_BACKEND`](configuration.md#PAPERLESS_AI_LLM_EMBEDDING_BACKEND)
|
||||
(`huggingface` for fully-local embeddings, or `ollama` / `openai-like`). The index is only
|
||||
built when AI is enabled **and** an embedding backend is set.
|
||||
(`huggingface` for fully-local embeddings, or `ollama` / `openai-like`). By default, the main
|
||||
LLM API key and endpoint are used, but an optional embedding-specific[API key](configuration.md#PAPERLESS_AI_LLM_EMBEDDING_API_KEY)
|
||||
and [endpoint](configuration.md#PAPERLESS_AI_LLM_EMBEDDING_ENDPOINT) can be configured.
|
||||
|
||||
The index is only built when AI is enabled **and** an embedding backend is set.
|
||||
|
||||
The index is updated automatically on a schedule controlled by
|
||||
[`PAPERLESS_LLM_INDEX_TASK_CRON`](configuration.md#PAPERLESS_LLM_INDEX_TASK_CRON) (daily by
|
||||
|
||||
@@ -2133,6 +2133,13 @@ for language and resource considerations.
|
||||
|
||||
Defaults to None.
|
||||
|
||||
#### [`PAPERLESS_AI_LLM_EMBEDDING_API_KEY=<str>`](#PAPERLESS_AI_LLM_EMBEDDING_API_KEY) {#PAPERLESS_AI_LLM_EMBEDDING_API_KEY}
|
||||
|
||||
: The API key to use for the embedding backend. If not supplied, embeddings use
|
||||
`PAPERLESS_AI_LLM_API_KEY`.
|
||||
|
||||
Defaults to None.
|
||||
|
||||
#### [`PAPERLESS_AI_LLM_EMBEDDING_ENDPOINT=<str>`](#PAPERLESS_AI_LLM_EMBEDDING_ENDPOINT) {#PAPERLESS_AI_LLM_EMBEDDING_ENDPOINT}
|
||||
|
||||
: The endpoint / url to use for the embedding backend. If not supplied, embeddings use
|
||||
@@ -2217,6 +2224,19 @@ used with the OpenAI-compatible backend to target a custom provider or local gat
|
||||
|
||||
Defaults to true, which allows internal endpoints.
|
||||
|
||||
#### [`PAPERLESS_AI_LLM_EXTRA_PARAMS=<json>`](#PAPERLESS_AI_LLM_EXTRA_PARAMS) {#PAPERLESS_AI_LLM_EXTRA_PARAMS}
|
||||
|
||||
: A JSON object of extra parameters sent with every LLM request, for providers that require a parameter Paperless does not
|
||||
set itself. Values here override Paperless' own, and no validation is performed. Whatever you put here is passed to the
|
||||
backend as-is, so an invalid parameter will simply be rejected by your provider. For example, current OpenAI reasoning
|
||||
models refuse tool calls on the chat completions API unless reasoning is off:
|
||||
|
||||
```
|
||||
PAPERLESS_AI_LLM_EXTRA_PARAMS={"reasoning_effort": "none"}
|
||||
```
|
||||
|
||||
Defaults to empty, which adds nothing to requests.
|
||||
|
||||
#### [`PAPERLESS_LLM_INDEX_TASK_CRON=<cron expression>`](#PAPERLESS_LLM_INDEX_TASK_CRON) {#PAPERLESS_LLM_INDEX_TASK_CRON}
|
||||
|
||||
: Configures the schedule to update the AI embeddings of text content and metadata for all documents. Only performed if
|
||||
|
||||
@@ -150,6 +150,7 @@ pnpm ng build --configuration production
|
||||
is loaded as well. However, the tests rely on the default
|
||||
configuration. This is not ideal. But for now, make sure no settings
|
||||
except for DEBUG are overridden when testing.
|
||||
- Tests run in a random order each session, so that one test cannot quietly depend on another having run first. The seed is printed at the top of the run; pass `--randomly-seed=<seed>` to replay that exact order, or `--randomly-seed=last` to repeat the previous run.
|
||||
|
||||
!!! note
|
||||
|
||||
|
||||
+23
-12
@@ -1,7 +1,9 @@
|
||||
[project]
|
||||
name = "paperless-ngx"
|
||||
version = "3.2.1"
|
||||
description = "A community-supported supercharged document management system: scan, index and archive all your physical documents"
|
||||
description = """\
|
||||
A community-supported supercharged document management system: scan, index and archive all your physical documents\
|
||||
"""
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.11"
|
||||
classifiers = [
|
||||
@@ -10,13 +12,13 @@ classifiers = [
|
||||
"Programming Language :: Python :: 3.12",
|
||||
"Programming Language :: Python :: 3.13",
|
||||
"Programming Language :: Python :: 3.14",
|
||||
"Programming Language :: Python :: 3.15",
|
||||
]
|
||||
# TODO: Move certain things to groups and then utilize that further
|
||||
# This will allow testing to not install a webserver, mysql, etc
|
||||
dependencies = [
|
||||
"azure-ai-documentintelligence>=1.0.2",
|
||||
"babel>=2.17",
|
||||
"bleach~=6.4.0",
|
||||
"celery[redis]~=5.6.2",
|
||||
"channels~=4.2",
|
||||
"channels-redis~=4.2",
|
||||
@@ -32,7 +34,7 @@ dependencies = [
|
||||
"django-cors-headers~=4.9.0",
|
||||
"django-extensions~=4.1",
|
||||
"django-filter~=25.1",
|
||||
"django-guardian>=3.3.3,<3.5",
|
||||
"django-guardian>=3.3.3,<3.6",
|
||||
"django-multiselectfield~=1.0.1",
|
||||
"django-rich~=2.2.0",
|
||||
"django-soft-delete~=1.0.18",
|
||||
@@ -42,7 +44,7 @@ dependencies = [
|
||||
"drf-spectacular-sidecar>=2026.7.1,<2026.10",
|
||||
"drf-writable-nested~=0.7.1",
|
||||
"filelock~=3.32.0",
|
||||
"flower>=2.0.1,<2.2",
|
||||
"flower>=2.0.1,<2.3",
|
||||
"gotenberg-client[httpx]~=1.0",
|
||||
"httpx-oauth~=0.17",
|
||||
"ijson>=3.5.1",
|
||||
@@ -74,6 +76,7 @@ dependencies = [
|
||||
"tantivy~=0.26.0",
|
||||
"tika-client[httpx]~=1.0",
|
||||
"torch>=2.13,<2.15",
|
||||
"turbohtml~=1.10.0",
|
||||
"watchfiles>=1.2",
|
||||
"whitenoise~=6.11",
|
||||
"whoosh-compat[tantivy]==0.3",
|
||||
@@ -81,13 +84,13 @@ dependencies = [
|
||||
]
|
||||
[project.optional-dependencies]
|
||||
mariadb = [
|
||||
"mysqlclient~=2.2.7",
|
||||
"mysqlclient>=2.2.7,<2.4",
|
||||
]
|
||||
postgres = [
|
||||
"psycopg[c,pool]==3.3.4",
|
||||
# Direct dependency for proper resolution of the pre-built wheels
|
||||
"psycopg-c==3.3.4",
|
||||
"psycopg-pool==3.3.1",
|
||||
"psycopg-pool==3.3.2",
|
||||
]
|
||||
webserver = [
|
||||
"granian[uvloop]>=2.7,<2.9",
|
||||
@@ -109,7 +112,7 @@ lint = [
|
||||
testing = [
|
||||
"daphne",
|
||||
"factory-boy~=3.3.1",
|
||||
"faker>=40.36,<40.39",
|
||||
"faker>=40.36,<40.40",
|
||||
"imagehash",
|
||||
"pytest~=9.1.1",
|
||||
"pytest-cov~=7.1.0",
|
||||
@@ -117,7 +120,7 @@ testing = [
|
||||
"pytest-env~=1.7.0",
|
||||
"pytest-httpx",
|
||||
"pytest-mock~=3.15.1",
|
||||
# "pytest-randomly~=4.0.1",
|
||||
"pytest-randomly~=5.0.0",
|
||||
"pytest-rerunfailures~=16.4",
|
||||
"pytest-sugar",
|
||||
"pytest-xdist~=3.8.0",
|
||||
@@ -133,7 +136,6 @@ typing = [
|
||||
"mypy",
|
||||
"mypy-baseline",
|
||||
"pyrefly",
|
||||
"types-bleach",
|
||||
"types-channels",
|
||||
"types-colorama",
|
||||
"types-dateparser",
|
||||
@@ -148,6 +150,7 @@ typing = [
|
||||
|
||||
[tool.uv]
|
||||
required-version = ">=0.9.0"
|
||||
prerelease = "disallow"
|
||||
environments = [
|
||||
"sys_platform == 'darwin'",
|
||||
"sys_platform == 'linux'",
|
||||
@@ -244,6 +247,10 @@ per-file-ignores."docker/wait-for-redis.py" = [
|
||||
per-file-ignores."src/documents/models.py" = [
|
||||
"SIM115",
|
||||
]
|
||||
per-file-ignores."src/documents/tests/*.py" = [
|
||||
"TID251",
|
||||
]
|
||||
flake8-tidy-imports.banned-api."documents.tests".msg = "Shared test infrastructure lives in src/paperless_testing/."
|
||||
isort.force-single-line = true
|
||||
|
||||
[tool.codespell]
|
||||
@@ -271,9 +278,9 @@ plugins = [
|
||||
]
|
||||
|
||||
[tool.pyrefly]
|
||||
baseline = ".pyrefly-baseline.json"
|
||||
python-platform = "linux"
|
||||
search-path = [ "src" ]
|
||||
baseline = ".pyrefly-baseline.json"
|
||||
|
||||
[tool.django-stubs]
|
||||
django_settings_module = "paperless.settings"
|
||||
@@ -326,6 +333,8 @@ PAPERLESS_CACHE_BACKEND = "django.core.cache.backends.locmem.LocMemCache"
|
||||
PAPERLESS_CHANNELS_BACKEND = "channels.layers.InMemoryChannelLayer"
|
||||
# I don't think anything hits this, but just in case, basically infinite
|
||||
PAPERLESS_TOKEN_THROTTLE_RATE = "1000/min"
|
||||
# The 0.1s production default trips on a stalled CI runner, the date parsing tests then find no dates
|
||||
PAPERLESS_MATCH_REGEX_TIMEOUT_SECONDS = "5"
|
||||
|
||||
[tool.coverage.run]
|
||||
source = [
|
||||
@@ -334,13 +343,15 @@ source = [
|
||||
omit = [
|
||||
"*/tests/*",
|
||||
"manage.py",
|
||||
"paperless/wsgi.py",
|
||||
"paperless/auth.py",
|
||||
"paperless/wsgi.py",
|
||||
"src/conftest.py",
|
||||
"src/paperless_testing/*",
|
||||
]
|
||||
[tool.coverage.report]
|
||||
exclude_also = [
|
||||
"if settings.AUDIT_LOG_ENABLED:",
|
||||
"if AUDIT_LOG_ENABLED:",
|
||||
"if settings.AUDIT_LOG_ENABLED:",
|
||||
"if TYPE_CHECKING:",
|
||||
]
|
||||
|
||||
|
||||
@@ -91,6 +91,11 @@
|
||||
"glob": "**/*",
|
||||
"input": "node_modules/pdfjs-dist/iccs/",
|
||||
"output": "/assets/iccs/"
|
||||
},
|
||||
{
|
||||
"glob": "**/*",
|
||||
"input": "node_modules/pdfjs-dist/cmaps/",
|
||||
"output": "/assets/cmaps/"
|
||||
}
|
||||
],
|
||||
"styles": [
|
||||
|
||||
+315
-238
File diff suppressed because it is too large
Load Diff
+22
-22
@@ -15,16 +15,16 @@
|
||||
},
|
||||
"private": true,
|
||||
"dependencies": {
|
||||
"@angular/cdk": "^22.1.6",
|
||||
"@angular/common": "~22.1.6",
|
||||
"@angular/compiler": "~22.1.6",
|
||||
"@angular/core": "~22.1.6",
|
||||
"@angular/forms": "~22.1.6",
|
||||
"@angular/localize": "~22.1.6",
|
||||
"@angular/platform-browser": "~22.1.6",
|
||||
"@angular/router": "~22.1.6",
|
||||
"@angular/cdk": "^22.2.0",
|
||||
"@angular/common": "~22.2.0",
|
||||
"@angular/compiler": "~22.2.0",
|
||||
"@angular/core": "~22.2.0",
|
||||
"@angular/forms": "~22.2.0",
|
||||
"@angular/localize": "~22.2.0",
|
||||
"@angular/platform-browser": "~22.2.0",
|
||||
"@angular/router": "~22.2.0",
|
||||
"@ng-bootstrap/ng-bootstrap": "^21.0.0",
|
||||
"@ng-select/ng-select": "~24.1.1",
|
||||
"@ng-select/ng-select": "~24.1.2",
|
||||
"@ngneat/dirty-check-forms": "^3.0.3",
|
||||
"@popperjs/core": "^2.11.8",
|
||||
"bootstrap": "^5.3.8",
|
||||
@@ -45,29 +45,29 @@
|
||||
},
|
||||
"devDependencies": {
|
||||
"@angular-builders/jest": "^22.0.1",
|
||||
"@angular-devkit/core": "^22.1.8",
|
||||
"@angular-devkit/schematics": "^22.1.8",
|
||||
"@angular-devkit/core": "^22.2.0",
|
||||
"@angular-devkit/schematics": "^22.2.0",
|
||||
"@angular-eslint/builder": "22.5.0",
|
||||
"@angular-eslint/eslint-plugin": "22.5.0",
|
||||
"@angular-eslint/eslint-plugin-template": "22.5.0",
|
||||
"@angular-eslint/schematics": "22.5.0",
|
||||
"@angular-eslint/template-parser": "22.5.0",
|
||||
"@angular/build": "22.1.8",
|
||||
"@angular/cli": "22.1.8",
|
||||
"@angular/compiler-cli": "~22.1.6",
|
||||
"@angular/build": "22.2.0",
|
||||
"@angular/cli": "22.2.0",
|
||||
"@angular/compiler-cli": "~22.2.0",
|
||||
"@playwright/test": "^1.62.1",
|
||||
"@types/jest": "^30.0.0",
|
||||
"@types/node": "^26.5.0",
|
||||
"@typescript-eslint/eslint-plugin": "^8.70.0",
|
||||
"@typescript-eslint/parser": "^8.70.0",
|
||||
"@typescript-eslint/utils": "^8.70.0",
|
||||
"eslint": "^10.10.0",
|
||||
"jest": "30.5.1",
|
||||
"jest-environment-jsdom": "^30.5.1",
|
||||
"@types/node": "^26.6.2",
|
||||
"@typescript-eslint/eslint-plugin": "^8.70.1",
|
||||
"@typescript-eslint/parser": "^8.70.1",
|
||||
"@typescript-eslint/utils": "^8.70.1",
|
||||
"eslint": "^10.11.0",
|
||||
"jest": "30.5.2",
|
||||
"jest-environment-jsdom": "^30.5.2",
|
||||
"jest-junit": "^17.0.0",
|
||||
"jest-preset-angular": "^17.0.0",
|
||||
"jest-websocket-mock": "^2.5.0",
|
||||
"prettier": "^3.9.6",
|
||||
"prettier": "^3.9.9",
|
||||
"prettier-plugin-organize-imports": "^4.3.0",
|
||||
"ts-node": "~10.9.2",
|
||||
"typescript": "^6.0.3"
|
||||
|
||||
Generated
+1312
-1960
File diff suppressed because it is too large
Load Diff
@@ -253,6 +253,24 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@if (aiEnabled) {
|
||||
<div class="row mb-3">
|
||||
<div class="col-md-3 col-form-label pt-0">
|
||||
<span i18n>Suggestions default to</span>
|
||||
</div>
|
||||
<div class="col">
|
||||
<fieldset class="btn-group btn-group-sm">
|
||||
<input type="radio" class="btn-check" id="suggestionSourceBoth" [value]="SuggestionSource.Both" formControlName="documentEditingSuggestionSource">
|
||||
<label class="btn btn-outline-primary" for="suggestionSourceBoth"><ng-container i18n>Both</ng-container></label>
|
||||
<input type="radio" class="btn-check" id="suggestionSourceML" [value]="SuggestionSource.ML" formControlName="documentEditingSuggestionSource">
|
||||
<label class="btn btn-outline-primary" for="suggestionSourceML"><i-bs class="me-1" name="cpu"></i-bs><ng-container i18n>ML only</ng-container></label>
|
||||
<input type="radio" class="btn-check" id="suggestionSourceAI" [value]="SuggestionSource.AI" formControlName="documentEditingSuggestionSource">
|
||||
<label class="btn btn-outline-primary" for="suggestionSourceAI"><i-bs class="me-1" name="stars"></i-bs><ng-container i18n>AI only</ng-container></label>
|
||||
</fieldset>
|
||||
</div>
|
||||
</div>
|
||||
}
|
||||
|
||||
<div class="row">
|
||||
<div class="col">
|
||||
<pngx-input-check i18n-title title="Automatically request suggestions for inbox documents" i18n-hint hint="If un-checked, suggestions must be requested via the Suggest button." formControlName="documentEditingAutoSuggest"></pngx-input-check>
|
||||
|
||||
@@ -307,7 +307,7 @@ describe('SettingsComponent', () => {
|
||||
expect(toastErrorSpy).toHaveBeenCalled()
|
||||
expect(storeSpy).toHaveBeenCalled()
|
||||
expect(appearanceSettingsSpy).not.toHaveBeenCalled()
|
||||
expect(setSpy).toHaveBeenCalledTimes(34)
|
||||
expect(setSpy).toHaveBeenCalledTimes(35)
|
||||
expect(setSpy).toHaveBeenCalledWith(SETTINGS_KEYS.SIDEBAR_HIDDEN_ITEMS, [
|
||||
HideableSidebarItemID.Workflows,
|
||||
])
|
||||
|
||||
@@ -44,6 +44,7 @@ import {
|
||||
HIDEABLE_SIDEBAR_ITEM_IDS,
|
||||
HideableSidebarItemID,
|
||||
SETTINGS_KEYS,
|
||||
SuggestionSource,
|
||||
} from 'src/app/data/ui-settings'
|
||||
import { User } from 'src/app/data/user'
|
||||
import { IfPermissionsDirective } from 'src/app/directives/if-permissions.directive'
|
||||
@@ -184,6 +185,7 @@ export class SettingsComponent
|
||||
documentEditingRemoveInboxTags: new FormControl(null),
|
||||
documentEditingOverlayThumbnail: new FormControl(null),
|
||||
documentEditingAutoSuggest: new FormControl(null),
|
||||
documentEditingSuggestionSource: new FormControl(null),
|
||||
documentDetailsHiddenFields: new FormControl([]),
|
||||
searchDbOnly: new FormControl(null),
|
||||
searchLink: new FormControl(null),
|
||||
@@ -217,6 +219,11 @@ export class SettingsComponent
|
||||
public readonly PdfZoomScale = PdfZoomScale
|
||||
|
||||
public readonly PdfEditorEditMode = PdfEditorEditMode
|
||||
public readonly SuggestionSource = SuggestionSource
|
||||
|
||||
get aiEnabled(): boolean {
|
||||
return this.settings.get(SETTINGS_KEYS.AI_ENABLED)
|
||||
}
|
||||
|
||||
public readonly documentDetailFieldOptions = documentDetailFieldOptions
|
||||
public readonly sidebarItemOptions = HIDEABLE_SIDEBAR_ITEM_IDS.map((id) => ({
|
||||
@@ -404,6 +411,9 @@ export class SettingsComponent
|
||||
documentEditingAutoSuggest: this.settings.get(
|
||||
SETTINGS_KEYS.DOCUMENT_EDITING_AUTO_SUGGEST
|
||||
),
|
||||
documentEditingSuggestionSource: this.settings.get(
|
||||
SETTINGS_KEYS.DOCUMENT_EDITING_SUGGESTION_SOURCE
|
||||
),
|
||||
documentDetailsHiddenFields: this.settings.get(
|
||||
SETTINGS_KEYS.DOCUMENT_DETAILS_HIDDEN_FIELDS
|
||||
),
|
||||
@@ -625,6 +635,10 @@ export class SettingsComponent
|
||||
SETTINGS_KEYS.DOCUMENT_EDITING_AUTO_SUGGEST,
|
||||
this.settingsForm.value.documentEditingAutoSuggest
|
||||
)
|
||||
this.settings.set(
|
||||
SETTINGS_KEYS.DOCUMENT_EDITING_SUGGESTION_SOURCE,
|
||||
this.settingsForm.value.documentEditingSuggestionSource
|
||||
)
|
||||
this.settings.set(
|
||||
SETTINGS_KEYS.DOCUMENT_DETAILS_HIDDEN_FIELDS,
|
||||
this.settingsForm.value.documentDetailsHiddenFields
|
||||
|
||||
@@ -154,11 +154,29 @@
|
||||
& section {
|
||||
position: absolute;
|
||||
text-align: initial;
|
||||
pointer-events: auto;
|
||||
box-sizing: border-box;
|
||||
transform-origin: 0 0;
|
||||
}
|
||||
|
||||
& :is(.linkAnnotation, .buttonWidgetAnnotation.pushButton) > a {
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
font-size: 1em;
|
||||
transition: none;
|
||||
}
|
||||
|
||||
& :is(.linkAnnotation, .buttonWidgetAnnotation.pushButton):not(.hasBorder)
|
||||
> a:hover {
|
||||
opacity: 0.2;
|
||||
background-color: rgb(255 255 0);
|
||||
}
|
||||
|
||||
& .annotationTextContent {
|
||||
opacity: 0;
|
||||
}
|
||||
}
|
||||
|
||||
:host ::ng-deep .textLayer.selecting ~ .annotationLayer section {
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
import { SimpleChange } from '@angular/core'
|
||||
import { ComponentFixture, TestBed } from '@angular/core/testing'
|
||||
import * as pdfjs from 'pdfjs-dist/legacy/build/pdf.mjs'
|
||||
import { PDFSinglePageViewer, PDFViewer } from 'pdfjs-dist/web/pdf_viewer.mjs'
|
||||
import {
|
||||
LinkTarget,
|
||||
PDFSinglePageViewer,
|
||||
PDFViewer,
|
||||
} from 'pdfjs-dist/web/pdf_viewer.mjs'
|
||||
import { PngxPdfViewerComponent } from './pdf-viewer.component'
|
||||
import { PdfRenderMode, PdfZoomLevel, PdfZoomScale } from './pdf-viewer.types'
|
||||
|
||||
@@ -58,6 +62,16 @@ describe('PngxPdfViewerComponent', () => {
|
||||
expect((component as any).pdfViewer).toBeInstanceOf(PDFViewer)
|
||||
})
|
||||
|
||||
it('opens external links in a new tab', () => {
|
||||
const linkService = (component as any).linkService
|
||||
expect(linkService.options).toEqual(
|
||||
expect.objectContaining({
|
||||
externalLinkTarget: LinkTarget.BLANK,
|
||||
externalLinkRel: 'noopener noreferrer nofollow',
|
||||
})
|
||||
)
|
||||
})
|
||||
|
||||
it('resolves the worker source relative to the document base URI', async () => {
|
||||
setBaseHref('/paperless/')
|
||||
const getDocumentSpy = jest.spyOn(pdfjs, 'getDocument')
|
||||
@@ -76,6 +90,7 @@ describe('PngxPdfViewerComponent', () => {
|
||||
withCredentials: true,
|
||||
wasmUrl: expect.stringContaining('/paperless/assets/wasm/'),
|
||||
iccUrl: expect.stringContaining('/paperless/assets/iccs/'),
|
||||
cMapUrl: expect.stringContaining('/paperless/assets/cmaps/'),
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
@@ -21,6 +21,7 @@ import {
|
||||
} from 'pdfjs-dist/legacy/build/pdf.mjs'
|
||||
import {
|
||||
EventBus,
|
||||
LinkTarget,
|
||||
PDFFindController,
|
||||
PDFLinkService,
|
||||
PDFSinglePageViewer,
|
||||
@@ -75,7 +76,11 @@ export class PngxPdfViewerComponent
|
||||
private lastViewerPage?: number
|
||||
|
||||
private readonly eventBus = new EventBus()
|
||||
private readonly linkService = new PDFLinkService({ eventBus: this.eventBus })
|
||||
private readonly linkService = new PDFLinkService({
|
||||
eventBus: this.eventBus,
|
||||
externalLinkTarget: LinkTarget.BLANK,
|
||||
externalLinkRel: 'noopener noreferrer nofollow',
|
||||
})
|
||||
private readonly findController = new PDFFindController({
|
||||
eventBus: this.eventBus,
|
||||
linkService: this.linkService,
|
||||
@@ -185,6 +190,7 @@ export class PngxPdfViewerComponent
|
||||
withCredentials: true,
|
||||
wasmUrl: new URL('assets/wasm/', this.document.baseURI).toString(),
|
||||
iccUrl: new URL('assets/iccs/', this.document.baseURI).toString(),
|
||||
cMapUrl: new URL('assets/cmaps/', this.document.baseURI).toString(),
|
||||
}
|
||||
this.loadingTask = getDocument(initOptions)
|
||||
try {
|
||||
|
||||
+77
-51
@@ -1,58 +1,84 @@
|
||||
<div class="btn-group">
|
||||
<button type="button" class="btn btn-sm btn-outline-primary" (click)="clickSuggest()" [disabled]="disabled() || loading() || (suggestions() && !aiEnabled())" [aria-label]="noSuggestions ? 'No suggestions' : 'Suggest'" i18n-aria-label>
|
||||
@if (loading()) {
|
||||
<div class="spinner-border spinner-border-sm" role="status"></div>
|
||||
} @else if (noSuggestions) {
|
||||
<i-bs width="1.2em" height="1.2em" name="check-circle"></i-bs>
|
||||
} @else {
|
||||
<i-bs width="1.2em" height="1.2em" name="stars"></i-bs>
|
||||
<div class="d-flex align-items-center">
|
||||
<div class="btn-group">
|
||||
<button type="button" class="btn btn-sm btn-outline-primary" (click)="clickSuggest()" [disabled]="disabled() || loading() || (suggestions() && !aiEnabled())" [aria-label]="noSuggestions ? 'No suggestions' : 'Suggest'" i18n-aria-label>
|
||||
@if (loading()) {
|
||||
<div class="spinner-border spinner-border-sm" role="status"></div>
|
||||
} @else if (noSuggestions) {
|
||||
<i-bs width="1.2em" height="1.2em" name="check-circle"></i-bs>
|
||||
} @else {
|
||||
<i-bs width="1.2em" height="1.2em" name="lightbulb"></i-bs>
|
||||
}
|
||||
@if (noSuggestions) {
|
||||
<span class="d-none d-lg-inline ps-1" i18n>No suggestions</span>
|
||||
} @else {
|
||||
<span class="d-none d-lg-inline ps-1" i18n>Suggest</span>
|
||||
}
|
||||
@if (totalSuggestions > 0) {
|
||||
<span class="badge bg-primary ms-2">{{ totalSuggestions }}</span>
|
||||
}
|
||||
</button>
|
||||
|
||||
@if (aiEnabled()) {
|
||||
<div class="btn-group" ngbDropdown #dropdown="ngbDropdown" [popperOptions]="popperOptions">
|
||||
<button type="button" class="btn btn-sm btn-outline-primary" ngbDropdownToggle [disabled]="disabled() || loading() || !suggestions()" aria-expanded="false" aria-controls="suggestionsDropdown" aria-label="Suggestions dropdown">
|
||||
<span class="visually-hidden" i18n>Show suggestions</span>
|
||||
</button>
|
||||
|
||||
<div ngbDropdownMenu aria-labelledby="suggestionsDropdown" class="shadow suggestions-dropdown">
|
||||
<div class="list-group list-group-flush small pb-0">
|
||||
@if (novelSuggestions === 0 && reusableSuggestions === 0) {
|
||||
<div class="list-group-item text-muted fst-italic">
|
||||
<small class="text-muted small fst-italic" i18n>No novel suggestions</small>
|
||||
</div>
|
||||
}
|
||||
@if (suggestions()?.suggested_tags?.length > 0) {
|
||||
<small class="list-group-item text-uppercase text-muted small"><i-bs class="me-2" name="tags"></i-bs><ng-container i18n>Tags</ng-container></small>
|
||||
@for (tag of suggestions().suggested_tags; track tag) {
|
||||
<button type="button" class="list-group-item list-group-item-action bg-light" (click)="addTag.emit(tag)">{{ tag }}</button>
|
||||
}
|
||||
}
|
||||
@if (suggestions()?.suggested_document_types?.length > 0) {
|
||||
<div class="list-group-item text-uppercase text-muted small"><i-bs class="me-2" name="hash"></i-bs><ng-container i18n>Document Types</ng-container></div>
|
||||
@for (type of suggestions().suggested_document_types; track type) {
|
||||
<button type="button" class="list-group-item list-group-item-action bg-light" (click)="addDocumentType.emit(type)">{{ type }}</button>
|
||||
}
|
||||
}
|
||||
@if (suggestions()?.suggested_correspondents?.length > 0) {
|
||||
<div class="list-group-item text-uppercase text-muted small"><i-bs class="me-2" name="person"></i-bs><ng-container i18n>Correspondents</ng-container></div>
|
||||
@for (correspondent of suggestions().suggested_correspondents; track correspondent) {
|
||||
<button type="button" class="list-group-item list-group-item-action bg-light" (click)="addCorrespondent.emit(correspondent)">{{ correspondent }}</button>
|
||||
}
|
||||
}
|
||||
@if (reusableSuggestions > 0) {
|
||||
<div class="list-group-item text-muted fst-italic">
|
||||
<small class="text-muted small fst-italic" i18n>{reusableSuggestions, plural, =1 {1 existing value suggested below} other {{{reusableSuggestions}} existing values suggested below}}</small>
|
||||
</div>
|
||||
}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
}
|
||||
@if (noSuggestions) {
|
||||
<span class="d-none d-lg-inline ps-1" i18n>No suggestions</span>
|
||||
} @else {
|
||||
<span class="d-none d-lg-inline ps-1" i18n>Suggest</span>
|
||||
}
|
||||
@if (totalSuggestions > 0) {
|
||||
<span class="badge bg-primary ms-2">{{ totalSuggestions }}</span>
|
||||
}
|
||||
</button>
|
||||
</div>
|
||||
|
||||
@if (aiEnabled()) {
|
||||
<div class="btn-group" ngbDropdown #dropdown="ngbDropdown" [popperOptions]="popperOptions">
|
||||
<button type="button" class="btn btn-sm btn-outline-primary" ngbDropdownToggle [disabled]="disabled() || loading() || !suggestions()" aria-expanded="false" aria-controls="suggestionsDropdown" aria-label="Suggestions dropdown">
|
||||
<span class="visually-hidden" i18n>Show suggestions</span>
|
||||
<div ngbDropdown autoClose="outside" placement="bottom-end" [popperOptions]="popperOptions">
|
||||
<button type="button" class="btn btn-sm btn-link position-relative" ngbDropdownToggle [disabled]="disabled() || loading()" i18n-title title="Suggestion options">
|
||||
<i-bs name="three-dots"></i-bs>
|
||||
@if (source() !== defaultSource()) {
|
||||
<span class="position-absolute top-0 start-100 translate-middle p-1 bg-primary border border-light rounded-circle">
|
||||
<span class="visually-hidden" i18n>Not using default</span>
|
||||
</span>
|
||||
}
|
||||
</button>
|
||||
|
||||
<div ngbDropdownMenu aria-labelledby="suggestionsDropdown" class="shadow suggestions-dropdown">
|
||||
<div class="list-group list-group-flush small pb-0">
|
||||
@if (novelSuggestions === 0 && reusableSuggestions === 0) {
|
||||
<div class="list-group-item text-muted fst-italic">
|
||||
<small class="text-muted small fst-italic" i18n>No novel suggestions</small>
|
||||
</div>
|
||||
}
|
||||
@if (suggestions()?.suggested_tags?.length > 0) {
|
||||
<small class="list-group-item text-uppercase text-muted small"><i-bs class="me-2" name="tags"></i-bs><ng-container i18n>Tags</ng-container></small>
|
||||
@for (tag of suggestions().suggested_tags; track tag) {
|
||||
<button type="button" class="list-group-item list-group-item-action bg-light" (click)="addTag.emit(tag)">{{ tag }}</button>
|
||||
}
|
||||
}
|
||||
@if (suggestions()?.suggested_document_types?.length > 0) {
|
||||
<div class="list-group-item text-uppercase text-muted small"><i-bs class="me-2" name="hash"></i-bs><ng-container i18n>Document Types</ng-container></div>
|
||||
@for (type of suggestions().suggested_document_types; track type) {
|
||||
<button type="button" class="list-group-item list-group-item-action bg-light" (click)="addDocumentType.emit(type)">{{ type }}</button>
|
||||
}
|
||||
}
|
||||
@if (suggestions()?.suggested_correspondents?.length > 0) {
|
||||
<div class="list-group-item text-uppercase text-muted small"><i-bs class="me-2" name="person"></i-bs><ng-container i18n>Correspondents</ng-container></div>
|
||||
@for (correspondent of suggestions().suggested_correspondents; track correspondent) {
|
||||
<button type="button" class="list-group-item list-group-item-action bg-light" (click)="addCorrespondent.emit(correspondent)">{{ correspondent }}</button>
|
||||
}
|
||||
}
|
||||
@if (reusableSuggestions > 0) {
|
||||
<div class="list-group-item text-muted fst-italic">
|
||||
<small class="text-muted small fst-italic" i18n>{reusableSuggestions, plural, =1 {1 existing value suggested below} other {{{reusableSuggestions}} existing values suggested below}}</small>
|
||||
</div>
|
||||
}
|
||||
<div ngbDropdownMenu class="shadow p-3">
|
||||
<div class="small text-muted mb-2" i18n>Suggest using:</div>
|
||||
<div class="form-check small">
|
||||
<input class="form-check-input" type="checkbox" id="suggestionSourceML" [checked]="useML" [disabled]="useML && !useAI" (change)="setSources($event.target.checked, useAI)">
|
||||
<label class="form-check-label d-inline-flex align-items-center gap-1" for="suggestionSourceML"><i-bs name="cpu"></i-bs><ng-container i18n>ML</ng-container></label>
|
||||
</div>
|
||||
<div class="form-check small">
|
||||
<input class="form-check-input" type="checkbox" id="suggestionSourceAI" [checked]="useAI" [disabled]="useAI && !useML" (change)="setSources(useML, $event.target.checked)">
|
||||
<label class="form-check-label d-inline-flex align-items-center gap-1" for="suggestionSourceAI"><i-bs name="stars"></i-bs><ng-container i18n>AI</ng-container></label>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
+4
@@ -1,3 +1,7 @@
|
||||
.suggestions-dropdown {
|
||||
min-width: 250px;
|
||||
}
|
||||
|
||||
.btn-link.dropdown-toggle::after {
|
||||
display: none;
|
||||
}
|
||||
|
||||
+59
-1
@@ -1,6 +1,7 @@
|
||||
import { ComponentFixture, TestBed } from '@angular/core/testing'
|
||||
import { NgbDropdownModule } from '@ng-bootstrap/ng-bootstrap'
|
||||
import { NgxBootstrapIconsModule, allIcons } from 'ngx-bootstrap-icons'
|
||||
import { SuggestionSource } from 'src/app/data/ui-settings'
|
||||
import { SuggestionsDropdownComponent } from './suggestions-dropdown.component'
|
||||
|
||||
describe('SuggestionsDropdownComponent', () => {
|
||||
@@ -179,14 +180,71 @@ describe('SuggestionsDropdownComponent', () => {
|
||||
|
||||
it('should toggle dropdown when clickSuggest is called and suggestions are not null', () => {
|
||||
fixture.componentRef.setInput('aiEnabled', true)
|
||||
fixture.componentRef.setInput('fetchedSources', [SuggestionSource.ML])
|
||||
fixture.detectChanges()
|
||||
fixture.componentRef.setInput('suggestions', {
|
||||
suggested_correspondents: [],
|
||||
suggested_tags: [],
|
||||
suggested_document_types: [],
|
||||
})
|
||||
fixture.detectChanges()
|
||||
component.clickSuggest()
|
||||
expect(component.dropdown.open).toBeTruthy()
|
||||
expect(component.dropdown.isOpen()).toBeTruthy()
|
||||
expect(fixture.nativeElement.textContent).toContain('No novel suggestions')
|
||||
})
|
||||
|
||||
it('should fetch unfetched sources and show existing suggestions', () => {
|
||||
jest.spyOn(component.getSuggestions, 'emit')
|
||||
fixture.componentRef.setInput('aiEnabled', true)
|
||||
fixture.componentRef.setInput('source', SuggestionSource.Both)
|
||||
fixture.componentRef.setInput('fetchedSources', [SuggestionSource.ML])
|
||||
fixture.componentRef.setInput('suggestions', { tags: [1] })
|
||||
fixture.detectChanges()
|
||||
component.clickSuggest()
|
||||
expect(component.getSuggestions.emit).toHaveBeenCalledWith(
|
||||
SuggestionSource.Both
|
||||
)
|
||||
expect(component.dropdown.isOpen()).toBeTruthy()
|
||||
})
|
||||
|
||||
it('should only show source options when AI is enabled', () => {
|
||||
expect(
|
||||
fixture.nativeElement.querySelector('#suggestionSourceML')
|
||||
).toBeNull()
|
||||
fixture.componentRef.setInput('aiEnabled', true)
|
||||
fixture.detectChanges()
|
||||
fixture.nativeElement
|
||||
.querySelector('button[title="Suggestion options"]')
|
||||
.click()
|
||||
fixture.detectChanges()
|
||||
expect(
|
||||
fixture.nativeElement.querySelector('#suggestionSourceML')
|
||||
).not.toBeNull()
|
||||
})
|
||||
|
||||
it('should emit source changes and never allow no source', () => {
|
||||
const emitSpy = jest.spyOn(component.sourceChange, 'emit')
|
||||
component.setSources(true, true)
|
||||
expect(emitSpy).toHaveBeenCalledWith(SuggestionSource.Both)
|
||||
component.setSources(true, false)
|
||||
expect(emitSpy).toHaveBeenCalledWith(SuggestionSource.ML)
|
||||
component.setSources(false, true)
|
||||
expect(emitSpy).toHaveBeenCalledWith(SuggestionSource.AI)
|
||||
|
||||
emitSpy.mockClear()
|
||||
component.setSources(false, false)
|
||||
expect(emitSpy).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('should indicate a non-default source', () => {
|
||||
fixture.componentRef.setInput('aiEnabled', true)
|
||||
fixture.componentRef.setInput('source', SuggestionSource.AI)
|
||||
fixture.componentRef.setInput('defaultSource', SuggestionSource.AI)
|
||||
fixture.detectChanges()
|
||||
expect(fixture.nativeElement.textContent).not.toContain('Not using default')
|
||||
|
||||
fixture.componentRef.setInput('source', SuggestionSource.Both)
|
||||
fixture.detectChanges()
|
||||
expect(fixture.nativeElement.textContent).toContain('Not using default')
|
||||
})
|
||||
})
|
||||
|
||||
+40
-3
@@ -8,6 +8,7 @@ import {
|
||||
import { NgbDropdown, NgbDropdownModule } from '@ng-bootstrap/ng-bootstrap'
|
||||
import { NgxBootstrapIconsModule } from 'ngx-bootstrap-icons'
|
||||
import { DocumentSuggestions } from 'src/app/data/document-suggestions'
|
||||
import { SuggestionSource } from 'src/app/data/ui-settings'
|
||||
import { pngxPopperOptions } from 'src/app/utils/popper-options'
|
||||
|
||||
@Component({
|
||||
@@ -18,12 +19,16 @@ import { pngxPopperOptions } from 'src/app/utils/popper-options'
|
||||
})
|
||||
export class SuggestionsDropdownComponent {
|
||||
public popperOptions = pngxPopperOptions
|
||||
public readonly SuggestionSource = SuggestionSource
|
||||
|
||||
@ViewChild('dropdown') dropdown: NgbDropdown
|
||||
readonly suggestions = input<DocumentSuggestions>(null)
|
||||
readonly aiEnabled = input(false)
|
||||
readonly loading = input(false)
|
||||
readonly disabled = input(false)
|
||||
readonly source = input<SuggestionSource>(SuggestionSource.ML)
|
||||
readonly defaultSource = input<SuggestionSource>(SuggestionSource.ML)
|
||||
readonly fetchedSources = input<SuggestionSource[]>([])
|
||||
|
||||
readonly appliedTags = input<number[]>([])
|
||||
readonly appliedCorrespondent = input<number>(null)
|
||||
@@ -31,8 +36,10 @@ export class SuggestionsDropdownComponent {
|
||||
readonly appliedStoragePath = input<number>(null)
|
||||
|
||||
@Output()
|
||||
getSuggestions: EventEmitter<SuggestionsDropdownComponent> =
|
||||
new EventEmitter()
|
||||
getSuggestions: EventEmitter<SuggestionSource> = new EventEmitter()
|
||||
|
||||
@Output()
|
||||
sourceChange: EventEmitter<SuggestionSource> = new EventEmitter()
|
||||
|
||||
@Output()
|
||||
addTag: EventEmitter<string> = new EventEmitter()
|
||||
@@ -53,12 +60,42 @@ export class SuggestionsDropdownComponent {
|
||||
}
|
||||
|
||||
if (!this.suggestions()) {
|
||||
this.getSuggestions.emit(this)
|
||||
this.getSuggestions.emit(this.source())
|
||||
} else if (this.hasUnfetchedSources) {
|
||||
// sources changed, fetch the rest and show what we have meanwhile
|
||||
this.getSuggestions.emit(this.source())
|
||||
this.dropdown?.open()
|
||||
} else {
|
||||
this.dropdown?.toggle()
|
||||
}
|
||||
}
|
||||
|
||||
get useML(): boolean {
|
||||
return this.source() !== SuggestionSource.AI
|
||||
}
|
||||
|
||||
get useAI(): boolean {
|
||||
return this.source() !== SuggestionSource.ML
|
||||
}
|
||||
|
||||
get hasUnfetchedSources(): boolean {
|
||||
const fetched = this.fetchedSources()
|
||||
return (
|
||||
(this.useML && !fetched.includes(SuggestionSource.ML)) ||
|
||||
(this.useAI && !fetched.includes(SuggestionSource.AI))
|
||||
)
|
||||
}
|
||||
|
||||
public setSources(ml: boolean, ai: boolean) {
|
||||
if (ml && ai) {
|
||||
this.sourceChange.emit(SuggestionSource.Both)
|
||||
} else if (ml) {
|
||||
this.sourceChange.emit(SuggestionSource.ML)
|
||||
} else if (ai) {
|
||||
this.sourceChange.emit(SuggestionSource.AI)
|
||||
}
|
||||
}
|
||||
|
||||
get novelSuggestions(): number {
|
||||
return (
|
||||
(this.suggestions()?.suggested_correspondents?.length ?? 0) +
|
||||
|
||||
@@ -134,11 +134,15 @@
|
||||
[loading]="suggestionsLoading()"
|
||||
[suggestions]="suggestions()"
|
||||
[aiEnabled]="aiEnabled"
|
||||
[source]="suggestionSource"
|
||||
[defaultSource]="defaultSuggestionSource"
|
||||
[fetchedSources]="fetchedSuggestionSources()"
|
||||
[appliedTags]="documentForm.value.tags"
|
||||
[appliedCorrespondent]="documentForm.value.correspondent"
|
||||
[appliedDocumentType]="documentForm.value.document_type"
|
||||
[appliedStoragePath]="documentForm.value.storage_path"
|
||||
(getSuggestions)="getSuggestions()"
|
||||
(getSuggestions)="getSuggestions($event)"
|
||||
(sourceChange)="suggestionSourceOverride.set($event)"
|
||||
(addTag)="createTag($event)"
|
||||
(addDocumentType)="createDocumentType($event)"
|
||||
(addCorrespondent)="createCorrespondent($event)">
|
||||
|
||||
@@ -43,7 +43,7 @@ import {
|
||||
} from 'src/app/data/filter-rule-type'
|
||||
import { StoragePath } from 'src/app/data/storage-path'
|
||||
import { Tag } from 'src/app/data/tag'
|
||||
import { SETTINGS_KEYS } from 'src/app/data/ui-settings'
|
||||
import { SETTINGS_KEYS, SuggestionSource } from 'src/app/data/ui-settings'
|
||||
import { PermissionsGuard } from 'src/app/guards/permissions.guard'
|
||||
import { CustomDatePipe } from 'src/app/pipes/custom-date.pipe'
|
||||
import { DocumentTitlePipe } from 'src/app/pipes/document-title.pipe'
|
||||
@@ -1528,6 +1528,113 @@ describe('DocumentDetailComponent', () => {
|
||||
expect(component.suggestionsLoading()).toBeFalsy()
|
||||
})
|
||||
|
||||
it('should get and merge ML and AI suggestions when source is both', () => {
|
||||
settingsService.set(
|
||||
SETTINGS_KEYS.DOCUMENT_EDITING_SUGGESTION_SOURCE,
|
||||
SuggestionSource.Both
|
||||
)
|
||||
const getSetting = settingsService.get.bind(settingsService)
|
||||
jest
|
||||
.spyOn(settingsService, 'get')
|
||||
.mockImplementation((key) =>
|
||||
key === SETTINGS_KEYS.AI_ENABLED ? true : getSetting(key)
|
||||
)
|
||||
const suggestionsSpy = jest
|
||||
.spyOn(documentService, 'getSuggestions')
|
||||
.mockReturnValue(of({ tags: [42], dates: ['2024-01-01'] }))
|
||||
const aiSuggestionsSpy = jest
|
||||
.spyOn(documentService, 'getAiSuggestions')
|
||||
.mockReturnValue(
|
||||
of({ title: 'AI title', tags: [42, 43], suggested_tags: ['New'] })
|
||||
)
|
||||
initNormally()
|
||||
expect(suggestionsSpy).toHaveBeenCalled()
|
||||
expect(aiSuggestionsSpy).toHaveBeenCalled()
|
||||
expect(component.suggestions().title).toEqual('AI title')
|
||||
expect(component.suggestions().tags).toEqual([42, 43])
|
||||
expect(component.suggestions().suggested_tags).toEqual(['New'])
|
||||
expect(component.suggestions().dates).toEqual(['2024-01-01'])
|
||||
})
|
||||
|
||||
it('should only fetch sources not yet fetched for the document', () => {
|
||||
settingsService.set(SETTINGS_KEYS.DOCUMENT_EDITING_AUTO_SUGGEST, false)
|
||||
settingsService.set(
|
||||
SETTINGS_KEYS.DOCUMENT_EDITING_SUGGESTION_SOURCE,
|
||||
SuggestionSource.ML
|
||||
)
|
||||
const getSetting = settingsService.get.bind(settingsService)
|
||||
jest
|
||||
.spyOn(settingsService, 'get')
|
||||
.mockImplementation((key) =>
|
||||
key === SETTINGS_KEYS.AI_ENABLED ? true : getSetting(key)
|
||||
)
|
||||
const suggestionsSpy = jest
|
||||
.spyOn(documentService, 'getSuggestions')
|
||||
.mockReturnValue(of({ tags: [42] }))
|
||||
const aiSuggestionsSpy = jest
|
||||
.spyOn(documentService, 'getAiSuggestions')
|
||||
.mockReturnValue(of({ tags: [43] }))
|
||||
initNormally()
|
||||
|
||||
component.getSuggestions()
|
||||
expect(suggestionsSpy).toHaveBeenCalledTimes(1)
|
||||
expect(aiSuggestionsSpy).not.toHaveBeenCalled()
|
||||
|
||||
component.getSuggestions(SuggestionSource.Both)
|
||||
expect(suggestionsSpy).toHaveBeenCalledTimes(1)
|
||||
expect(aiSuggestionsSpy).toHaveBeenCalledTimes(1)
|
||||
expect(component.suggestions().tags).toEqual([42, 43])
|
||||
|
||||
component.getSuggestions(SuggestionSource.Both)
|
||||
expect(suggestionsSpy).toHaveBeenCalledTimes(1)
|
||||
expect(aiSuggestionsSpy).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('should use the per-document source override and reset it on document change', () => {
|
||||
settingsService.set(SETTINGS_KEYS.DOCUMENT_EDITING_AUTO_SUGGEST, false)
|
||||
const getSetting = settingsService.get.bind(settingsService)
|
||||
jest
|
||||
.spyOn(settingsService, 'get')
|
||||
.mockImplementation((key) =>
|
||||
key === SETTINGS_KEYS.AI_ENABLED ? true : getSetting(key)
|
||||
)
|
||||
initNormally()
|
||||
expect(component.suggestionSource).toEqual(SuggestionSource.AI)
|
||||
component.suggestionSourceOverride.set(SuggestionSource.ML)
|
||||
expect(component.suggestionSource).toEqual(SuggestionSource.ML)
|
||||
|
||||
jest
|
||||
.spyOn(documentService, 'get')
|
||||
.mockReturnValueOnce(of(Object.assign({}, doc)))
|
||||
;(component as any).loadDocument(doc.id, true)
|
||||
expect(component.suggestionSourceOverride()).toBeNull()
|
||||
expect(component.fetchedSuggestionSources()).toEqual([])
|
||||
})
|
||||
|
||||
it('should keep suggestions from one source if the other fails', () => {
|
||||
settingsService.set(
|
||||
SETTINGS_KEYS.DOCUMENT_EDITING_SUGGESTION_SOURCE,
|
||||
SuggestionSource.Both
|
||||
)
|
||||
const getSetting = settingsService.get.bind(settingsService)
|
||||
jest
|
||||
.spyOn(settingsService, 'get')
|
||||
.mockImplementation((key) =>
|
||||
key === SETTINGS_KEYS.AI_ENABLED ? true : getSetting(key)
|
||||
)
|
||||
const errorSpy = jest.spyOn(toastService, 'showError')
|
||||
jest
|
||||
.spyOn(documentService, 'getSuggestions')
|
||||
.mockReturnValue(of({ tags: [42] }))
|
||||
jest
|
||||
.spyOn(documentService, 'getAiSuggestions')
|
||||
.mockReturnValue(throwError(() => new Error('failed')))
|
||||
initNormally()
|
||||
expect(errorSpy).toHaveBeenCalled()
|
||||
expect(component.suggestions().tags).toEqual([42])
|
||||
expect(component.fetchedSuggestionSources()).toEqual([SuggestionSource.ML])
|
||||
})
|
||||
|
||||
it('should show error if needed for get suggestions', () => {
|
||||
const suggestionsSpy = jest.spyOn(documentService, 'getSuggestions')
|
||||
const errorSpy = jest.spyOn(toastService, 'showError')
|
||||
|
||||
@@ -28,7 +28,7 @@ import {
|
||||
import { dirtyCheck, DirtyComponent } from '@ngneat/dirty-check-forms'
|
||||
import { NgxBootstrapIconsModule } from 'ngx-bootstrap-icons'
|
||||
import { DeviceDetectorService } from 'ngx-device-detector'
|
||||
import { BehaviorSubject, Observable, of, Subject, timer } from 'rxjs'
|
||||
import { BehaviorSubject, merge, Observable, of, Subject, timer } from 'rxjs'
|
||||
import {
|
||||
catchError,
|
||||
debounceTime,
|
||||
@@ -48,7 +48,10 @@ import { DataType } from 'src/app/data/datatype'
|
||||
import { Document, DocumentVersionInfo } from 'src/app/data/document'
|
||||
import { DocumentMetadata } from 'src/app/data/document-metadata'
|
||||
import { DocumentNote } from 'src/app/data/document-note'
|
||||
import { DocumentSuggestions } from 'src/app/data/document-suggestions'
|
||||
import {
|
||||
DocumentSuggestions,
|
||||
mergeSuggestions,
|
||||
} from 'src/app/data/document-suggestions'
|
||||
import { DocumentType } from 'src/app/data/document-type'
|
||||
import { FilterRule } from 'src/app/data/filter-rule'
|
||||
import {
|
||||
@@ -63,7 +66,7 @@ import {
|
||||
import { ObjectWithId } from 'src/app/data/object-with-id'
|
||||
import { StoragePath } from 'src/app/data/storage-path'
|
||||
import { Tag } from 'src/app/data/tag'
|
||||
import { SETTINGS_KEYS } from 'src/app/data/ui-settings'
|
||||
import { SETTINGS_KEYS, SuggestionSource } from 'src/app/data/ui-settings'
|
||||
import { User } from 'src/app/data/user'
|
||||
import { IfPermissionsDirective } from 'src/app/directives/if-permissions.directive'
|
||||
import { CustomDatePipe } from 'src/app/pipes/custom-date.pipe'
|
||||
@@ -240,6 +243,10 @@ export class DocumentDetailComponent
|
||||
private readonly autoSuggestSetting = this.settings.getSignal<boolean>(
|
||||
SETTINGS_KEYS.DOCUMENT_EDITING_AUTO_SUGGEST
|
||||
)
|
||||
private readonly suggestionSourceSetting =
|
||||
this.settings.getSignal<SuggestionSource>(
|
||||
SETTINGS_KEYS.DOCUMENT_EDITING_SUGGESTION_SOURCE
|
||||
)
|
||||
private readonly hiddenFieldsSetting = this.settings.getSignal<
|
||||
DocumentDetailFieldID[]
|
||||
>(SETTINGS_KEYS.DOCUMENT_DETAILS_HIDDEN_FIELDS)
|
||||
@@ -261,6 +268,9 @@ export class DocumentDetailComponent
|
||||
readonly metadata = signal<DocumentMetadata>(undefined)
|
||||
readonly suggestions = signal<DocumentSuggestions>(undefined)
|
||||
readonly suggestionsLoading = signal(false)
|
||||
// per-document, resets on navigation
|
||||
readonly suggestionSourceOverride = signal<SuggestionSource>(null)
|
||||
readonly fetchedSuggestionSources = signal<SuggestionSource[]>([])
|
||||
readonly users = signal<User[]>(undefined)
|
||||
|
||||
readonly title = signal<string>(undefined)
|
||||
@@ -365,6 +375,15 @@ export class DocumentDetailComponent
|
||||
return this.autoSuggestSetting()
|
||||
}
|
||||
|
||||
get defaultSuggestionSource(): SuggestionSource {
|
||||
return this.aiEnabled ? this.suggestionSourceSetting() : SuggestionSource.ML
|
||||
}
|
||||
|
||||
get suggestionSource(): SuggestionSource {
|
||||
if (!this.aiEnabled) return SuggestionSource.ML
|
||||
return this.suggestionSourceOverride() ?? this.defaultSuggestionSource
|
||||
}
|
||||
|
||||
get archiveContentRenderType(): ContentRenderType {
|
||||
const hasArchiveVersion =
|
||||
this.metadata()?.has_archive_version ??
|
||||
@@ -590,6 +609,8 @@ export class DocumentDetailComponent
|
||||
}
|
||||
this.documentId.set(doc.id)
|
||||
this.suggestions.set(null)
|
||||
this.suggestionSourceOverride.set(null)
|
||||
this.fetchedSuggestionSources.set([])
|
||||
const openDocument = this.openDocumentService.getOpenDocument(
|
||||
this.documentId()
|
||||
)
|
||||
@@ -1077,29 +1098,44 @@ export class DocumentDetailComponent
|
||||
return this.documentForm.get('custom_fields') as FormArray
|
||||
}
|
||||
|
||||
getSuggestions() {
|
||||
getSuggestions(source: SuggestionSource = this.suggestionSource) {
|
||||
const sources = (
|
||||
source === SuggestionSource.Both
|
||||
? [SuggestionSource.ML, SuggestionSource.AI]
|
||||
: [source]
|
||||
).filter((s) => !this.fetchedSuggestionSources().includes(s))
|
||||
if (!sources.length) return
|
||||
|
||||
this.suggestionsLoading.set(true)
|
||||
const suggestionsObservable = this.aiEnabled
|
||||
? this.documentsService.getAiSuggestions(this.documentId())
|
||||
: this.documentsService.getSuggestions(this.documentId())
|
||||
suggestionsObservable
|
||||
merge(
|
||||
...sources.map((s) =>
|
||||
(s === SuggestionSource.AI
|
||||
? this.documentsService.getAiSuggestions(this.documentId())
|
||||
: this.documentsService.getSuggestions(this.documentId())
|
||||
).pipe(
|
||||
first(),
|
||||
map((result) => ({ source: s, result })),
|
||||
catchError((error) => {
|
||||
this.toastService.showError(
|
||||
$localize`Error retrieving suggestions.`,
|
||||
error
|
||||
)
|
||||
return of(null)
|
||||
})
|
||||
)
|
||||
)
|
||||
)
|
||||
.pipe(
|
||||
first(),
|
||||
takeUntil(this.unsubscribeNotifier),
|
||||
takeUntil(this.docChangeNotifier),
|
||||
finalize(() => this.suggestionsLoading.set(false))
|
||||
)
|
||||
.subscribe({
|
||||
next: (result) => {
|
||||
this.suggestions.set(result)
|
||||
},
|
||||
error: (error) => {
|
||||
this.suggestions.set(null)
|
||||
this.toastService.showError(
|
||||
$localize`Error retrieving suggestions.`,
|
||||
error
|
||||
)
|
||||
},
|
||||
.subscribe((response) => {
|
||||
if (!response) return
|
||||
this.fetchedSuggestionSources.update((f) => [...f, response.source])
|
||||
this.suggestions.set(
|
||||
mergeSuggestions(this.suggestions(), response.result)
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -146,6 +146,19 @@ describe('DocumentListComponent', () => {
|
||||
expect(reloadSpy).toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('should stop reloading on document deleted after destroy', () => {
|
||||
const reloadSpy = jest.spyOn(documentListService, 'reload')
|
||||
const documentDeletedSubject = new Subject<boolean>()
|
||||
jest
|
||||
.spyOn(websocketStatusService, 'onDocumentDeleted')
|
||||
.mockReturnValue(documentDeletedSubject)
|
||||
fixture.detectChanges()
|
||||
fixture.destroy()
|
||||
reloadSpy.mockClear()
|
||||
documentDeletedSubject.next(true)
|
||||
expect(reloadSpy).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('should show score sort fields on fulltext queries', () => {
|
||||
documentListService.setFilterRules([
|
||||
{
|
||||
|
||||
@@ -270,9 +270,12 @@ export class DocumentListComponent
|
||||
this.list.reload()
|
||||
})
|
||||
|
||||
this.websocketStatusService.onDocumentDeleted().subscribe(() => {
|
||||
this.list.reload()
|
||||
})
|
||||
this.websocketStatusService
|
||||
.onDocumentDeleted()
|
||||
.pipe(takeUntil(this.unsubscribeNotifier))
|
||||
.subscribe(() => {
|
||||
this.list.reload()
|
||||
})
|
||||
|
||||
this.route.paramMap
|
||||
.pipe(
|
||||
|
||||
@@ -15,3 +15,33 @@ export interface DocumentSuggestions {
|
||||
|
||||
dates?: string[] // ISO-formatted date string e.g. 2022-11-03
|
||||
}
|
||||
|
||||
const union = <T>(a: T[] = [], b: T[] = []): T[] => [...new Set([...a, ...b])]
|
||||
|
||||
export function mergeSuggestions(
|
||||
a: DocumentSuggestions,
|
||||
b: DocumentSuggestions
|
||||
): DocumentSuggestions {
|
||||
if (!a) return b
|
||||
return {
|
||||
title: a.title || b.title,
|
||||
tags: union(a.tags, b.tags),
|
||||
suggested_tags: union(a.suggested_tags, b.suggested_tags),
|
||||
correspondents: union(a.correspondents, b.correspondents),
|
||||
suggested_correspondents: union(
|
||||
a.suggested_correspondents,
|
||||
b.suggested_correspondents
|
||||
),
|
||||
document_types: union(a.document_types, b.document_types),
|
||||
suggested_document_types: union(
|
||||
a.suggested_document_types,
|
||||
b.suggested_document_types
|
||||
),
|
||||
storage_paths: union(a.storage_paths, b.storage_paths),
|
||||
suggested_storage_paths: union(
|
||||
a.suggested_storage_paths,
|
||||
b.suggested_storage_paths
|
||||
),
|
||||
dates: union(a.dates, b.dates),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -353,6 +353,14 @@ export const PaperlessConfigOptions: ConfigOption[] = [
|
||||
config_key: 'PAPERLESS_AI_LLM_EMBEDDING_MODEL',
|
||||
category: ConfigCategory.AI,
|
||||
},
|
||||
{
|
||||
key: 'llm_embedding_api_key',
|
||||
title: $localize`LLM Embedding API Key`,
|
||||
type: ConfigOptionType.Password,
|
||||
note: $localize`Used for embeddings when set, otherwise LLM API key is used.`,
|
||||
config_key: 'PAPERLESS_AI_LLM_EMBEDDING_API_KEY',
|
||||
category: ConfigCategory.AI,
|
||||
},
|
||||
{
|
||||
key: 'llm_embedding_endpoint',
|
||||
title: $localize`LLM Embedding Endpoint`,
|
||||
@@ -457,6 +465,7 @@ export interface PaperlessConfig extends ObjectWithId {
|
||||
ai_enabled: boolean
|
||||
llm_embedding_backend: string
|
||||
llm_embedding_model: string
|
||||
llm_embedding_api_key: string
|
||||
llm_embedding_endpoint: string
|
||||
llm_embedding_chunk_size: number
|
||||
llm_context_size: number
|
||||
|
||||
@@ -20,6 +20,12 @@ export enum GlobalSearchType {
|
||||
TITLE_CONTENT = 'title-content',
|
||||
}
|
||||
|
||||
export enum SuggestionSource {
|
||||
ML = 'ml',
|
||||
AI = 'ai',
|
||||
Both = 'both',
|
||||
}
|
||||
|
||||
export enum CollapsibleSection {
|
||||
ATTRIBUTES = 'attributes',
|
||||
}
|
||||
@@ -98,6 +104,8 @@ export const SETTINGS_KEYS = {
|
||||
'general-settings:document-editing:overlay-thumbnail',
|
||||
DOCUMENT_EDITING_AUTO_SUGGEST:
|
||||
'general-settings:document-editing:auto-suggest',
|
||||
DOCUMENT_EDITING_SUGGESTION_SOURCE:
|
||||
'general-settings:document-editing:suggestion-source',
|
||||
DOCUMENT_DETAILS_HIDDEN_FIELDS:
|
||||
'general-settings:document-details:hidden-fields',
|
||||
SEARCH_DB_ONLY: 'general-settings:search:db-only',
|
||||
@@ -326,6 +334,11 @@ export const SETTINGS: UiSetting[] = [
|
||||
type: 'boolean',
|
||||
default: true,
|
||||
},
|
||||
{
|
||||
key: SETTINGS_KEYS.DOCUMENT_EDITING_SUGGESTION_SOURCE,
|
||||
type: 'string',
|
||||
default: SuggestionSource.AI,
|
||||
},
|
||||
{
|
||||
key: SETTINGS_KEYS.DOCUMENT_DETAILS_HIDDEN_FIELDS,
|
||||
type: 'array',
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -74,6 +74,7 @@ import {
|
||||
clipboardCheckFill,
|
||||
clipboardFill,
|
||||
clockHistory,
|
||||
cpu,
|
||||
creditCard,
|
||||
dash,
|
||||
dashCircle,
|
||||
@@ -118,6 +119,7 @@ import {
|
||||
infoCircle,
|
||||
journalBookmarkFill,
|
||||
journals,
|
||||
lightbulb,
|
||||
link,
|
||||
list,
|
||||
listNested,
|
||||
@@ -322,6 +324,7 @@ const icons = {
|
||||
clipboardCheckFill,
|
||||
clipboardFill,
|
||||
clockHistory,
|
||||
cpu,
|
||||
cash,
|
||||
creditCard,
|
||||
dash,
|
||||
@@ -367,6 +370,7 @@ const icons = {
|
||||
infoCircle,
|
||||
journalBookmarkFill,
|
||||
journals,
|
||||
lightbulb,
|
||||
link,
|
||||
list,
|
||||
listNested,
|
||||
|
||||
@@ -292,6 +292,7 @@ a.btn-link:active,
|
||||
a.btn-link:focus-visible,
|
||||
.btn-link:hover,
|
||||
.btn-link:active,
|
||||
.btn-link.show,
|
||||
.btn-link:focus-visible {
|
||||
color: var(--pngx-primary-lighten-10) !important;
|
||||
.primary-light & {
|
||||
|
||||
@@ -25,10 +25,20 @@ export class PDFFindController {
|
||||
onIsPageVisible?: () => boolean
|
||||
}
|
||||
|
||||
export const LinkTarget = {
|
||||
NONE: 0,
|
||||
SELF: 1,
|
||||
BLANK: 2,
|
||||
PARENT: 3,
|
||||
TOP: 4,
|
||||
}
|
||||
|
||||
export class PDFLinkService {
|
||||
private document?: unknown
|
||||
private viewer?: unknown
|
||||
|
||||
constructor(readonly options: Record<string, unknown> = {}) {}
|
||||
|
||||
setDocument(document: unknown): void {
|
||||
this.document = document
|
||||
}
|
||||
|
||||
+151
@@ -0,0 +1,151 @@
|
||||
"""Fixtures available to every Paperless-ngx app.
|
||||
|
||||
Loaded automatically for every test path. Keep module-scope imports minimal:
|
||||
this file is imported for every session, so anything heavy belongs inside
|
||||
the fixture body that needs it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Generator
|
||||
from pathlib import Path
|
||||
|
||||
from django.contrib.auth.models import User
|
||||
from pytest_django.fixtures import Settings
|
||||
from rest_framework.test import APIClient
|
||||
|
||||
from paperless_testing.dirs import PaperlessDirs
|
||||
from paperless_testing.fakes.progress import FakeProgressManager
|
||||
|
||||
|
||||
@pytest.fixture(scope="session", autouse=True)
|
||||
def faker_session_locale() -> str:
|
||||
"""Pin Faker's locale so generated data does not follow the host locale.
|
||||
|
||||
The seed itself is left to pytest-randomly, which derives one per run.
|
||||
"""
|
||||
return "en_US"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _fast_password_hasher(settings: Settings) -> None:
|
||||
"""Hash test passwords with MD5 instead of Django's default PBKDF2.
|
||||
|
||||
PBKDF2 is deliberately slow, and every ``admin_user`` or
|
||||
``create_superuser`` call pays for it: about 600 ms each. No test depends
|
||||
on the hash format, only on ``check_password`` and on the stored value
|
||||
changing when the password does.
|
||||
"""
|
||||
settings.PASSWORD_HASHERS = ["django.contrib.auth.hashers.MD5PasswordHasher"]
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clear_content_type_caches() -> None:
|
||||
"""Clear Django's ContentType cache and guardian's lru_cache before each test.
|
||||
|
||||
Tests that delete and reinsert ContentType/Permission rows (e.g. the
|
||||
importer) corrupt both caches. Without this fixture a subsequent test on
|
||||
the same xdist worker sees stale ContentType objects and guardian raises
|
||||
MixedContentTypeError.
|
||||
"""
|
||||
from django.contrib.contenttypes.models import ContentType
|
||||
from guardian.shortcuts import clear_ct_cache
|
||||
|
||||
ContentType.objects.clear_cache()
|
||||
clear_ct_cache()
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clear_django_caches() -> None:
|
||||
"""Clear every configured cache before each test.
|
||||
|
||||
Cached values outlive the test that wrote them: the classifier keys its
|
||||
vectorized content on a hash of the content itself, so a second test
|
||||
generating the same fixture data takes the cache-hit path and never calls
|
||||
the code it is asserting against.
|
||||
"""
|
||||
from django.core.cache import caches
|
||||
|
||||
for cache in caches.all(initialized_only=False):
|
||||
cache.clear()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def paperless_dirs(
|
||||
tmp_path: Path,
|
||||
settings: Settings,
|
||||
) -> Generator[PaperlessDirs, None, None]:
|
||||
"""The standard temp directory layout, applied to Django settings."""
|
||||
from documents.search import reset_backend
|
||||
from paperless_testing.dirs import build_paperless_dirs
|
||||
from paperless_testing.dirs import dirs_settings
|
||||
|
||||
dirs = build_paperless_dirs(tmp_path)
|
||||
for name, value in dirs_settings(dirs).items():
|
||||
setattr(settings, name, value)
|
||||
|
||||
# Not directory settings, but they are needed alongside the layout by the
|
||||
# sanity checker tests.
|
||||
settings.IGNORABLE_FILES = {".DS_Store", "Thumbs.db", "desktop.ini"}
|
||||
settings.APP_LOGO = ""
|
||||
|
||||
reset_backend()
|
||||
yield dirs
|
||||
reset_backend()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def rest_api_client() -> APIClient:
|
||||
"""The basic DRF APIClient, unauthenticated."""
|
||||
from rest_framework.test import APIClient
|
||||
|
||||
return APIClient()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def regular_user(db: None) -> User:
|
||||
"""Unprivileged user for permission boundary tests."""
|
||||
from paperless_testing.factories import UserFactory
|
||||
|
||||
return UserFactory(username="regular")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def admin_client(rest_api_client: APIClient, admin_user: User) -> APIClient:
|
||||
"""Admin client pre-authenticated and sending the v10 Accept header."""
|
||||
rest_api_client.force_authenticate(user=admin_user)
|
||||
rest_api_client.credentials(HTTP_ACCEPT="application/json; version=10")
|
||||
return rest_api_client
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def v9_client(rest_api_client: APIClient, admin_user: User) -> APIClient:
|
||||
"""Admin client pre-authenticated and sending the v9 Accept header."""
|
||||
rest_api_client.force_authenticate(user=admin_user)
|
||||
rest_api_client.credentials(HTTP_ACCEPT="application/json; version=9")
|
||||
return rest_api_client
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def user_client(rest_api_client: APIClient, regular_user: User) -> APIClient:
|
||||
"""Regular-user client pre-authenticated and sending the v10 Accept header."""
|
||||
rest_api_client.force_authenticate(user=regular_user)
|
||||
rest_api_client.credentials(HTTP_ACCEPT="application/json; version=10")
|
||||
return rest_api_client
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_progress_manager(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> type[FakeProgressManager]:
|
||||
"""Replace documents.tasks.ProgressManager with the fake, so consuming a file
|
||||
in a test never tries to reach a broker."""
|
||||
from paperless_testing.fakes.progress import FakeProgressManager
|
||||
|
||||
monkeypatch.setattr("documents.tasks.ProgressManager", FakeProgressManager)
|
||||
return FakeProgressManager
|
||||
@@ -26,7 +26,6 @@ class DocumentsConfig(AppConfig):
|
||||
document_consumption_finished.connect(set_document_type)
|
||||
document_consumption_finished.connect(set_tags)
|
||||
document_consumption_finished.connect(set_storage_path)
|
||||
document_consumption_finished.connect(add_to_index)
|
||||
document_consumption_finished.connect(run_workflows_added)
|
||||
document_consumption_finished.connect(add_to_index)
|
||||
document_consumption_finished.connect(add_or_update_document_in_llm_index)
|
||||
|
||||
@@ -857,8 +857,9 @@ class ConsumerPlugin(
|
||||
self.log.debug(f"Creation date from parse_date: {create_date}")
|
||||
else:
|
||||
stats = Path(self.input_doc.original_file).stat()
|
||||
create_date = timezone.make_aware(
|
||||
datetime.datetime.fromtimestamp(stats.st_mtime),
|
||||
create_date = datetime.datetime.fromtimestamp(
|
||||
stats.st_mtime,
|
||||
tz=timezone.get_current_timezone(),
|
||||
)
|
||||
self.log.debug(f"Creation date from st_mtime: {create_date}")
|
||||
|
||||
|
||||
@@ -196,52 +196,49 @@ class WriteBatch:
|
||||
return self._raw_writer
|
||||
|
||||
def __enter__(self) -> Self:
|
||||
if self._backend._path is not None:
|
||||
lock_path = self._backend._path / ".tantivy.lock"
|
||||
self._lock = filelock.FileLock(str(lock_path))
|
||||
for attempt in range(_LOCK_RETRY_ATTEMPTS):
|
||||
try:
|
||||
self._lock.acquire(timeout=self._lock_timeout)
|
||||
break
|
||||
except filelock.Timeout:
|
||||
if attempt == _LOCK_RETRY_ATTEMPTS - 1:
|
||||
raise SearchIndexLockError(
|
||||
f"Could not acquire index lock after {_LOCK_RETRY_ATTEMPTS} "
|
||||
f"attempts (timeout={self._lock_timeout}s each)",
|
||||
)
|
||||
sleep_s = random.uniform(
|
||||
0,
|
||||
min(_LOCK_BACKOFF_CAP, _LOCK_BACKOFF_BASE * (2**attempt)),
|
||||
lock_path = self._backend._path / ".tantivy.lock"
|
||||
self._lock = filelock.FileLock(str(lock_path))
|
||||
for attempt in range(_LOCK_RETRY_ATTEMPTS):
|
||||
try:
|
||||
self._lock.acquire(timeout=self._lock_timeout)
|
||||
break
|
||||
except filelock.Timeout:
|
||||
if attempt == _LOCK_RETRY_ATTEMPTS - 1:
|
||||
raise SearchIndexLockError(
|
||||
f"Could not acquire index lock after {_LOCK_RETRY_ATTEMPTS} "
|
||||
f"attempts (timeout={self._lock_timeout}s each)",
|
||||
)
|
||||
logger.debug(
|
||||
"Index lock contention; retrying in %.2fs (attempt %d/%d)",
|
||||
sleep_s,
|
||||
attempt + 1,
|
||||
_LOCK_RETRY_ATTEMPTS,
|
||||
)
|
||||
time.sleep(sleep_s)
|
||||
sleep_s = random.uniform(
|
||||
0,
|
||||
min(_LOCK_BACKOFF_CAP, _LOCK_BACKOFF_BASE * (2**attempt)),
|
||||
)
|
||||
logger.debug(
|
||||
"Index lock contention; retrying in %.2fs (attempt %d/%d)",
|
||||
sleep_s,
|
||||
attempt + 1,
|
||||
_LOCK_RETRY_ATTEMPTS,
|
||||
)
|
||||
time.sleep(sleep_s)
|
||||
|
||||
# Open a fresh Index (and thus a fresh Tantivy ManagedDirectory)
|
||||
# for the write, rather than reusing the process-local cached
|
||||
# index. ManagedDirectory loads its GC bookkeeping (.managed.json)
|
||||
# once, at construction, and never re-reads it; paperless runs
|
||||
# several long-lived processes (Granian workers, Celery workers)
|
||||
# that take turns writing under the file lock above. A cached,
|
||||
# long-lived writer index would carry a stale managed-files view
|
||||
# and, on commit, overwrite .managed.json with that stale view -
|
||||
# permanently losing track of segment files other processes
|
||||
# registered in the meantime, so they can never be garbage
|
||||
# collected. Reopening fresh here always picks up the current
|
||||
# on-disk state. The long-lived self._backend._index is used for
|
||||
# reads only and is reloaded (not reopened) after commit below.
|
||||
write_index = tantivy.Index(
|
||||
build_schema(),
|
||||
path=str(self._backend._path),
|
||||
)
|
||||
register_tokenizers(write_index, settings.SEARCH_LANGUAGE)
|
||||
self._raw_writer = write_index.writer()
|
||||
else:
|
||||
self._raw_writer = self._backend._index.writer()
|
||||
# Open a fresh Index (and thus a fresh Tantivy ManagedDirectory)
|
||||
# for the write, rather than reusing the process-local cached
|
||||
# index. ManagedDirectory loads its GC bookkeeping (.managed.json)
|
||||
# once, at construction, and never re-reads it; paperless runs
|
||||
# several long-lived processes (Granian workers, Celery workers)
|
||||
# that take turns writing under the file lock above. A cached,
|
||||
# long-lived writer index would carry a stale managed-files view
|
||||
# and, on commit, overwrite .managed.json with that stale view -
|
||||
# permanently losing track of segment files other processes
|
||||
# registered in the meantime, so they can never be garbage
|
||||
# collected. Reopening fresh here always picks up the current
|
||||
# on-disk state. The long-lived self._backend._index is used for
|
||||
# reads only and is reloaded (not reopened) after commit below.
|
||||
write_index = tantivy.Index(
|
||||
build_schema(),
|
||||
path=str(self._backend._path),
|
||||
)
|
||||
register_tokenizers(write_index, settings.SEARCH_LANGUAGE)
|
||||
self._raw_writer = write_index.writer()
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
@@ -372,9 +369,8 @@ class TantivyBackend:
|
||||
Tantivy search backend with explicit lifecycle management.
|
||||
|
||||
Provides full-text search capabilities using the Tantivy search engine.
|
||||
Supports in-memory indexes (for testing) and persistent on-disk indexes
|
||||
(for production use). Handles document indexing, search queries, autocompletion,
|
||||
and "more like this" functionality.
|
||||
Keeps a persistent on-disk index. Handles document indexing, search queries,
|
||||
autocompletion, and "more like this" functionality.
|
||||
|
||||
The backend manages its own connection lifecycle and can be reset when
|
||||
the underlying index directory changes (e.g., during test isolation).
|
||||
@@ -408,9 +404,7 @@ class TantivyBackend:
|
||||
},
|
||||
)
|
||||
|
||||
def __init__(self, path: Path | None = None):
|
||||
# path=None → in-memory index (for tests)
|
||||
# path=some_dir → on-disk index (for production)
|
||||
def __init__(self, path: Path):
|
||||
self._path = path
|
||||
self._raw_index: tantivy.Index | None = None
|
||||
self._raw_schema: tantivy.Schema | None = None
|
||||
@@ -429,16 +423,13 @@ class TantivyBackend:
|
||||
"""
|
||||
Open or rebuild the index as needed.
|
||||
|
||||
For disk-based indexes, checks if rebuilding is needed due to schema
|
||||
version or language changes. Registers custom tokenizers after opening.
|
||||
Checks if rebuilding is needed due to schema version or language
|
||||
changes. Registers custom tokenizers after opening.
|
||||
Safe to call multiple times - subsequent calls are no-ops.
|
||||
"""
|
||||
if self._raw_index is not None:
|
||||
return # pragma: no cover
|
||||
if self._path is not None:
|
||||
self._raw_index = open_or_rebuild_index(self._path)
|
||||
else:
|
||||
self._raw_index = tantivy.Index(build_schema())
|
||||
self._raw_index = open_or_rebuild_index(self._path)
|
||||
register_tokenizers(self._raw_index, settings.SEARCH_LANGUAGE)
|
||||
self._raw_schema = self._raw_index.schema
|
||||
|
||||
@@ -1102,13 +1093,9 @@ class TantivyBackend:
|
||||
writer's threads). Larger values buffer more docs in RAM before
|
||||
flushing a segment, deferring merge work; they do not avoid it.
|
||||
"""
|
||||
# Create new index (on-disk or in-memory)
|
||||
if self._path is not None:
|
||||
wipe_index(self._path)
|
||||
new_index = tantivy.Index(build_schema(), path=str(self._path))
|
||||
_write_sentinels(self._path)
|
||||
else:
|
||||
new_index = tantivy.Index(build_schema())
|
||||
wipe_index(self._path)
|
||||
new_index = tantivy.Index(build_schema(), path=str(self._path))
|
||||
_write_sentinels(self._path)
|
||||
register_tokenizers(new_index, settings.SEARCH_LANGUAGE)
|
||||
|
||||
# Point instance at the new index so _build_tantivy_doc uses it
|
||||
|
||||
@@ -1079,6 +1079,16 @@ class DocumentSerializer(
|
||||
)
|
||||
|
||||
def get_page_count(self, obj) -> int | None:
|
||||
# Like content versions get their own page count from the newest version,
|
||||
# use the prefetched versions cache to avoid an extra query
|
||||
prefetched_cache = getattr(obj, "_prefetched_objects_cache", None)
|
||||
prefetched_versions = (
|
||||
prefetched_cache.get("versions")
|
||||
if isinstance(prefetched_cache, dict)
|
||||
else None
|
||||
)
|
||||
if obj.root_document_id is None and prefetched_versions:
|
||||
return sort_versions_newest_first(prefetched_versions)[0].page_count
|
||||
return obj.page_count
|
||||
|
||||
@extend_schema_field(DuplicateDocumentSummarySerializer(many=True))
|
||||
@@ -2098,6 +2108,8 @@ class BulkEditSerializer(
|
||||
if not isinstance(parameters["pages"], str):
|
||||
raise serializers.ValidationError("invalid pages specified")
|
||||
page_count = Document.objects.get(id=document_id).page_count
|
||||
if not page_count:
|
||||
raise serializers.ValidationError("document page count is unknown")
|
||||
pages = []
|
||||
for group in parameters["pages"].split(","):
|
||||
start, is_range, end = group.partition("-")
|
||||
@@ -2107,7 +2119,7 @@ class BulkEditSerializer(
|
||||
except ValueError as e:
|
||||
raise serializers.ValidationError("invalid pages specified") from e
|
||||
# Bound the range before building it, a huge one would exhaust memory
|
||||
if not 1 <= first <= last or (page_count and last > page_count):
|
||||
if not 1 <= first <= last <= page_count:
|
||||
raise serializers.ValidationError("invalid pages specified")
|
||||
pages.append(list(range(first, last + 1)))
|
||||
parameters["pages"] = pages
|
||||
|
||||
@@ -56,6 +56,7 @@ from documents.permissions import get_objects_for_user_owner_aware
|
||||
from documents.plugins.helpers import DocumentsStatusManager
|
||||
from documents.templating.utils import convert_format_str_to_template_format
|
||||
from documents.utils import compute_checksum
|
||||
from documents.utils import copy_file_with_basic_stats
|
||||
from documents.workflows.actions import build_workflow_action_context
|
||||
from documents.workflows.actions import execute_email_action
|
||||
from documents.workflows.actions import execute_move_to_trash_action
|
||||
@@ -363,7 +364,11 @@ def cleanup_document_deletion(sender, instance, **kwargs) -> None:
|
||||
|
||||
logger.debug(f"Moving {instance.source_path} to trash at {new_file_path}")
|
||||
try:
|
||||
shutil.move(instance.source_path, new_file_path)
|
||||
shutil.move(
|
||||
instance.source_path,
|
||||
new_file_path,
|
||||
copy_function=copy_file_with_basic_stats,
|
||||
)
|
||||
except OSError as e:
|
||||
logger.error(
|
||||
f"Failed to move {instance.source_path} to trash at "
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user