diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 943fed8cd..56495186f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -58,6 +58,7 @@ repos: rev: "v2.24.1" hooks: - id: pyproject-fmt + additional_dependencies: [tomli] # Dockerfile hooks - repo: https://github.com/AleksaC/hadolint-py rev: v2.14.0 diff --git a/docker/compose/docker-compose.mariadb-tika.yml b/docker/compose/docker-compose.mariadb-tika.yml index eaae67157..54c7c1811 100644 --- a/docker/compose/docker-compose.mariadb-tika.yml +++ b/docker/compose/docker-compose.mariadb-tika.yml @@ -30,7 +30,7 @@ # documentation. services: broker: - image: docker.io/library/redis:8 + image: docker.io/valkey/valkey:9-alpine restart: unless-stopped volumes: - redisdata:/data diff --git a/docker/compose/docker-compose.mariadb.yml b/docker/compose/docker-compose.mariadb.yml index 2265703da..8f828d3b1 100644 --- a/docker/compose/docker-compose.mariadb.yml +++ b/docker/compose/docker-compose.mariadb.yml @@ -26,7 +26,7 @@ # documentation. services: broker: - image: docker.io/library/redis:8 + image: docker.io/valkey/valkey:9-alpine restart: unless-stopped volumes: - redisdata:/data diff --git a/docker/compose/docker-compose.portainer.yml b/docker/compose/docker-compose.portainer.yml index a44b4b692..0d2fbec11 100644 --- a/docker/compose/docker-compose.portainer.yml +++ b/docker/compose/docker-compose.portainer.yml @@ -27,7 +27,7 @@ # documentation. services: broker: - image: docker.io/library/redis:8 + image: docker.io/valkey/valkey:9-alpine restart: unless-stopped volumes: - redisdata:/data diff --git a/docker/compose/docker-compose.postgres-tika.yml b/docker/compose/docker-compose.postgres-tika.yml index e988a537b..754264afc 100644 --- a/docker/compose/docker-compose.postgres-tika.yml +++ b/docker/compose/docker-compose.postgres-tika.yml @@ -30,7 +30,7 @@ # documentation. services: broker: - image: docker.io/library/redis:8 + image: docker.io/valkey/valkey:9-alpine restart: unless-stopped volumes: - redisdata:/data diff --git a/docker/compose/docker-compose.postgres.yml b/docker/compose/docker-compose.postgres.yml index 5ce15f463..705432b67 100644 --- a/docker/compose/docker-compose.postgres.yml +++ b/docker/compose/docker-compose.postgres.yml @@ -26,7 +26,7 @@ # documentation. services: broker: - image: docker.io/library/redis:8 + image: docker.io/valkey/valkey:9-alpine restart: unless-stopped volumes: - redisdata:/data diff --git a/docker/compose/docker-compose.sqlite-tika.yml b/docker/compose/docker-compose.sqlite-tika.yml index d3ef2630a..e8143216d 100644 --- a/docker/compose/docker-compose.sqlite-tika.yml +++ b/docker/compose/docker-compose.sqlite-tika.yml @@ -30,7 +30,7 @@ # documentation. services: broker: - image: docker.io/library/redis:8 + image: docker.io/valkey/valkey:9-alpine restart: unless-stopped volumes: - redisdata:/data diff --git a/docker/compose/docker-compose.sqlite.yml b/docker/compose/docker-compose.sqlite.yml index 37c7f0c46..1688c53c0 100644 --- a/docker/compose/docker-compose.sqlite.yml +++ b/docker/compose/docker-compose.sqlite.yml @@ -23,7 +23,7 @@ # documentation. services: broker: - image: docker.io/library/redis:8 + image: docker.io/valkey/valkey:9-alpine restart: unless-stopped volumes: - redisdata:/data diff --git a/docs/administration.md b/docs/administration.md index c876468c8..9b6bed961 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -65,6 +65,11 @@ copies you created in the steps above. Please review the [migration instructions](migration-v3.md) before upgrading Paperless-ngx to v3.0, it includes some breaking changes that require manual intervention before upgrading. +!!! note + + Upgrading to v3 clears the existing task history; previously completed, failed, or + acknowledged tasks will no longer appear in the task list afterward. No action is required. + ### Docker Route {#docker-updating} If a new release of paperless-ngx is available, upgrading depends on how @@ -500,6 +505,33 @@ task scheduler. python3 manage.py document_index reindex --if-needed ``` +### Managing the LLM (AI) index {#llm-index} + +When the [AI features](advanced_usage.md#ai-features) are enabled with an embedding +backend, Paperless-ngx maintains a vector index of your documents used for +Retrieval-Augmented Generation (RAG), similar-document retrieval, and document chat. The +index is updated automatically on the schedule set by +[`PAPERLESS_LLM_INDEX_TASK_CRON`](configuration.md#PAPERLESS_LLM_INDEX_TASK_CRON), but you +can manage it manually: + +``` +document_llmindex {rebuild,update,compact} +``` + +Specify `rebuild` to build the index from scratch from all documents in the database. Use +this the first time you enable the feature, or after changing the embedding backend or +model. + +Specify `update` to incrementally index new and changed documents. This is what the +scheduled task runs. + +Specify `compact` to reclaim space and optimize the on-disk vector store. + +!!! note + + These commands have no effect unless AI is enabled and an embedding backend is + configured. + ### Clearing the database read cache If the database read cache is enabled, **you must run this command** after making any changes to the database outside the application context. diff --git a/docs/advanced_usage.md b/docs/advanced_usage.md index a07005320..74e31b454 100644 --- a/docs/advanced_usage.md +++ b/docs/advanced_usage.md @@ -97,6 +97,85 @@ when using this feature: of these correspondents to ANY new document, if both are set to automatic matching. +## AI features {#ai-features} + +Paperless-ngx includes a set of optional features backed by a large language model +(LLM): AI-assisted suggestions, similar-document retrieval, and a document chat. They +are **off by default** and never replace the built-in, non-LLM +[matching and suggestions](#matching). + +!!! warning + + Enabling these features sends document content (and metadata) to the LLM backend you + configure. If that backend is a remote/hosted provider, your documents leave your + server and may incur usage charges. Consider the privacy implications before enabling, + and prefer a local backend (Ollama, or a self-hosted OpenAI-compatible gateway) if that + matters to you. + +All AI settings can be supplied as `PAPERLESS_AI_*` environment variables (see +[configuration](configuration.md#ai)) or set in the admin under +**Settings → Application Configuration**; the database value takes precedence over the +environment. + +### Enabling the AI features + +At a minimum you need to enable AI and choose an LLM backend: + +- [`PAPERLESS_AI_ENABLED`](configuration.md#PAPERLESS_AI_ENABLED) — master switch. +- [`PAPERLESS_AI_LLM_BACKEND`](configuration.md#PAPERLESS_AI_LLM_BACKEND) — `ollama` + (runs locally) or `openai-like` (OpenAI itself or any OpenAI-compatible API). +- [`PAPERLESS_AI_LLM_MODEL`](configuration.md#PAPERLESS_AI_LLM_MODEL), and for + `openai-like` usually [`PAPERLESS_AI_LLM_API_KEY`](configuration.md#PAPERLESS_AI_LLM_API_KEY) + and/or [`PAPERLESS_AI_LLM_ENDPOINT`](configuration.md#PAPERLESS_AI_LLM_ENDPOINT). Ollama + requires `PAPERLESS_AI_LLM_ENDPOINT` pointing at your Ollama server. + +### AI-assisted suggestions + +With AI enabled, Paperless-ngx can suggest a title, tags, correspondent, document type, +storage path and dates by sending the document to the LLM. This is **opt-in per request** +and surfaces through the "Suggest" control on the document detail page, alongside the +classic classifier-based suggestions — it does not disable them. Suggestion output +language can be steered with +[`PAPERLESS_AI_LLM_OUTPUT_LANGUAGE`](configuration.md#PAPERLESS_AI_LLM_OUTPUT_LANGUAGE) +(otherwise it follows the user's UI language). + +### The LLM index (RAG) and similar documents + +Setting an embedding backend turns on the **LLM index**, a vector index of your documents +that enables Retrieval-Augmented Generation (RAG). When enabled, suggestions are grounded +in similar existing documents, and the document chat can retrieve relevant context. + +Enable it by setting +[`PAPERLESS_AI_LLM_EMBEDDING_BACKEND`](configuration.md#PAPERLESS_AI_LLM_EMBEDDING_BACKEND) +(`huggingface` for fully-local embeddings, or `ollama` / `openai-like`). The index is only +built when AI is enabled **and** an embedding backend is set. + +The index is updated automatically on a schedule controlled by +[`PAPERLESS_LLM_INDEX_TASK_CRON`](configuration.md#PAPERLESS_LLM_INDEX_TASK_CRON) (daily by +default), and can be rebuilt or compacted manually — see +[Managing the LLM index](administration.md#llm-index). + +!!! note + + Local embeddings via `huggingface` download the embedding model on first use into the + Paperless data directory. The first run therefore needs network access and some disk + space. + +### Document chat + +When the LLM index is enabled, the chat control in the top app toolbar answers questions +about your documents. It operates over a single document or across multiple documents +depending on the current view, and its answers include links to the source documents it +drew from. + +### AI Security notes + +- Document content is passed to the LLM as **untrusted data**. +- By default Paperless-ngx allows AI endpoints that resolve to private/loopback addresses + (for local backends). Set + [`PAPERLESS_AI_LLM_ALLOW_INTERNAL_ENDPOINTS`](configuration.md#PAPERLESS_AI_LLM_ALLOW_INTERNAL_ENDPOINTS) + to `false` to block them. + ## Hooking into the consumption process {#consume-hooks} Sometimes you may want to do something arbitrary whenever a document is @@ -846,7 +925,7 @@ Paperless is able to utilize barcodes for automatically performing some tasks. B At this time, the library utilized for detection of barcodes supports the following types: -- AN-13/UPC-A +- EAN-13/UPC-A - UPC-E - EAN-8 - Code 128 @@ -855,7 +934,9 @@ At this time, the library utilized for detection of barcodes supports the follow - Codabar - Interleaved 2 of 5 - QR Code -- SQ Code +- Data Matrix +- Aztec +- PDF417 For usage in Paperless, the type of barcode does not matter, only the contents of it. diff --git a/docs/api.md b/docs/api.md index af1190f3d..b8ba5fc35 100644 --- a/docs/api.md +++ b/docs/api.md @@ -227,6 +227,7 @@ Version-aware endpoints: - `PATCH /api/documents/{id}/`: content updates target the selected version (`?version={version_id}`) or latest version by default; non-content metadata updates target the root document. - `GET /api/documents/{id}/download/`, `GET /api/documents/{id}/preview/`, `GET /api/documents/{id}/thumb/`, `GET /api/documents/{id}/metadata/`: accept `?version={version_id}`. - `POST /api/documents/{id}/update_version/`: uploads a new version using multipart form field `document` and optional `version_label`. +- `PATCH /api/documents/{id}/versions/{version_id}/`: updates the `version_label` of a specific version. - `DELETE /api/documents/{root_id}/versions/{version_id}/`: deletes a non-root version. ## Permissions @@ -445,3 +446,9 @@ Initial API version. large lists of object IDs for operations affecting many objects. - The legacy `title_content` document search parameter is deprecated and will be removed in a future version. Clients should use `text` for simple title-and-content search and `title_search` for title-only search. +- The task tracking system was redesigned. The tasks list (`/api/tasks/`) is now paginated, and the + task object exposes `task_type` (formerly `task_name`) and `trigger_source` (formerly `type`). New + read-only endpoints `/api/tasks/summary/`, `/api/tasks/status_counts/`, and `/api/tasks/active/` + provide aggregate views, and `POST /api/tasks/run/` lets privileged users dispatch supported tasks. + API v9 continues to serve the unpaginated list with the legacy field names until support for v9 is + dropped. diff --git a/docs/configuration.md b/docs/configuration.md index 9780aa94d..e400492b1 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -22,7 +22,11 @@ or applicable default will be utilized instead. ## Required services -### Redis Broker +### Message Broker + +Paperless-ngx uses a Redis-compatible message broker. Any broker that +speaks the Redis protocol works here, including [Valkey](https://valkey.io/) +(the default in the bundled Docker Compose files) and Redis itself. #### [`PAPERLESS_REDIS=`](#PAPERLESS_REDIS) {#PAPERLESS_REDIS} @@ -30,21 +34,21 @@ or applicable default will be utilized instead. fetching, index optimization and for training the automatic document matcher. - - If your Redis server needs login credentials PAPERLESS_REDIS = + - If your broker needs login credentials PAPERLESS_REDIS = `redis://:@:` - With the requirepass option PAPERLESS_REDIS = `redis://:@:` - - To include the redis database index PAPERLESS_REDIS = + - To include the database index PAPERLESS_REDIS = `redis://:@:/` - [More information on securing your Redis - Instance](https://redis.io/docs/latest/operate/oss_and_stack/management/security). + [More information on securing your broker + instance](https://valkey.io/topics/security/). Defaults to `redis://localhost:6379`. #### [`PAPERLESS_REDIS_PREFIX=`](#PAPERLESS_REDIS_PREFIX) {#PAPERLESS_REDIS_PREFIX} -: Prefix to be used in Redis for keys and channels. Useful for sharing one Redis server among multiple Paperless instances. +: Prefix to be used in the broker for keys and channels. Useful for sharing one broker among multiple Paperless instances. Defaults to no prefix. @@ -58,14 +62,14 @@ and the relevant connection variables. #### [`PAPERLESS_DBENGINE=`](#PAPERLESS_DBENGINE) {#PAPERLESS_DBENGINE} : Specifies the database engine to use. Accepted values are `sqlite`, `postgresql`, -and `mariadb`. - - Defaults to `sqlite` if not set. +and `mariadb`. PostgreSQL and MariaDB users must set this explicitly. PostgreSQL and MariaDB both require [`PAPERLESS_DBHOST`](#PAPERLESS_DBHOST) to be set. SQLite does not use any other connection variables; the database file is always located at `/db.sqlite3`. + Defaults to `sqlite`. + !!! warning Using MariaDB comes with some caveats. See [MySQL Caveats](advanced_usage.md#mysql-caveats). @@ -238,7 +242,7 @@ dictionaries; for example, `pool.max_size=20` sets #### [`PAPERLESS_DB_READ_CACHE_ENABLED=`](#PAPERLESS_DB_READ_CACHE_ENABLED) {#PAPERLESS_DB_READ_CACHE_ENABLED} -: Caches the database read query results into Redis. This can significantly improve application response times by caching database queries, at the cost of slightly increased memory usage. +: Caches the database read query results into the broker. This can significantly improve application response times by caching database queries, at the cost of slightly increased memory usage. Defaults to `false`. @@ -258,18 +262,18 @@ dictionaries; for example, `pool.max_size=20` sets A high TTL increases memory usage over time. Memory may be used until end of TTL, even if the cache is invalidated with the `invalidate_cachalot` command. -In case of an out-of-memory (OOM) situation, Redis may stop accepting new data — including cache entries, scheduled tasks, and documents to consume. -If your system has limited RAM, consider configuring a dedicated Redis instance for the read cache, with a memory limit and the eviction policy set to `allkeys-lru`. -For more details, refer to the [Redis eviction policy documentation](https://redis.io/docs/latest/develop/reference/eviction/), and see the `PAPERLESS_READ_CACHE_REDIS_URL` setting to specify a separate Redis broker. +In case of an out-of-memory (OOM) situation, the broker may stop accepting new data — including cache entries, scheduled tasks, and documents to consume. +If your system has limited RAM, consider configuring a dedicated broker instance for the read cache, with a memory limit and the eviction policy set to `allkeys-lru`. +For more details, refer to the [Redis eviction policy documentation](https://redis.io/docs/latest/develop/reference/eviction/), and see the `PAPERLESS_READ_CACHE_REDIS_URL` setting to specify a separate broker. #### [`PAPERLESS_READ_CACHE_REDIS_URL=`](#PAPERLESS_READ_CACHE_REDIS_URL) {#PAPERLESS_READ_CACHE_REDIS_URL} -: Defines the Redis instance used for the read cache. +: Defines the broker instance used for the read cache. Defaults to `None`. !!! Note - If this value is not set, the same Redis instance used for scheduled tasks will be used for caching as well. + If this value is not set, the same broker instance used for scheduled tasks will be used for caching as well. ## Optional Services @@ -888,7 +892,7 @@ modes are available: The default is `auto`. - For the `skip`, `redo`, and `force` modes, read more about OCR + For the `redo` and `force` modes, read more about OCR behaviour in the [OCRmyPDF documentation](https://ocrmypdf.readthedocs.io/en/latest/advanced.html#when-ocr-is-skipped). @@ -2068,6 +2072,13 @@ context by default. Defaults to 8192. +#### [`PAPERLESS_AI_LLM_REQUEST_TIMEOUT=`](#PAPERLESS_AI_LLM_REQUEST_TIMEOUT) {#PAPERLESS_AI_LLM_REQUEST_TIMEOUT} + +: The timeout, in seconds, for requests to the configured AI backend. Increase this when using +local or slow inference servers that need more time to generate responses. + + Defaults to 120. + #### [`PAPERLESS_AI_LLM_BACKEND=`](#PAPERLESS_AI_LLM_BACKEND) {#PAPERLESS_AI_LLM_BACKEND} : The AI backend to use. This can be either "openai-like" or "ollama". If set to "ollama", the AI @@ -2120,7 +2131,7 @@ used with the OpenAI-compatible backend to target a custom provider or local gat Defaults to true, which allows internal endpoints. -#### [`PAPERLESS_AI_LLM_INDEX_TASK_CRON=`](#PAPERLESS_AI_LLM_INDEX_TASK_CRON) {#PAPERLESS_AI_LLM_INDEX_TASK_CRON} +#### [`PAPERLESS_LLM_INDEX_TASK_CRON=`](#PAPERLESS_LLM_INDEX_TASK_CRON) {#PAPERLESS_LLM_INDEX_TASK_CRON} : Configures the schedule to update the AI embeddings of text content and metadata for all documents. Only performed if AI is enabled and the LLM embedding backend is set. diff --git a/docs/development.md b/docs/development.md index 11e078a67..46b84f703 100644 --- a/docs/development.md +++ b/docs/development.md @@ -94,16 +94,16 @@ first-time setup. ``` 7. You can now either ... - - install Redis or + - install a Redis-compatible broker (e.g. Valkey or Redis) or - use the included `scripts/start_services.sh` to use Docker to fire - up a Redis instance (and some other services such as Tika, + up a broker instance (and some other services such as Tika, Gotenberg and a database server) or - - spin up a bare Redis container + - spin up a bare broker container ```bash - docker run -d -p 6379:6379 --restart unless-stopped redis:latest + docker run -d -p 6379:6379 --restart unless-stopped docker.io/valkey/valkey:9-alpine ``` 8. Continue with either back-end or front-end development – or both :-). @@ -132,7 +132,7 @@ uv run manage.py runserver & \ ``` You might need the front end to test your back end code. -This assumes that you have AngularJS installed on your system. +This assumes that you have Angular installed on your system. Go to the [Front end development](#front-end-development) section for further details. To build the front end once use this command: @@ -174,7 +174,7 @@ To add a new development package `uv add --dev ` ## Front end development -The front end is built using AngularJS. In order to get started, you need Node.js (version 24+) and +The front end is built using Angular. In order to get started, you need Node.js (version 24+) and `pnpm`. !!! note @@ -248,12 +248,12 @@ that authentication is working. ## Localization Paperless-ngx is available in many different languages. Since Paperless-ngx -consists both of a Django application and an AngularJS front end, both +consists both of a Django application and an Angular front end, both these parts have to be translated separately. ### Front end localization -- The AngularJS front end does localization according to the [Angular +- The Angular front end does localization according to the [Angular documentation](https://angular.io/guide/i18n). - The source language of the project is "en_US". - The source strings end up in the file `src-ui/messages.xlf`. @@ -495,7 +495,7 @@ class MyCustomParser: self._tempdir = Path( tempfile.mkdtemp(prefix="paperless-", dir=settings.SCRATCH_DIR) ) - self._text: str | None = None + self._text: str = "" self._archive_path: Path | None = None def __enter__(self) -> Self: @@ -553,7 +553,8 @@ def parse( **Result accessors** ```python -def get_text(self) -> str | None: +def get_text(self) -> str: + # Return the extracted text, or an empty string if none was found. return self._text def get_date(self) -> "datetime.datetime | None": @@ -684,7 +685,7 @@ class XmlDocumentParser: def __init__(self, logging_group: object = None) -> None: settings.SCRATCH_DIR.mkdir(parents=True, exist_ok=True) self._tempdir = Path(tempfile.mkdtemp(prefix="paperless-", dir=settings.SCRATCH_DIR)) - self._text: str | None = None + self._text: str = "" def __enter__(self) -> Self: return self @@ -702,7 +703,7 @@ class XmlDocumentParser: except ET.ParseError as e: raise ParseError(f"XML parse error: {e}") from e - def get_text(self) -> str | None: + def get_text(self) -> str: return self._text def get_date(self): diff --git a/docs/faq.md b/docs/faq.md index 2ef67222b..a5ec3c154 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -70,7 +70,16 @@ elsewhere. Here are a couple notes about that. Paperless-ngx determines the type of a file by inspecting its content rather than its file extensions. However, files processed via the consumption directory will be rejected if they have a file extension that -not supported by any of the available parsers. +is not supported by any of the available parsers. + +## _Are duplicate documents rejected?_ + +**A:** Not by default. As of v3, a file whose contents match an existing document is still +consumed, and the duplicate is flagged in the UI — open the document and check the +**Duplicates** tab to review documents that share the same content. If you prefer the old +behavior of rejecting duplicates during consumption, set +[`PAPERLESS_CONSUMER_DELETE_DUPLICATES`](configuration.md#PAPERLESS_CONSUMER_DELETE_DUPLICATES) +to `true`. ## _Will paperless-ngx run on Raspberry Pi?_ @@ -118,10 +127,24 @@ able to run paperless, you're a bit on your own. If you can't run the docker image, the documentation has instructions for bare metal installs. -## _What about the Redis licensing change and using one of the open source forks_? +## _Does Paperless-ngx use AI, and is my data private?_ -Currently (October 2024), forks of Redis such as Valkey or Redirect are not officially supported by our upstream -libraries, so using one of these to replace Redis is not officially supported. +**A:** Paperless-ngx includes optional AI features — LLM-based suggestions, document chat, +and similar-document retrieval — that are **disabled by default**. They only run when you +enable them and configure an LLM backend. The built-in tag/correspondent suggestions use a +local, non-LLM machine-learning model and do not send your data anywhere. If you enable the +LLM features, document content is sent to whichever backend you configure — this can be a +fully local backend (e.g. Ollama) or a remote provider. See +[AI features](advanced_usage.md#ai-features) for details. -However, they do claim to be compatible with the Redis protocol and will likely work, but we will -not be updating from using Redis as the broker officially just yet. +## _Which message broker should I use_? + +Paperless-ngx talks to a Redis-compatible message broker, so any broker that +implements the Redis protocol will work. The bundled Docker Compose files +default to [Valkey](https://valkey.io/), the open-source fork created after +Redis' licensing change, but Redis itself and other wire-compatible brokers +(such as Microsoft's Garnet) are equally fine. + +Existing installs can switch broker implementations in place: point +[`PAPERLESS_REDIS`](configuration.md#PAPERLESS_REDIS) at the new instance and +reuse the same data volume. diff --git a/docs/index.md b/docs/index.md index a2fead850..f2dcb3cfc 100644 --- a/docs/index.md +++ b/docs/index.md @@ -35,9 +35,10 @@ physical documents into a searchable online archive so you can keep, well, _less - _New!_ Supports remote OCR with Azure AI (opt-in). - Documents are saved as PDF/A format which is designed for long term storage, alongside the unaltered originals. - Uses machine-learning to automatically add tags, correspondents and document types to your documents. -- **New**: Paperless-ngx can now leverage AI (Large Language Models or LLMs) for document suggestions. This is an optional feature that can be enabled (and is disabled by default). +- **New**: Paperless-ngx can optionally leverage AI (Large Language Models or LLMs) for document suggestions, chatting with your documents, and similar-document retrieval. These features are opt-in and disabled by default. - Supports PDF documents, images, plain text files, Office documents (Word, Excel, PowerPoint, and LibreOffice equivalents)[^1] and more. - Paperless stores your documents plain on disk. Filenames and folders are managed by paperless and their format can be configured freely with different configurations assigned to different documents. +- Keep multiple **versions** of a document's file under a single entry, sharing one set of metadata. - **Beautiful, modern web application** that features: - Customizable dashboard with statistics. - Filtering by tags, correspondents, types, and more. diff --git a/docs/setup.md b/docs/setup.md index 5580dde92..9420ba40b 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -178,7 +178,7 @@ to enable polling and disable inotify. See [here](configuration.md#polling). - `fonts-liberation` for generating thumbnails for plain text files - `imagemagick` >= 6 for PDF conversion - - `gnupg` for handling encrypted documents + - `gnupg` for decrypting GPG-encrypted email - `libpq-dev` for PostgreSQL - `libmagic-dev` for mime type detection - `mariadb-client` for MariaDB compile time @@ -226,7 +226,8 @@ to enable polling and disable inotify. See [here](configuration.md#polling). build-essential python3-setuptools python3-wheel ``` -2. Install `redis` >= 6.0 and configure it to start automatically. +2. Install a Redis-compatible broker (a current release of Valkey or + Redis) and configure it to start automatically. 3. Optional: Install `postgresql` and configure a database, user, and password for Paperless-ngx. If you do not wish to use PostgreSQL, @@ -268,10 +269,10 @@ to enable polling and disable inotify. See [here](configuration.md#polling). 6. Configure Paperless-ngx. See [configuration](configuration.md) for details. Edit the included `paperless.conf` and adjust the settings to your needs. Required settings for getting Paperless-ngx running are: - - [`PAPERLESS_REDIS`](configuration.md#PAPERLESS_REDIS) should point to your Redis server, such as + - [`PAPERLESS_REDIS`](configuration.md#PAPERLESS_REDIS) should point to your broker, such as `redis://localhost:6379`. - - [`PAPERLESS_DBENGINE`](configuration.md#PAPERLESS_DBENGINE) is optional, and should be one of `postgres`, - `mariadb`, or `sqlite` + - [`PAPERLESS_DBENGINE`](configuration.md#PAPERLESS_DBENGINE) should be one of `postgresql`, + `mariadb`, or `sqlite`. PostgreSQL and MariaDB users must set this explicitly. - [`PAPERLESS_DBHOST`](configuration.md#PAPERLESS_DBHOST) should be the hostname on which your PostgreSQL server is running. Do not configure this to use SQLite instead. Also configure port, database name, user and @@ -297,7 +298,7 @@ to enable polling and disable inotify. See [here](configuration.md#polling). !!! warning - Ensure your Redis instance [is secured](https://redis.io/docs/latest/operate/oss_and_stack/management/security/). + Ensure your broker instance [is secured](https://valkey.io/topics/security/). 7. Create the following directories if they do not already exist: - `/opt/paperless/media` @@ -389,9 +390,9 @@ to enable polling and disable inotify. See [here](configuration.md#polling). `Require=paperless-webserver.socket` in the `webserver` script and configure `granian` to listen on port 80 (set `GRANIAN_PORT`). - These services rely on Redis and optionally the database server, but + These services rely on the broker and optionally the database server, but don't need to be started in any particular order. The example files - depend on Redis being started. If you use a database server, you + depend on the broker being started. If you use a database server, you should add additional dependencies. !!! note @@ -449,6 +450,12 @@ development documentation. You can migrate to Paperless-ngx from Paperless-ng or from the original Paperless project. +!!! note + + Upgrading an existing Paperless-ngx installation from v2 to v3 has its own + breaking changes and required steps. See the [v3 migration guide](migration-v3.md) + before upgrading. +

Migrating from Paperless-ng

Paperless-ngx is meant to be a drop-in replacement for Paperless-ng, and @@ -494,7 +501,7 @@ installation. Keep these points in mind: for other services, you might as well use it for Paperless as well. - The task scheduler of Paperless, which is used to execute periodic tasks such as email checking and maintenance, requires a - [Redis](https://redis.io/) message broker instance. The + Redis-compatible message broker instance (such as Valkey or Redis). The Docker Compose route takes care of that. - The layout of the folder structure for your documents and data remains the same, so you can plug your old Docker volumes into @@ -582,16 +589,16 @@ commands as well. 1. Stop and remove the Paperless container. 2. If using an external database, stop that container. -3. Update Redis configuration. +3. Update broker configuration. 1. If `REDIS_URL` is already set, change it to [`PAPERLESS_REDIS`](configuration.md#PAPERLESS_REDIS) and continue to step 4. - 1. Otherwise, add a new Redis service in `docker-compose.yml`, + 1. Otherwise, add a new broker service in `docker-compose.yml`, following [the example compose files](https://github.com/paperless-ngx/paperless-ngx/tree/main/docker/compose) 1. Set the environment variable [`PAPERLESS_REDIS`](configuration.md#PAPERLESS_REDIS) so it points to - the new Redis container. + the new broker container. 4. Update user mapping. 1. If set, change the environment variable `PUID` to `USERMAP_UID`. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 47a10bf41..b372bea90 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -10,9 +10,9 @@ Check for the following issues: `CONSUMPTION_DIR` setting. Don't adjust this setting if you're using docker. -- Ensure that redis is up and running. Paperless does its task +- Ensure that the broker is up and running. Paperless does its task processing asynchronously, and for documents to arrive at the task - processor, it needs redis to run. + processor, it needs the broker to run. - Ensure that the task processor is running. Docker does this automatically. Manually invoke the task processor by executing @@ -149,37 +149,6 @@ operating system, if these are different from `1000`. See [Docker setup](setup.m Also ensure that you are able to read and write to the consumption directory on the host. -## OSError: \[Errno 19\] No such device when consuming files - -If you experience errors such as: - -```shell-session -File "/usr/local/lib/python3.7/site-packages/whoosh/codec/base.py", line 570, in open_compound_file -return CompoundStorage(dbfile, use_mmap=storage.supports_mmap) -File "/usr/local/lib/python3.7/site-packages/whoosh/filedb/compound.py", line 75, in __init__ -self._source = mmap.mmap(fileno, 0, access=mmap.ACCESS_READ) -OSError: [Errno 19] No such device - -During handling of the above exception, another exception occurred: - -Traceback (most recent call last): -File "/usr/local/lib/python3.7/site-packages/django_q/cluster.py", line 436, in worker -res = f(*task["args"], **task["kwargs"]) -File "/usr/src/paperless/src/documents/tasks.py", line 73, in consume_file -override_tag_ids=override_tag_ids) -File "/usr/src/paperless/src/documents/consumer.py", line 271, in try_consume_file -raise ConsumerError(e) -``` - -Paperless uses a search index to provide better and faster full text -searching. This search index is stored inside the `data` folder. The -search index uses memory-mapped files (mmap). The above error indicates -that paperless was unable to create and open these files. - -This happens when you're trying to store the data directory on certain -file systems (mostly network shares) that don't support memory-mapped -files. - ## Web-UI stuck at "Loading\..." This might have multiple reasons. diff --git a/docs/usage.md b/docs/usage.md index 27d3dee9e..72bef8d58 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -292,6 +292,23 @@ Once setup, navigating to the email settings page in Paperless-ngx will allow yo You can also submit a document using the REST API, see [POSTing documents](api.md#file-uploads) for details. +### Duplicate documents + +By default, Paperless-ngx **does not reject duplicates**. If you consume a file whose +contents exactly match an existing document (same checksum), the new copy is still +consumed and a warning is logged. The task entry for the upload also flags that a +duplicate was detected and links to the existing document(s). + +To review duplicates, open a document and switch to the **Duplicates** tab on the +document detail page. It lists other documents that share the same content, including any +that are in the trash (shown with a badge), and links to each so you can decide which to +keep. + +If you would rather reject duplicates at consumption time (the pre-v3 behavior), set +[`PAPERLESS_CONSUMER_DELETE_DUPLICATES`](configuration.md#PAPERLESS_CONSUMER_DELETE_DUPLICATES) +to `true`. The duplicate file is then deleted instead of consumed, and the task fails with +a "document already exists" message. + ## Document Suggestions Paperless-ngx can suggest tags, correspondents, document types and storage paths for documents based on the content of the document. This is done using a (non-LLM) machine learning model that is trained on the documents in your database. The suggestions are shown in the document detail page and can be accepted or rejected by the user. @@ -306,7 +323,9 @@ Paperless-ngx includes several features that use AI to enhance the document mana so consider the privacy implications of using these features, especially if using a remote model or API provider instead of the default local model. -The AI features work by creating an embedding of the text content and metadata of documents, which is then used for various tasks such as similarity search and question answering. This uses the FAISS vector store. +The AI features work by creating an embedding of the text content and metadata of documents, which is then used for various tasks such as similarity search and question answering. + +See [AI features](advanced_usage.md#ai-features) for how to enable and configure these features, including choosing an LLM backend and setting up the LLM index for RAG. ### AI-Enhanced Suggestions @@ -1097,7 +1116,7 @@ Paperless-ngx consists of the following components: errors (i.e., wrong email credentials, errors during consuming a specific file, etc). -- A [redis](https://redis.io/) message broker: This is a really +- A message broker (such as Valkey or Redis): This is a really lightweight service that is responsible for getting the tasks from the webserver and the consumer to the task scheduler. These run in a different process (maybe even on different machines!), and diff --git a/pyproject.toml b/pyproject.toml index 10101a09e..7a96a264a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,6 @@ dependencies = [ "drf-spectacular~=0.28", "drf-spectacular-sidecar~=2026.5.1", "drf-writable-nested~=0.7.1", - "faiss-cpu>=1.10", "filelock~=3.29.0", "flower~=2.0.1", "gotenberg-client~=0.14.0", @@ -57,7 +56,6 @@ dependencies = [ "llama-index-embeddings-openai-like>=0.2.2", "llama-index-llms-ollama>=0.9.1", "llama-index-llms-openai-like>=0.7.1", - "llama-index-vector-stores-faiss>=0.5.2", "nltk~=3.9.1", "ocrmypdf~=17.4.2", "openai>=2.32", @@ -74,6 +72,7 @@ dependencies = [ "scikit-learn~=1.8.0", "sentence-transformers>=5.4.1", "setproctitle~=1.3.4", + "sqlite-vec==0.1.9", "tantivy~=0.26.0", "tika-client~=0.11.0", "torch~=2.12.0", diff --git a/src-ui/jest.config.js b/src-ui/jest.config.js index 7b06016dd..f3557702a 100644 --- a/src-ui/jest.config.js +++ b/src-ui/jest.config.js @@ -26,7 +26,7 @@ module.exports = { 'abstract-paperless-service', ], transformIgnorePatterns: [ - 'node_modules/(?!.*(\\.mjs$|tslib|lodash-es|@angular/common/locales/.*\\.js$))', + 'node_modules/(?!.*(\\.mjs$|tslib|lodash-es|normalize-diacritics|@angular/common/locales/.*\\.js$))', ], moduleNameMapper: { ...esmPreset.moduleNameMapper, diff --git a/src-ui/package.json b/src-ui/package.json index 94074a688..b0ad88fc3 100644 --- a/src-ui/package.json +++ b/src-ui/package.json @@ -32,6 +32,7 @@ "ngx-cookie-service": "^21.3.1", "ngx-device-detector": "^11.0.0", "ngx-ui-tour-ng-bootstrap": "^18.0.0", + "normalize-diacritics": "^5.0.0", "pdfjs-dist": "^5.7.284", "rxjs": "^7.8.2", "tslib": "^2.8.1", diff --git a/src-ui/pnpm-lock.yaml b/src-ui/pnpm-lock.yaml index b7db948dd..caf7d1c60 100644 --- a/src-ui/pnpm-lock.yaml +++ b/src-ui/pnpm-lock.yaml @@ -71,6 +71,9 @@ importers: ngx-ui-tour-ng-bootstrap: specifier: ^18.0.0 version: 18.0.0(4ccfccfbcf381a309618492b31e99276) + normalize-diacritics: + specifier: ^5.0.0 + version: 5.0.0 pdfjs-dist: specifier: ^5.7.284 version: 5.7.284 @@ -5565,6 +5568,10 @@ packages: engines: {node: ^20.17.0 || >=22.9.0} hasBin: true + normalize-diacritics@5.0.0: + resolution: {integrity: sha512-t6czCJOpbAtckN1wCC2qPWnO3GQvNANb9bcUNbiOLEqojVuP31+ELIs5KhEG8jyz0TH7iD9BWxWz8O3ic2/rMQ==} + engines: {node: '>= 14.x', npm: '>= 6.x'} + normalize-path@3.0.0: resolution: {integrity: sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==} engines: {node: '>=0.10.0'} @@ -12985,6 +12992,10 @@ snapshots: dependencies: abbrev: 4.0.0 + normalize-diacritics@5.0.0: + dependencies: + tslib: 2.8.1 + normalize-path@3.0.0: {} npm-bundled@5.0.0: diff --git a/src-ui/src/app/components/admin/tasks/tasks.component.html b/src-ui/src/app/components/admin/tasks/tasks.component.html index 116d35f89..e1d6bc900 100644 --- a/src-ui/src/app/components/admin/tasks/tasks.component.html +++ b/src-ui/src/app/components/admin/tasks/tasks.component.html @@ -11,6 +11,9 @@ +
@@ -81,7 +84,7 @@
@for (t of filterTargets; track t.id) { - + }
diff --git a/src-ui/src/app/components/admin/tasks/tasks.component.spec.ts b/src-ui/src/app/components/admin/tasks/tasks.component.spec.ts index 962895295..315dc2455 100644 --- a/src-ui/src/app/components/admin/tasks/tasks.component.spec.ts +++ b/src-ui/src/app/components/admin/tasks/tasks.component.spec.ts @@ -11,7 +11,7 @@ import { Router } from '@angular/router' import { RouterTestingModule } from '@angular/router/testing' import { NgbModal, NgbModalRef, NgbModule } from '@ng-bootstrap/ng-bootstrap' import { allIcons, NgxBootstrapIconsModule } from 'ngx-bootstrap-icons' -import { throwError } from 'rxjs' +import { of, throwError } from 'rxjs' import { routes } from 'src/app/app-routing.module' import { PaperlessTask, @@ -29,7 +29,11 @@ import { ToastService } from 'src/app/services/toast.service' import { environment } from 'src/environments/environment' import { ConfirmDialogComponent } from '../../common/confirm-dialog/confirm-dialog.component' import { PageHeaderComponent } from '../../common/page-header/page-header.component' -import { TasksComponent, TaskSection } from './tasks.component' +import { + TaskFilterTargetID, + TasksComponent, + TaskSection, +} from './tasks.component' const tasks: PaperlessTask[] = [ { @@ -154,6 +158,13 @@ const paginatedTasks: Results = { results: tasks, } +const sectionCountResponse = { + all: 7, + needs_attention: 2, + in_progress: 3, + completed: 2, +} + describe('TasksComponent', () => { let component: TasksComponent let fixture: ComponentFixture @@ -221,6 +232,15 @@ describe('TasksComponent', () => { req.params.get('page') === '1' ) .flush(paginatedTasks) + + httpTestingController + .expectOne( + (req) => + req.url === `${environment.apiBaseUrl}tasks/status_counts/` && + req.params.get('acknowledged') === 'false' && + !req.params.has('status') + ) + .flush(sectionCountResponse) }) it('should display task sections with counts', () => { @@ -295,6 +315,7 @@ describe('TasksComponent', () => { const headerText = header.nativeElement.textContent expect(headerText).toContain('Dismiss visible') + expect(headerText).toContain('Dismiss all') expect(headerText).toContain('Auto refresh') expect(headerText).not.toContain('All types') expect(headerText).not.toContain('All sources') @@ -327,6 +348,74 @@ describe('TasksComponent', () => { expect(pagination).not.toBeNull() }) + it('should apply the selected section to the server-side task query', () => { + component.setSection(TaskSection.NeedsAttention) + + const req = httpTestingController.expectOne( + (request) => + request.url === `${environment.apiBaseUrl}tasks/` && + request.params.get('page') === '1' && + request.params.get('page_size') === '25' && + request.params.get('acknowledged') === 'false' && + request.params.getAll('status').includes(PaperlessTaskStatus.Failure) && + request.params.getAll('status').includes(PaperlessTaskStatus.Revoked) + ) + + req.flush({ count: 2, results: [tasks[0], tasks[1]] }) + expect(component.totalTasks).toBe(2) + }) + + it('should apply task type and trigger source filters to the server-side task query', () => { + component.setTaskType(PaperlessTaskType.SanityCheck) + + httpTestingController + .expectOne( + (request) => + request.url === `${environment.apiBaseUrl}tasks/` && + request.params.get('page_size') === '25' && + request.params.get('task_type') === PaperlessTaskType.SanityCheck + ) + .flush({ count: 1, results: [tasks[6]] }) + + component.setTriggerSource(PaperlessTaskTriggerSource.System) + + httpTestingController + .expectOne( + (request) => + request.url === `${environment.apiBaseUrl}tasks/` && + request.params.get('page_size') === '25' && + request.params.get('task_type') === PaperlessTaskType.SanityCheck && + request.params.get('trigger_source') === + PaperlessTaskTriggerSource.System + ) + .flush({ count: 1, results: [tasks[6]] }) + }) + + it('should apply text filters to the server-side task query', () => { + component.filterText = 'invoice' + jest.advanceTimersByTime(150) + + httpTestingController + .expectOne( + (request) => + request.url === `${environment.apiBaseUrl}tasks/` && + request.params.get('page_size') === '25' && + request.params.get('name') === 'invoice' + ) + .flush({ count: 1, results: [tasks[0]] }) + + component.setFilterTarget(TaskFilterTargetID.Result) + + httpTestingController + .expectOne( + (request) => + request.url === `${environment.apiBaseUrl}tasks/` && + request.params.get('page_size') === '25' && + request.params.get('result') === 'invoice' + ) + .flush({ count: 0, results: [] }) + }) + it('should load a different task page when pagination changes', () => { component.setPage(2) @@ -350,6 +439,27 @@ describe('TasksComponent', () => { expect(component.pagedTasks).toEqual([tasks[0]]) }) + it('should not replace section counts with current-page counts', () => { + component.setPage(2) + + httpTestingController + .expectOne( + (req) => + req.url === `${environment.apiBaseUrl}tasks/` && + req.params.get('acknowledged') === 'false' && + req.params.get('page_size') === '25' && + req.params.get('page') === '2' + ) + .flush({ + count: 30, + results: [tasks[0]], + }) + + expect(component.sectionCount(TaskSection.NeedsAttention)).toBe(2) + expect(component.sectionCount(TaskSection.InProgress)).toBe(3) + expect(component.sectionCount(TaskSection.Completed)).toBe(2) + }) + it('should expose stable task type options and disable empty ones', () => { expect(component.taskTypeOptions.map((option) => option.value)).toContain( PaperlessTaskType.TrainClassifier @@ -495,6 +605,46 @@ describe('TasksComponent', () => { expect(dismissSpy).toHaveBeenCalledWith(new Set([467, 466])) }) + it('should support dismiss all tasks', () => { + let modal: NgbModalRef + modalService.activeInstances.subscribe((m) => (modal = m[m.length - 1])) + const dismissSpy = jest + .spyOn(tasksService, 'dismissAllTasks') + .mockReturnValue(of({})) + const reloadPageSpy = jest + .spyOn(component as any, 'reloadPage') + .mockImplementation(() => undefined) + + component.dismissAllTasks() + + expect(modal).not.toBeUndefined() + expect(modal.componentInstance.messageBold).toBe('Dismiss all 7 tasks?') + modal.componentInstance.confirmClicked.emit() + expect(dismissSpy).toHaveBeenCalled() + expect(reloadPageSpy).toHaveBeenCalledWith(false) + expect(component.selectedTasks.size).toBe(0) + }) + + it('should show an error and re-enable modal buttons when dismissing all tasks fails', () => { + const error = new Error('dismiss all failed') + const toastSpy = jest.spyOn(toastService, 'showError') + const dismissSpy = jest + .spyOn(tasksService, 'dismissAllTasks') + .mockReturnValue(throwError(() => error)) + + let modal: NgbModalRef + modalService.activeInstances.subscribe((m) => (modal = m[m.length - 1])) + + component.dismissAllTasks() + expect(modal).not.toBeUndefined() + + modal.componentInstance.confirmClicked.emit() + + expect(dismissSpy).toHaveBeenCalled() + expect(toastSpy).toHaveBeenCalledWith('Error dismissing tasks', error) + expect(modal.componentInstance.buttonsEnabled).toBe(true) + }) + it('should dismiss the currently visible scoped and filtered tasks', () => { component.setSection(TaskSection.InProgress) component.setTaskType(PaperlessTaskType.SanityCheck) @@ -673,6 +823,9 @@ describe('TasksComponent', () => { }) it('should keep clearing selection independent from resetting filters', () => { + component.resetFilter() + expect(component.filterText).toBe('') + component.setTaskType(PaperlessTaskType.ConsumeFile) component.toggleSelected(tasks[0]) expect(component.selectedTasks.size).toBe(1) diff --git a/src-ui/src/app/components/admin/tasks/tasks.component.ts b/src-ui/src/app/components/admin/tasks/tasks.component.ts index 884ede0d6..276dc6a8f 100644 --- a/src-ui/src/app/components/admin/tasks/tasks.component.ts +++ b/src-ui/src/app/components/admin/tasks/tasks.component.ts @@ -40,7 +40,7 @@ export enum TaskSection { Completed = 'completed', } -enum TaskFilterTargetID { +export enum TaskFilterTargetID { Name, Result, } @@ -167,6 +167,12 @@ export class TasksComponent public readonly pageSize = 25 public page: number = 1 public totalTasks: number = 0 + public sectionCounts: Record = { + [TaskSection.All]: 0, + [TaskSection.NeedsAttention]: 0, + [TaskSection.InProgress]: 0, + [TaskSection.Completed]: 0, + } public pagedTasks: PaperlessTask[] = [] public selectedSection: TaskSection = TaskSection.All public selectedTaskType: PaperlessTaskType | null = null @@ -282,6 +288,7 @@ export class TasksComponent .subscribe((query) => { this._filterText = query this.clearSelection() + this.reloadPage(true) }) } @@ -334,6 +341,30 @@ export class TasksComponent } } + dismissAllTasks() { + let modal = this.modalService.open(ConfirmDialogComponent, { + backdrop: 'static', + }) + modal.componentInstance.title = $localize`Confirm Dismiss All` + modal.componentInstance.messageBold = $localize`Dismiss all ${this.totalTasks} tasks?` + modal.componentInstance.btnClass = 'btn-warning' + modal.componentInstance.btnCaption = $localize`Dismiss` + modal.componentInstance.confirmClicked.pipe(first()).subscribe(() => { + modal.componentInstance.buttonsEnabled = false + modal.close() + this.tasksService.dismissAllTasks().subscribe({ + next: () => { + this.reloadPage(false) + }, + error: (e) => { + this.toastService.showError($localize`Error dismissing tasks`, e) + modal.componentInstance.buttonsEnabled = true + }, + }) + this.clearSelection() + }) + } + expandTask(task: PaperlessTask) { this.expandedTask = this.expandedTask == task.id ? undefined : task.id } @@ -446,9 +477,7 @@ export class TasksComponent } sectionCount(section: TaskSection): number { - return this.pagedTasks.filter((task) => - this.taskBelongsToSection(task, section) - ).length + return this.sectionCounts[section] } sectionShowsResults(section: TaskSection): boolean { @@ -458,16 +487,27 @@ export class TasksComponent setSection(section: TaskSection) { this.selectedSection = section this.clearSelection() + this.reloadPage(true) } setTaskType(taskType: PaperlessTaskType | null) { this.selectedTaskType = taskType this.clearSelection() + this.reloadPage(true) } setTriggerSource(triggerSource: PaperlessTaskTriggerSource | null) { this.selectedTriggerSource = triggerSource this.clearSelection() + this.reloadPage(true) + } + + setFilterTarget(filterTargetID: TaskFilterTargetID) { + this.filterTargetID = filterTargetID + if (this._filterText.length) { + this.clearSelection() + this.reloadPage(true) + } } taskTypeOptionCount(taskType: PaperlessTaskType | null): number { @@ -505,19 +545,32 @@ export class TasksComponent } public resetFilter() { + if (!this._filterText.length) { + return + } + this._filterText = '' + this.clearSelection() + this.reloadPage(true) } public resetFilters() { + const hadFilter = this.isFiltered this.selectedTaskType = null this.selectedTriggerSource = null - this.resetFilter() + this._filterText = '' this.clearSelection() + + if (hadFilter) { + this.reloadPage(true) + } } filterInputKeyup(event: KeyboardEvent) { if (event.key == 'Enter') { this._filterText = (event.target as HTMLInputElement).value + this.clearSelection() + this.reloadPage(true) } else if (event.key === 'Escape') { this.resetFilter() } @@ -606,19 +659,86 @@ export class TasksComponent ) } + private reloadSectionCounts() { + this.tasksService + .statusCounts(this.getParamsForSection(TaskSection.All)) + .pipe(first(), takeUntil(this.unsubscribeNotifier)) + .subscribe((counts) => { + this.sectionCounts[TaskSection.All] = counts.all + this.sectionCounts[TaskSection.NeedsAttention] = counts.needs_attention + this.sectionCounts[TaskSection.InProgress] = counts.in_progress + this.sectionCounts[TaskSection.Completed] = counts.completed + }) + } + + private getParamsForSection( + section: TaskSection + ): Record { + const params: Record< + string, + string | number | boolean | readonly string[] + > = { + acknowledged: false, + } + + const statuses = this.statusesForSection(section) + if (statuses.length) { + params.status = statuses + } + + if (this.selectedTaskType !== null) { + params.task_type = this.selectedTaskType + } + + if (this.selectedTriggerSource !== null) { + params.trigger_source = this.selectedTriggerSource + } + + if (this._filterText.length) { + params[ + this.filterTargetID === TaskFilterTargetID.Name ? 'name' : 'result' + ] = this._filterText + } + + return params + } + + private statusesForSection(section: TaskSection): PaperlessTaskStatus[] { + switch (section) { + case TaskSection.NeedsAttention: + return [PaperlessTaskStatus.Failure, PaperlessTaskStatus.Revoked] + case TaskSection.InProgress: + return [PaperlessTaskStatus.Pending, PaperlessTaskStatus.Started] + case TaskSection.Completed: + return [PaperlessTaskStatus.Success] + default: + return [] + } + } + private reloadPage(resetToFirstPage: boolean = false) { if (resetToFirstPage) { this.page = 1 } + this.reloadSectionCounts() + this.loading = true this.tasksService - .list(this.page, this.pageSize, { acknowledged: false }) + .list( + this.page, + this.pageSize, + this.getParamsForSection(this.selectedSection) + ) .pipe(first(), takeUntil(this.unsubscribeNotifier)) .subscribe({ next: (result) => { this.pagedTasks = result.results this.totalTasks = result.count + this.sectionCounts[TaskSection.All] = result.count + if (this.selectedSection !== TaskSection.All) { + this.sectionCounts[this.selectedSection] = result.count + } this.loading = false if ( this.page > 1 && diff --git a/src-ui/src/app/components/chat/chat/chat.component.html b/src-ui/src/app/components/chat/chat/chat.component.html index c5cada978..78cd28ca3 100644 --- a/src-ui/src/app/components/chat/chat/chat.component.html +++ b/src-ui/src/app/components/chat/chat/chat.component.html @@ -8,7 +8,7 @@
@for (message of messages; track message) {
-
+
{{ message.content }} @if (message.isStreaming) { | } diff --git a/src-ui/src/app/components/chat/chat/chat.component.spec.ts b/src-ui/src/app/components/chat/chat/chat.component.spec.ts index a35117dc5..f7c4134b2 100644 --- a/src-ui/src/app/components/chat/chat/chat.component.spec.ts +++ b/src-ui/src/app/components/chat/chat/chat.component.spec.ts @@ -188,4 +188,14 @@ describe('ChatComponent', () => { component.searchInputKeyDown(event) expect(component.sendMessage).toHaveBeenCalled() }) + + it('should not send message on Enter key press while composing with IME', () => { + jest.spyOn(component, 'sendMessage') + const event = new KeyboardEvent('keydown', { + key: 'Enter', + isComposing: true, + }) + component.searchInputKeyDown(event) + expect(component.sendMessage).not.toHaveBeenCalled() + }) }) diff --git a/src-ui/src/app/components/chat/chat/chat.component.ts b/src-ui/src/app/components/chat/chat/chat.component.ts index ca17d4825..0920af027 100644 --- a/src-ui/src/app/components/chat/chat/chat.component.ts +++ b/src-ui/src/app/components/chat/chat/chat.component.ts @@ -155,7 +155,10 @@ export class ChatComponent implements OnInit { } public searchInputKeyDown(event: KeyboardEvent) { - if (event.key === 'Enter') { + if ( + event.key === 'Enter' && + !(event.isComposing || event.keyCode === 229) + ) { event.preventDefault() this.sendMessage() } diff --git a/src-ui/src/app/components/common/confirm-dialog/confirm-dialog.component.html b/src-ui/src/app/components/common/confirm-dialog/confirm-dialog.component.html index deee54402..437e7af94 100644 --- a/src-ui/src/app/components/common/confirm-dialog/confirm-dialog.component.html +++ b/src-ui/src/app/components/common/confirm-dialog/confirm-dialog.component.html @@ -5,10 +5,10 @@
@if (messageBold) { -

{{messageBold}}

+

{{messageBold}}

} @if (message) { -

+

}
diff --git a/src-ui/src/app/components/common/confirm-dialog/merge-confirm-dialog/merge-confirm-dialog.component.html b/src-ui/src/app/components/common/confirm-dialog/merge-confirm-dialog/merge-confirm-dialog.component.html index 90f555890..ab0951d54 100644 --- a/src-ui/src/app/components/common/confirm-dialog/merge-confirm-dialog/merge-confirm-dialog.component.html +++ b/src-ui/src/app/components/common/confirm-dialog/merge-confirm-dialog/merge-confirm-dialog.component.html @@ -9,8 +9,11 @@
    - @for (document of documents; track document.id) { + @for (documentID of documentIDs; track documentID) { + @let document = getDocument(documentID); + @if (document) {
  • @@ -27,6 +30,7 @@
    • + } }
diff --git a/src-ui/src/app/components/common/custom-fields-dropdown/custom-fields-dropdown.component.ts b/src-ui/src/app/components/common/custom-fields-dropdown/custom-fields-dropdown.component.ts index b0fcd48b8..ac7b3a541 100644 --- a/src-ui/src/app/components/common/custom-fields-dropdown/custom-fields-dropdown.component.ts +++ b/src-ui/src/app/components/common/custom-fields-dropdown/custom-fields-dropdown.component.ts @@ -23,6 +23,7 @@ import { import { CustomFieldsService } from 'src/app/services/rest/custom-fields.service' import { ToastService } from 'src/app/services/toast.service' import { pngxPopperOptions } from 'src/app/utils/popper-options' +import { matchesSearchText } from 'src/app/utils/text-search' import { LoadingComponentWithPermissions } from '../../loading-component/loading.component' import { CustomFieldEditDialogComponent } from '../edit-dialog/custom-field-edit-dialog/custom-field-edit-dialog.component' @@ -69,9 +70,7 @@ export class CustomFieldsDropdownComponent extends LoadingComponentWithPermissio public get filteredFields(): CustomField[] { return this.unusedFields.filter( - (f) => - !this.filterText || - f.name.toLowerCase().includes(this.filterText.toLowerCase()) + (f) => !this.filterText || matchesSearchText(f.name, this.filterText) ) } diff --git a/src-ui/src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.html b/src-ui/src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.html index ae2e93998..1c0134f70 100644 --- a/src-ui/src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.html +++ b/src-ui/src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.html @@ -63,6 +63,7 @@ [(ngModel)]="atom.value" [disabled]="disabled" [virtualScroll]="getSelectOptionsForField(atom.field)?.length > 100" + [searchFn]="selectOptionSearchFn" (mousedown)="$event.stopImmediatePropagation()" > } @else if (getCustomFieldByID(atom.field)?.data_type === CustomFieldDataType.DocumentLink) { @@ -81,6 +82,7 @@ [disabled]="disabled" bindLabel="name" bindValue="id" + [searchFn]="customFieldSearchFn" (mousedown)="$event.stopImmediatePropagation()" >