From 21db608d57856832f77a434f1d054048fae18cc8 Mon Sep 17 00:00:00 2001 From: Trenton H <797416+stumpylog@users.noreply.github.com> Date: Wed, 15 Apr 2026 15:26:00 -0700 Subject: [PATCH] Feature: Tune SQLite default settings for increased speed (#12580) --- docs/configuration.md | 28 ++++++-- docs/migration-v3.md | 2 +- src/paperless/settings/custom.py | 27 +++++++- .../tests/settings/test_custom_parsers.py | 66 +++++++++++++++---- 4 files changed, 103 insertions(+), 20 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index f8927a575..f1192acb9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -101,7 +101,7 @@ and `mariadb`. #### [`PAPERLESS_DB_OPTIONS=`](#PAPERLESS_DB_OPTIONS) {#PAPERLESS_DB_OPTIONS} -: Advanced database connection options as a semicolon-delimited key-value string. +: Advanced database connection options as a comma-delimited key-value string. Keys and values are separated by `=`. Dot-notation produces nested option dictionaries; for example, `pool.max_size=20` sets `OPTIONS["pool"]["max_size"] = 20`. @@ -123,18 +123,36 @@ dictionaries; for example, `pool.max_size=20` sets to handle all pool connections across all workers: `(web_workers + celery_workers) * pool.max_size + safety_margin`. + !!! note "SQLite defaults" + + SQLite connections are pre-configured with WAL journal mode, optimised + synchronous and cache settings, and a 5-second busy timeout. These defaults + suit most deployments. To override `init_command`, use `;` between PRAGMAs + within the value and `,` between options: + + ```bash + PAPERLESS_DB_OPTIONS="init_command=PRAGMA journal_mode=DELETE;PRAGMA synchronous=FULL,transaction_mode=DEFERRED" + ``` + + !!! note "MariaDB: READ COMMITTED isolation level" + + MariaDB connections default to `READ COMMITTED` isolation level, which + eliminates gap locking and reduces deadlock frequency. If binary logging is + enabled on your MariaDB server, this requires `binlog_format=ROW` (the + default for most managed MariaDB instances). Statement-based replication is + not compatible with `READ COMMITTED`. + **Examples:** ```bash title="PostgreSQL: require SSL, set a custom CA certificate, and limit the pool size" - PAPERLESS_DB_OPTIONS="sslmode=require;sslrootcert=/certs/ca.pem;pool.max_size=5" + PAPERLESS_DB_OPTIONS="sslmode=require,sslrootcert=/certs/ca.pem,pool.max_size=5" ``` ```bash title="MariaDB: require SSL with a custom CA certificate" - PAPERLESS_DB_OPTIONS="ssl_mode=REQUIRED;ssl.ca=/certs/ca.pem" + PAPERLESS_DB_OPTIONS="ssl_mode=REQUIRED,ssl.ca=/certs/ca.pem" ``` - ```bash title="SQLite: set a busy timeout of 30 seconds" - # PostgreSQL: set a connection timeout + ```bash title="PostgreSQL or MariaDB: set a connection timeout" PAPERLESS_DB_OPTIONS="connect_timeout=10" ``` diff --git a/docs/migration-v3.md b/docs/migration-v3.md index a3fa39321..f577b2039 100644 --- a/docs/migration-v3.md +++ b/docs/migration-v3.md @@ -120,7 +120,7 @@ Users with any of the deprecated variables set should migrate to `PAPERLESS_DB_O Multiple options are combined in a single value: ```bash -PAPERLESS_DB_OPTIONS="sslmode=require;sslrootcert=/certs/ca.pem;pool.max_size=10" +PAPERLESS_DB_OPTIONS="sslmode=require,sslrootcert=/certs/ca.pem,pool.max_size=10" ``` ## OCR and Archive File Generation Settings diff --git a/src/paperless/settings/custom.py b/src/paperless/settings/custom.py index 5459576c3..9e9468e21 100644 --- a/src/paperless/settings/custom.py +++ b/src/paperless/settings/custom.py @@ -224,7 +224,23 @@ def parse_db_settings(data_dir: Path) -> dict[str, dict[str, Any]]: "ENGINE": "django.db.backends.sqlite3", "NAME": str((data_dir / "db.sqlite3").resolve()), } - base_options = {} + base_options = { + # Django splits init_command on ";" and calls conn.execute() + # once per statement, so multiple PRAGMAs work correctly. + # foreign_keys is omitted — Django sets it natively. + "init_command": ( + "PRAGMA journal_mode=WAL;" + "PRAGMA synchronous=NORMAL;" + "PRAGMA busy_timeout=5000;" + "PRAGMA temp_store=MEMORY;" + "PRAGMA mmap_size=134217728;" + "PRAGMA journal_size_limit=67108864;" + "PRAGMA cache_size=-8000" # negative = KiB; -8000 ≈ 8 MB + ), + # IMMEDIATE acquires the write lock at BEGIN, ensuring + # busy_timeout is respected from the start of the transaction. + "transaction_mode": "IMMEDIATE", + } case "postgresql": db_config = { @@ -240,6 +256,7 @@ def parse_db_settings(data_dir: Path) -> dict[str, dict[str, Any]]: "sslrootcert": os.getenv("PAPERLESS_DBSSLROOTCERT"), "sslcert": os.getenv("PAPERLESS_DBSSLCERT"), "sslkey": os.getenv("PAPERLESS_DBSSLKEY"), + "application_name": "paperless-ngx", } if (pool_size := get_int_from_env("PAPERLESS_DB_POOLSIZE")) is not None: @@ -267,6 +284,12 @@ def parse_db_settings(data_dir: Path) -> dict[str, dict[str, Any]]: "cert": os.getenv("PAPERLESS_DBSSLCERT"), "key": os.getenv("PAPERLESS_DBSSLKEY"), }, + # READ COMMITTED eliminates gap locking and reduces deadlocks. + # Django also defaults to "read committed" for MySQL/MariaDB, but + # we set it explicitly so the intent is clear and survives any + # future changes to Django's default. + # Requires binlog_format=ROW if binary logging is enabled. + "isolation_level": "read committed", } case _: # pragma: no cover raise NotImplementedError(engine) @@ -287,7 +310,7 @@ def parse_db_settings(data_dir: Path) -> dict[str, dict[str, Any]]: db_config["OPTIONS"] = parse_dict_from_str( os.getenv("PAPERLESS_DB_OPTIONS"), defaults=base_options, - separator=";", + separator=",", type_map={ # SQLite options "timeout": int, diff --git a/src/paperless/tests/settings/test_custom_parsers.py b/src/paperless/tests/settings/test_custom_parsers.py index 6fa7ad8eb..0443c5e99 100644 --- a/src/paperless/tests/settings/test_custom_parsers.py +++ b/src/paperless/tests/settings/test_custom_parsers.py @@ -296,8 +296,19 @@ class TestParseDbSettings: { "default": { "ENGINE": "django.db.backends.sqlite3", - "NAME": None, # Will be replaced with tmp_path - "OPTIONS": {}, + "NAME": None, # replaced with tmp_path in test body + "OPTIONS": { + "init_command": ( + "PRAGMA journal_mode=WAL;" + "PRAGMA synchronous=NORMAL;" + "PRAGMA busy_timeout=5000;" + "PRAGMA temp_store=MEMORY;" + "PRAGMA mmap_size=134217728;" + "PRAGMA journal_size_limit=67108864;" + "PRAGMA cache_size=-8000" + ), + "transaction_mode": "IMMEDIATE", + }, }, }, id="default-sqlite", @@ -310,14 +321,41 @@ class TestParseDbSettings: { "default": { "ENGINE": "django.db.backends.sqlite3", - "NAME": None, # Will be replaced with tmp_path + "NAME": None, "OPTIONS": { + "init_command": ( + "PRAGMA journal_mode=WAL;" + "PRAGMA synchronous=NORMAL;" + "PRAGMA busy_timeout=5000;" + "PRAGMA temp_store=MEMORY;" + "PRAGMA mmap_size=134217728;" + "PRAGMA journal_size_limit=67108864;" + "PRAGMA cache_size=-8000" + ), + "transaction_mode": "IMMEDIATE", "timeout": 30, }, }, }, id="sqlite-with-timeout-override", ), + pytest.param( + { + "PAPERLESS_DBENGINE": "sqlite", + "PAPERLESS_DB_OPTIONS": "init_command=PRAGMA journal_mode=DELETE;PRAGMA synchronous=FULL,transaction_mode=DEFERRED", + }, + { + "default": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": None, + "OPTIONS": { + "init_command": "PRAGMA journal_mode=DELETE;PRAGMA synchronous=FULL", + "transaction_mode": "DEFERRED", + }, + }, + }, + id="sqlite-options-override", + ), pytest.param( { "PAPERLESS_DBENGINE": "postgresql", @@ -335,6 +373,7 @@ class TestParseDbSettings: "sslrootcert": None, "sslcert": None, "sslkey": None, + "application_name": "paperless-ngx", }, }, }, @@ -348,7 +387,7 @@ class TestParseDbSettings: "PAPERLESS_DBNAME": "customdb", "PAPERLESS_DBUSER": "customuser", "PAPERLESS_DBPASS": "custompass", - "PAPERLESS_DB_OPTIONS": "pool.max_size=50;pool.min_size=2;sslmode=require", + "PAPERLESS_DB_OPTIONS": "pool.max_size=50,pool.min_size=2,sslmode=require", }, { "default": { @@ -363,6 +402,7 @@ class TestParseDbSettings: "sslrootcert": None, "sslcert": None, "sslkey": None, + "application_name": "paperless-ngx", "pool": { "min_size": 2, "max_size": 50, @@ -390,6 +430,7 @@ class TestParseDbSettings: "sslrootcert": None, "sslcert": None, "sslkey": None, + "application_name": "paperless-ngx", "pool": { "min_size": 1, "max_size": 10, @@ -419,6 +460,7 @@ class TestParseDbSettings: "sslrootcert": "/certs/ca.crt", "sslcert": None, "sslkey": None, + "application_name": "paperless-ngx", "connect_timeout": 30, }, }, @@ -447,6 +489,7 @@ class TestParseDbSettings: "cert": None, "key": None, }, + "isolation_level": "read committed", }, }, }, @@ -455,18 +498,17 @@ class TestParseDbSettings: pytest.param( { "PAPERLESS_DBENGINE": "mariadb", - "PAPERLESS_DBHOST": "paperless-mariadb-host", - "PAPERLESS_DBPORT": "5555", + "PAPERLESS_DBHOST": "mariahost", + "PAPERLESS_DBNAME": "paperlessdb", "PAPERLESS_DBUSER": "my-cool-user", "PAPERLESS_DBPASS": "my-secure-password", - "PAPERLESS_DB_OPTIONS": "ssl.ca=/path/to/ca.pem;ssl_mode=REQUIRED", + "PAPERLESS_DB_OPTIONS": "ssl_mode=REQUIRED,ssl.ca=/path/to/ca.pem", }, { "default": { "ENGINE": "django.db.backends.mysql", - "HOST": "paperless-mariadb-host", - "PORT": 5555, - "NAME": "paperless", + "HOST": "mariahost", + "NAME": "paperlessdb", "USER": "my-cool-user", "PASSWORD": "my-secure-password", "OPTIONS": { @@ -479,6 +521,7 @@ class TestParseDbSettings: "cert": None, "key": None, }, + "isolation_level": "read committed", }, }, }, @@ -512,6 +555,7 @@ class TestParseDbSettings: "key": "/certs/client.key", }, "connect_timeout": 25, + "isolation_level": "read committed", }, }, }, @@ -527,10 +571,8 @@ class TestParseDbSettings: expected_database_settings: dict[str, dict], ) -> None: """Test various database configurations with defaults and overrides.""" - # Clear environment and set test vars mocker.patch.dict(os.environ, env_vars, clear=True) - # Update expected paths with actual tmp_path if ( "default" in expected_database_settings and expected_database_settings["default"]["NAME"] is None