Compare commits

...

3 Commits

Author SHA1 Message Date
Trenton H
33a27167d5 refactor: clarify isolation_level intent and cache_size unit
Add comment explaining isolation_level is explicit even though Django
defaults to the same value, so the intent survives future Django changes.
Add KiB unit comment to cache_size=-8000.
Rename sqlite-init-command-override test to sqlite-options-override to
reflect that both init_command and transaction_mode are being overridden.
2026-04-14 16:22:45 -07:00
Trenton H
a9b6b403ac docs: update PAPERLESS_DB_OPTIONS for comma separator and new engine defaults
Change all examples from semicolon to comma-delimited format. Add notes
for SQLite WAL defaults (with override example) and MariaDB READ COMMITTED
binlog_format=ROW prerequisite.
2026-04-14 15:36:30 -07:00
Trenton H
b595da9221 feat: add database tuning defaults to parse_db_settings
Change PAPERLESS_DB_OPTIONS separator from ';' to ',' so SQLite
init_command values (which use ';' between PRAGMAs) can be overridden
without escaping.

SQLite: WAL journal mode, NORMAL synchronous, 5s busy timeout, memory
temp store, 128MB mmap, 64MB journal size limit, 8MB cache, and
IMMEDIATE transaction mode for correct busy_timeout behaviour.
PostgreSQL: application_name=paperless-ngx for pg_stat_activity.
MariaDB: isolation_level=read committed to eliminate gap locking.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-04-14 15:31:10 -07:00
4 changed files with 103 additions and 20 deletions

View File

@@ -101,7 +101,7 @@ and `mariadb`.
#### [`PAPERLESS_DB_OPTIONS=<options>`](#PAPERLESS_DB_OPTIONS) {#PAPERLESS_DB_OPTIONS}
: Advanced database connection options as a semicolon-delimited key-value string.
: Advanced database connection options as a comma-delimited key-value string.
Keys and values are separated by `=`. Dot-notation produces nested option
dictionaries; for example, `pool.max_size=20` sets
`OPTIONS["pool"]["max_size"] = 20`.
@@ -123,18 +123,36 @@ dictionaries; for example, `pool.max_size=20` sets
to handle all pool connections across all workers:
`(web_workers + celery_workers) * pool.max_size + safety_margin`.
!!! note "SQLite defaults"
SQLite connections are pre-configured with WAL journal mode, optimised
synchronous and cache settings, and a 5-second busy timeout. These defaults
suit most deployments. To override `init_command`, use `;` between PRAGMAs
within the value and `,` between options:
```bash
PAPERLESS_DB_OPTIONS="init_command=PRAGMA journal_mode=DELETE;PRAGMA synchronous=FULL,transaction_mode=DEFERRED"
```
!!! note "MariaDB: READ COMMITTED isolation level"
MariaDB connections default to `READ COMMITTED` isolation level, which
eliminates gap locking and reduces deadlock frequency. If binary logging is
enabled on your MariaDB server, this requires `binlog_format=ROW` (the
default for most managed MariaDB instances). Statement-based replication is
not compatible with `READ COMMITTED`.
**Examples:**
```bash title="PostgreSQL: require SSL, set a custom CA certificate, and limit the pool size"
PAPERLESS_DB_OPTIONS="sslmode=require;sslrootcert=/certs/ca.pem;pool.max_size=5"
PAPERLESS_DB_OPTIONS="sslmode=require,sslrootcert=/certs/ca.pem,pool.max_size=5"
```
```bash title="MariaDB: require SSL with a custom CA certificate"
PAPERLESS_DB_OPTIONS="ssl_mode=REQUIRED;ssl.ca=/certs/ca.pem"
PAPERLESS_DB_OPTIONS="ssl_mode=REQUIRED,ssl.ca=/certs/ca.pem"
```
```bash title="SQLite: set a busy timeout of 30 seconds"
# PostgreSQL: set a connection timeout
```bash title="PostgreSQL or MariaDB: set a connection timeout"
PAPERLESS_DB_OPTIONS="connect_timeout=10"
```

View File

@@ -120,7 +120,7 @@ Users with any of the deprecated variables set should migrate to `PAPERLESS_DB_O
Multiple options are combined in a single value:
```bash
PAPERLESS_DB_OPTIONS="sslmode=require;sslrootcert=/certs/ca.pem;pool.max_size=10"
PAPERLESS_DB_OPTIONS="sslmode=require,sslrootcert=/certs/ca.pem,pool.max_size=10"
```
## OCR and Archive File Generation Settings

View File

@@ -224,7 +224,23 @@ def parse_db_settings(data_dir: Path) -> dict[str, dict[str, Any]]:
"ENGINE": "django.db.backends.sqlite3",
"NAME": str((data_dir / "db.sqlite3").resolve()),
}
base_options = {}
base_options = {
# Django splits init_command on ";" and calls conn.execute()
# once per statement, so multiple PRAGMAs work correctly.
# foreign_keys is omitted — Django sets it natively.
"init_command": (
"PRAGMA journal_mode=WAL;"
"PRAGMA synchronous=NORMAL;"
"PRAGMA busy_timeout=5000;"
"PRAGMA temp_store=MEMORY;"
"PRAGMA mmap_size=134217728;"
"PRAGMA journal_size_limit=67108864;"
"PRAGMA cache_size=-8000" # negative = KiB; -8000 ≈ 8 MB
),
# IMMEDIATE acquires the write lock at BEGIN, ensuring
# busy_timeout is respected from the start of the transaction.
"transaction_mode": "IMMEDIATE",
}
case "postgresql":
db_config = {
@@ -240,6 +256,7 @@ def parse_db_settings(data_dir: Path) -> dict[str, dict[str, Any]]:
"sslrootcert": os.getenv("PAPERLESS_DBSSLROOTCERT"),
"sslcert": os.getenv("PAPERLESS_DBSSLCERT"),
"sslkey": os.getenv("PAPERLESS_DBSSLKEY"),
"application_name": "paperless-ngx",
}
if (pool_size := get_int_from_env("PAPERLESS_DB_POOLSIZE")) is not None:
@@ -267,6 +284,12 @@ def parse_db_settings(data_dir: Path) -> dict[str, dict[str, Any]]:
"cert": os.getenv("PAPERLESS_DBSSLCERT"),
"key": os.getenv("PAPERLESS_DBSSLKEY"),
},
# READ COMMITTED eliminates gap locking and reduces deadlocks.
# Django also defaults to "read committed" for MySQL/MariaDB, but
# we set it explicitly so the intent is clear and survives any
# future changes to Django's default.
# Requires binlog_format=ROW if binary logging is enabled.
"isolation_level": "read committed",
}
case _: # pragma: no cover
raise NotImplementedError(engine)
@@ -287,7 +310,7 @@ def parse_db_settings(data_dir: Path) -> dict[str, dict[str, Any]]:
db_config["OPTIONS"] = parse_dict_from_str(
os.getenv("PAPERLESS_DB_OPTIONS"),
defaults=base_options,
separator=";",
separator=",",
type_map={
# SQLite options
"timeout": int,

View File

@@ -296,8 +296,19 @@ class TestParseDbSettings:
{
"default": {
"ENGINE": "django.db.backends.sqlite3",
"NAME": None, # Will be replaced with tmp_path
"OPTIONS": {},
"NAME": None, # replaced with tmp_path in test body
"OPTIONS": {
"init_command": (
"PRAGMA journal_mode=WAL;"
"PRAGMA synchronous=NORMAL;"
"PRAGMA busy_timeout=5000;"
"PRAGMA temp_store=MEMORY;"
"PRAGMA mmap_size=134217728;"
"PRAGMA journal_size_limit=67108864;"
"PRAGMA cache_size=-8000"
),
"transaction_mode": "IMMEDIATE",
},
},
},
id="default-sqlite",
@@ -310,14 +321,41 @@ class TestParseDbSettings:
{
"default": {
"ENGINE": "django.db.backends.sqlite3",
"NAME": None, # Will be replaced with tmp_path
"NAME": None,
"OPTIONS": {
"init_command": (
"PRAGMA journal_mode=WAL;"
"PRAGMA synchronous=NORMAL;"
"PRAGMA busy_timeout=5000;"
"PRAGMA temp_store=MEMORY;"
"PRAGMA mmap_size=134217728;"
"PRAGMA journal_size_limit=67108864;"
"PRAGMA cache_size=-8000"
),
"transaction_mode": "IMMEDIATE",
"timeout": 30,
},
},
},
id="sqlite-with-timeout-override",
),
pytest.param(
{
"PAPERLESS_DBENGINE": "sqlite",
"PAPERLESS_DB_OPTIONS": "init_command=PRAGMA journal_mode=DELETE;PRAGMA synchronous=FULL,transaction_mode=DEFERRED",
},
{
"default": {
"ENGINE": "django.db.backends.sqlite3",
"NAME": None,
"OPTIONS": {
"init_command": "PRAGMA journal_mode=DELETE;PRAGMA synchronous=FULL",
"transaction_mode": "DEFERRED",
},
},
},
id="sqlite-options-override",
),
pytest.param(
{
"PAPERLESS_DBENGINE": "postgresql",
@@ -335,6 +373,7 @@ class TestParseDbSettings:
"sslrootcert": None,
"sslcert": None,
"sslkey": None,
"application_name": "paperless-ngx",
},
},
},
@@ -348,7 +387,7 @@ class TestParseDbSettings:
"PAPERLESS_DBNAME": "customdb",
"PAPERLESS_DBUSER": "customuser",
"PAPERLESS_DBPASS": "custompass",
"PAPERLESS_DB_OPTIONS": "pool.max_size=50;pool.min_size=2;sslmode=require",
"PAPERLESS_DB_OPTIONS": "pool.max_size=50,pool.min_size=2,sslmode=require",
},
{
"default": {
@@ -363,6 +402,7 @@ class TestParseDbSettings:
"sslrootcert": None,
"sslcert": None,
"sslkey": None,
"application_name": "paperless-ngx",
"pool": {
"min_size": 2,
"max_size": 50,
@@ -390,6 +430,7 @@ class TestParseDbSettings:
"sslrootcert": None,
"sslcert": None,
"sslkey": None,
"application_name": "paperless-ngx",
"pool": {
"min_size": 1,
"max_size": 10,
@@ -419,6 +460,7 @@ class TestParseDbSettings:
"sslrootcert": "/certs/ca.crt",
"sslcert": None,
"sslkey": None,
"application_name": "paperless-ngx",
"connect_timeout": 30,
},
},
@@ -447,6 +489,7 @@ class TestParseDbSettings:
"cert": None,
"key": None,
},
"isolation_level": "read committed",
},
},
},
@@ -455,18 +498,17 @@ class TestParseDbSettings:
pytest.param(
{
"PAPERLESS_DBENGINE": "mariadb",
"PAPERLESS_DBHOST": "paperless-mariadb-host",
"PAPERLESS_DBPORT": "5555",
"PAPERLESS_DBHOST": "mariahost",
"PAPERLESS_DBNAME": "paperlessdb",
"PAPERLESS_DBUSER": "my-cool-user",
"PAPERLESS_DBPASS": "my-secure-password",
"PAPERLESS_DB_OPTIONS": "ssl.ca=/path/to/ca.pem;ssl_mode=REQUIRED",
"PAPERLESS_DB_OPTIONS": "ssl_mode=REQUIRED,ssl.ca=/path/to/ca.pem",
},
{
"default": {
"ENGINE": "django.db.backends.mysql",
"HOST": "paperless-mariadb-host",
"PORT": 5555,
"NAME": "paperless",
"HOST": "mariahost",
"NAME": "paperlessdb",
"USER": "my-cool-user",
"PASSWORD": "my-secure-password",
"OPTIONS": {
@@ -479,6 +521,7 @@ class TestParseDbSettings:
"cert": None,
"key": None,
},
"isolation_level": "read committed",
},
},
},
@@ -512,6 +555,7 @@ class TestParseDbSettings:
"key": "/certs/client.key",
},
"connect_timeout": 25,
"isolation_level": "read committed",
},
},
},
@@ -527,10 +571,8 @@ class TestParseDbSettings:
expected_database_settings: dict[str, dict],
) -> None:
"""Test various database configurations with defaults and overrides."""
# Clear environment and set test vars
mocker.patch.dict(os.environ, env_vars, clear=True)
# Update expected paths with actual tmp_path
if (
"default" in expected_database_settings
and expected_database_settings["default"]["NAME"] is None