amorfo77/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2026-05-17 12:05:24 +00:00
Code Issues Projects Releases Wiki Activity

Adds database tuning defaults to parse_db_settings

Browse Source 
SQLite: WAL journal mode, NORMAL synchronous, 5s busy timeout, memory
temp store, 128MB mmap, 64MB journal size limit, 8MB cache, and
IMMEDIATE transaction mode for correct busy_timeout behaviour.
PostgreSQL: application_name=paperless-ngx for pg_stat_activity.
MariaDB: isolation_level=read committed to eliminate gap locking.
This commit is contained in:
Trenton H
2026-04-14 15:31:10 -07:00
parent d9061d5e55
commit a1a7949d01
2 changed files with 76 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/paperless/settings/custom.py
+22 -2
View File
@@ -224,7 +224,23 @@ def parse_db_settings(data_dir: Path) -> dict[str, dict[str, Any]]:
"ENGINE": "django.db.backends.sqlite3",
"NAME": str((data_dir / "db.sqlite3").resolve()),
}
base_options = {}
base_options = {
# Django splits init_command on ";" and calls conn.execute()
# once per statement, so multiple PRAGMAs work correctly.
# foreign_keys is omitted — Django sets it natively.
"init_command": (
"PRAGMA journal_mode=WAL;"
"PRAGMA synchronous=NORMAL;"
"PRAGMA busy_timeout=5000;"
"PRAGMA temp_store=MEMORY;"
"PRAGMA mmap_size=134217728;"
"PRAGMA journal_size_limit=67108864;"
"PRAGMA cache_size=-8000"
),
# IMMEDIATE acquires the write lock at BEGIN, ensuring
# busy_timeout is respected from the start of the transaction.
"transaction_mode": "IMMEDIATE",
}
case "postgresql":
db_config = {
@@ -240,6 +256,7 @@ def parse_db_settings(data_dir: Path) -> dict[str, dict[str, Any]]:
"sslrootcert": os.getenv("PAPERLESS_DBSSLROOTCERT"),
"sslcert": os.getenv("PAPERLESS_DBSSLCERT"),
"sslkey": os.getenv("PAPERLESS_DBSSLKEY"),
"application_name": "paperless-ngx",
}
if (pool_size := get_int_from_env("PAPERLESS_DB_POOLSIZE")) is not None:
@@ -267,6 +284,9 @@ def parse_db_settings(data_dir: Path) -> dict[str, dict[str, Any]]:
"cert": os.getenv("PAPERLESS_DBSSLCERT"),
"key": os.getenv("PAPERLESS_DBSSLKEY"),
},
# READ COMMITTED eliminates gap locking and reduces deadlocks.
# Requires binlog_format=ROW if binary logging is enabled.
"isolation_level": "read committed",
}
case _: # pragma: no cover
raise NotImplementedError(engine)
@@ -287,7 +307,7 @@ def parse_db_settings(data_dir: Path) -> dict[str, dict[str, Any]]:
db_config["OPTIONS"] = parse_dict_from_str(
os.getenv("PAPERLESS_DB_OPTIONS"),
defaults=base_options,
separator=";",
separator=",",
type_map={
# SQLite options
"timeout": int,
src/paperless/tests/settings/test_custom_parsers.py
+54 -12
View File
@@ -296,8 +296,19 @@ class TestParseDbSettings:
{
"default": {
"ENGINE": "django.db.backends.sqlite3",
"NAME": None, # Will be replaced with tmp_path
"OPTIONS": {},
"NAME": None, # replaced with tmp_path in test body
"OPTIONS": {
"init_command": (
"PRAGMA journal_mode=WAL;"
"PRAGMA synchronous=NORMAL;"
"PRAGMA busy_timeout=5000;"
"PRAGMA temp_store=MEMORY;"
"PRAGMA mmap_size=134217728;"
"PRAGMA journal_size_limit=67108864;"
"PRAGMA cache_size=-8000"
),
"transaction_mode": "IMMEDIATE",
},
},
},
id="default-sqlite",
@@ -310,14 +321,41 @@ class TestParseDbSettings:
{
"default": {
"ENGINE": "django.db.backends.sqlite3",
"NAME": None, # Will be replaced with tmp_path
"NAME": None,
"OPTIONS": {
"init_command": (
"PRAGMA journal_mode=WAL;"
"PRAGMA synchronous=NORMAL;"
"PRAGMA busy_timeout=5000;"
"PRAGMA temp_store=MEMORY;"
"PRAGMA mmap_size=134217728;"
"PRAGMA journal_size_limit=67108864;"
"PRAGMA cache_size=-8000"
),
"transaction_mode": "IMMEDIATE",
"timeout": 30,
},
},
},
id="sqlite-with-timeout-override",
),
pytest.param(
{
"PAPERLESS_DBENGINE": "sqlite",
"PAPERLESS_DB_OPTIONS": "init_command=PRAGMA journal_mode=DELETE;PRAGMA synchronous=FULL,transaction_mode=DEFERRED",
},
{
"default": {
"ENGINE": "django.db.backends.sqlite3",
"NAME": None,
"OPTIONS": {
"init_command": "PRAGMA journal_mode=DELETE;PRAGMA synchronous=FULL",
"transaction_mode": "DEFERRED",
},
},
},
id="sqlite-init-command-override",
),
pytest.param(
{
"PAPERLESS_DBENGINE": "postgresql",
@@ -335,6 +373,7 @@ class TestParseDbSettings:
"sslrootcert": None,
"sslcert": None,
"sslkey": None,
"application_name": "paperless-ngx",
},
},
},
@@ -348,7 +387,7 @@ class TestParseDbSettings:
"PAPERLESS_DBNAME": "customdb",
"PAPERLESS_DBUSER": "customuser",
"PAPERLESS_DBPASS": "custompass",
"PAPERLESS_DB_OPTIONS": "pool.max_size=50;pool.min_size=2;sslmode=require",
"PAPERLESS_DB_OPTIONS": "pool.max_size=50,pool.min_size=2,sslmode=require",
},
{
"default": {
@@ -363,6 +402,7 @@ class TestParseDbSettings:
"sslrootcert": None,
"sslcert": None,
"sslkey": None,
"application_name": "paperless-ngx",
"pool": {
"min_size": 2,
"max_size": 50,
@@ -390,6 +430,7 @@ class TestParseDbSettings:
"sslrootcert": None,
"sslcert": None,
"sslkey": None,
"application_name": "paperless-ngx",
"pool": {
"min_size": 1,
"max_size": 10,
@@ -419,6 +460,7 @@ class TestParseDbSettings:
"sslrootcert": "/certs/ca.crt",
"sslcert": None,
"sslkey": None,
"application_name": "paperless-ngx",
"connect_timeout": 30,
},
},
@@ -447,6 +489,7 @@ class TestParseDbSettings:
"cert": None,
"key": None,
},
"isolation_level": "read committed",
},
},
},
@@ -455,18 +498,17 @@ class TestParseDbSettings:
pytest.param(
{
"PAPERLESS_DBENGINE": "mariadb",
"PAPERLESS_DBHOST": "paperless-mariadb-host",
"PAPERLESS_DBPORT": "5555",
"PAPERLESS_DBHOST": "mariahost",
"PAPERLESS_DBNAME": "paperlessdb",
"PAPERLESS_DBUSER": "my-cool-user",
"PAPERLESS_DBPASS": "my-secure-password",
"PAPERLESS_DB_OPTIONS": "ssl.ca=/path/to/ca.pem;ssl_mode=REQUIRED",
"PAPERLESS_DB_OPTIONS": "ssl_mode=REQUIRED,ssl.ca=/path/to/ca.pem",
},
{
"default": {
"ENGINE": "django.db.backends.mysql",
"HOST": "paperless-mariadb-host",
"PORT": 5555,
"NAME": "paperless",
"HOST": "mariahost",
"NAME": "paperlessdb",
"USER": "my-cool-user",
"PASSWORD": "my-secure-password",
"OPTIONS": {
@@ -479,6 +521,7 @@ class TestParseDbSettings:
"cert": None,
"key": None,
},
"isolation_level": "read committed",
},
},
},
@@ -512,6 +555,7 @@ class TestParseDbSettings:
"key": "/certs/client.key",
},
"connect_timeout": 25,
"isolation_level": "read committed",
},
},
},
@@ -527,10 +571,8 @@ class TestParseDbSettings:
expected_database_settings: dict[str, dict],
) -> None:
"""Test various database configurations with defaults and overrides."""
# Clear environment and set test vars
mocker.patch.dict(os.environ, env_vars, clear=True)
# Update expected paths with actual tmp_path
if (
"default" in expected_database_settings
and expected_database_settings["default"]["NAME"] is None