From a1a7949d01af8ee4de20723e1e1afe7add166ee7 Mon Sep 17 00:00:00 2001 From: Trenton H <797416+stumpylog@users.noreply.github.com> Date: Tue, 14 Apr 2026 15:31:10 -0700 Subject: [PATCH] Adds database tuning defaults to parse_db_settings SQLite: WAL journal mode, NORMAL synchronous, 5s busy timeout, memory temp store, 128MB mmap, 64MB journal size limit, 8MB cache, and IMMEDIATE transaction mode for correct busy_timeout behaviour. PostgreSQL: application_name=paperless-ngx for pg_stat_activity. MariaDB: isolation_level=read committed to eliminate gap locking. --- src/paperless/settings/custom.py | 24 ++++++- .../tests/settings/test_custom_parsers.py | 66 +++++++++++++++---- 2 files changed, 76 insertions(+), 14 deletions(-) diff --git a/src/paperless/settings/custom.py b/src/paperless/settings/custom.py index 5459576c3..274dbbc05 100644 --- a/src/paperless/settings/custom.py +++ b/src/paperless/settings/custom.py @@ -224,7 +224,23 @@ def parse_db_settings(data_dir: Path) -> dict[str, dict[str, Any]]: "ENGINE": "django.db.backends.sqlite3", "NAME": str((data_dir / "db.sqlite3").resolve()), } - base_options = {} + base_options = { + # Django splits init_command on ";" and calls conn.execute() + # once per statement, so multiple PRAGMAs work correctly. + # foreign_keys is omitted — Django sets it natively. + "init_command": ( + "PRAGMA journal_mode=WAL;" + "PRAGMA synchronous=NORMAL;" + "PRAGMA busy_timeout=5000;" + "PRAGMA temp_store=MEMORY;" + "PRAGMA mmap_size=134217728;" + "PRAGMA journal_size_limit=67108864;" + "PRAGMA cache_size=-8000" + ), + # IMMEDIATE acquires the write lock at BEGIN, ensuring + # busy_timeout is respected from the start of the transaction. + "transaction_mode": "IMMEDIATE", + } case "postgresql": db_config = { @@ -240,6 +256,7 @@ def parse_db_settings(data_dir: Path) -> dict[str, dict[str, Any]]: "sslrootcert": os.getenv("PAPERLESS_DBSSLROOTCERT"), "sslcert": os.getenv("PAPERLESS_DBSSLCERT"), "sslkey": os.getenv("PAPERLESS_DBSSLKEY"), + "application_name": "paperless-ngx", } if (pool_size := get_int_from_env("PAPERLESS_DB_POOLSIZE")) is not None: @@ -267,6 +284,9 @@ def parse_db_settings(data_dir: Path) -> dict[str, dict[str, Any]]: "cert": os.getenv("PAPERLESS_DBSSLCERT"), "key": os.getenv("PAPERLESS_DBSSLKEY"), }, + # READ COMMITTED eliminates gap locking and reduces deadlocks. + # Requires binlog_format=ROW if binary logging is enabled. + "isolation_level": "read committed", } case _: # pragma: no cover raise NotImplementedError(engine) @@ -287,7 +307,7 @@ def parse_db_settings(data_dir: Path) -> dict[str, dict[str, Any]]: db_config["OPTIONS"] = parse_dict_from_str( os.getenv("PAPERLESS_DB_OPTIONS"), defaults=base_options, - separator=";", + separator=",", type_map={ # SQLite options "timeout": int, diff --git a/src/paperless/tests/settings/test_custom_parsers.py b/src/paperless/tests/settings/test_custom_parsers.py index 6fa7ad8eb..cf0321e80 100644 --- a/src/paperless/tests/settings/test_custom_parsers.py +++ b/src/paperless/tests/settings/test_custom_parsers.py @@ -296,8 +296,19 @@ class TestParseDbSettings: { "default": { "ENGINE": "django.db.backends.sqlite3", - "NAME": None, # Will be replaced with tmp_path - "OPTIONS": {}, + "NAME": None, # replaced with tmp_path in test body + "OPTIONS": { + "init_command": ( + "PRAGMA journal_mode=WAL;" + "PRAGMA synchronous=NORMAL;" + "PRAGMA busy_timeout=5000;" + "PRAGMA temp_store=MEMORY;" + "PRAGMA mmap_size=134217728;" + "PRAGMA journal_size_limit=67108864;" + "PRAGMA cache_size=-8000" + ), + "transaction_mode": "IMMEDIATE", + }, }, }, id="default-sqlite", @@ -310,14 +321,41 @@ class TestParseDbSettings: { "default": { "ENGINE": "django.db.backends.sqlite3", - "NAME": None, # Will be replaced with tmp_path + "NAME": None, "OPTIONS": { + "init_command": ( + "PRAGMA journal_mode=WAL;" + "PRAGMA synchronous=NORMAL;" + "PRAGMA busy_timeout=5000;" + "PRAGMA temp_store=MEMORY;" + "PRAGMA mmap_size=134217728;" + "PRAGMA journal_size_limit=67108864;" + "PRAGMA cache_size=-8000" + ), + "transaction_mode": "IMMEDIATE", "timeout": 30, }, }, }, id="sqlite-with-timeout-override", ), + pytest.param( + { + "PAPERLESS_DBENGINE": "sqlite", + "PAPERLESS_DB_OPTIONS": "init_command=PRAGMA journal_mode=DELETE;PRAGMA synchronous=FULL,transaction_mode=DEFERRED", + }, + { + "default": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": None, + "OPTIONS": { + "init_command": "PRAGMA journal_mode=DELETE;PRAGMA synchronous=FULL", + "transaction_mode": "DEFERRED", + }, + }, + }, + id="sqlite-init-command-override", + ), pytest.param( { "PAPERLESS_DBENGINE": "postgresql", @@ -335,6 +373,7 @@ class TestParseDbSettings: "sslrootcert": None, "sslcert": None, "sslkey": None, + "application_name": "paperless-ngx", }, }, }, @@ -348,7 +387,7 @@ class TestParseDbSettings: "PAPERLESS_DBNAME": "customdb", "PAPERLESS_DBUSER": "customuser", "PAPERLESS_DBPASS": "custompass", - "PAPERLESS_DB_OPTIONS": "pool.max_size=50;pool.min_size=2;sslmode=require", + "PAPERLESS_DB_OPTIONS": "pool.max_size=50,pool.min_size=2,sslmode=require", }, { "default": { @@ -363,6 +402,7 @@ class TestParseDbSettings: "sslrootcert": None, "sslcert": None, "sslkey": None, + "application_name": "paperless-ngx", "pool": { "min_size": 2, "max_size": 50, @@ -390,6 +430,7 @@ class TestParseDbSettings: "sslrootcert": None, "sslcert": None, "sslkey": None, + "application_name": "paperless-ngx", "pool": { "min_size": 1, "max_size": 10, @@ -419,6 +460,7 @@ class TestParseDbSettings: "sslrootcert": "/certs/ca.crt", "sslcert": None, "sslkey": None, + "application_name": "paperless-ngx", "connect_timeout": 30, }, }, @@ -447,6 +489,7 @@ class TestParseDbSettings: "cert": None, "key": None, }, + "isolation_level": "read committed", }, }, }, @@ -455,18 +498,17 @@ class TestParseDbSettings: pytest.param( { "PAPERLESS_DBENGINE": "mariadb", - "PAPERLESS_DBHOST": "paperless-mariadb-host", - "PAPERLESS_DBPORT": "5555", + "PAPERLESS_DBHOST": "mariahost", + "PAPERLESS_DBNAME": "paperlessdb", "PAPERLESS_DBUSER": "my-cool-user", "PAPERLESS_DBPASS": "my-secure-password", - "PAPERLESS_DB_OPTIONS": "ssl.ca=/path/to/ca.pem;ssl_mode=REQUIRED", + "PAPERLESS_DB_OPTIONS": "ssl_mode=REQUIRED,ssl.ca=/path/to/ca.pem", }, { "default": { "ENGINE": "django.db.backends.mysql", - "HOST": "paperless-mariadb-host", - "PORT": 5555, - "NAME": "paperless", + "HOST": "mariahost", + "NAME": "paperlessdb", "USER": "my-cool-user", "PASSWORD": "my-secure-password", "OPTIONS": { @@ -479,6 +521,7 @@ class TestParseDbSettings: "cert": None, "key": None, }, + "isolation_level": "read committed", }, }, }, @@ -512,6 +555,7 @@ class TestParseDbSettings: "key": "/certs/client.key", }, "connect_timeout": 25, + "isolation_level": "read committed", }, }, }, @@ -527,10 +571,8 @@ class TestParseDbSettings: expected_database_settings: dict[str, dict], ) -> None: """Test various database configurations with defaults and overrides.""" - # Clear environment and set test vars mocker.patch.dict(os.environ, env_vars, clear=True) - # Update expected paths with actual tmp_path if ( "default" in expected_database_settings and expected_database_settings["default"]["NAME"] is None