mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-07-31 16:15:58 +00:00
stumpylog
91647eecb5
Adds standalone profiling tooling (never merged into dev/main): a persistent, self-healing Postgres 18 container helper (run_with_postgres.sh/stop_postgres.sh), a scale-profile dataset seeder (seed.py) whose document counts and guardian permission-row ratios mirror real bug reports (#13276, #13161), and a typed profiling harness (harness.py) for timing/query-count comparisons and EXPLAIN ANALYZE capture, gated by require_postgres() so it never silently runs on SQLite. seed.py samples distinct (subject, document) pairs up front rather than drawing with replacement, since guardian's assign_perm() dedupes on the (subject, object, permission) triple -- with-replacement sampling against a small group pool collides heavily (birthday paradox) and undercounts the target permission-row ratios otherwise.
68 lines
2.0 KiB
Python
68 lines
2.0 KiB
Python
# profiling/harness.py
|
|
from __future__ import annotations
|
|
|
|
import time
|
|
from dataclasses import dataclass
|
|
from typing import TYPE_CHECKING
|
|
from typing import Any
|
|
from typing import Generic
|
|
from typing import TypeVar
|
|
|
|
import pytest
|
|
from django.db import connection
|
|
from django.test.utils import CaptureQueriesContext
|
|
|
|
if TYPE_CHECKING:
|
|
from collections.abc import Callable
|
|
|
|
from django.db.models import QuerySet
|
|
|
|
T = TypeVar("T")
|
|
|
|
|
|
def require_postgres() -> None:
|
|
if connection.vendor != "postgresql":
|
|
pytest.skip(
|
|
"Profiling requires PostgreSQL to reflect production query "
|
|
"planning; SQLite does not reproduce the varchar-cast planner "
|
|
"pathology this work fixes.",
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class ProfileResult(Generic[T]):
|
|
best_seconds: float
|
|
all_seconds: tuple[float, ...]
|
|
query_count: int
|
|
result: T
|
|
|
|
|
|
def run_profile(fn: Callable[[], T], *, repeat: int = 5) -> ProfileResult[T]:
|
|
require_postgres()
|
|
all_seconds: list[float] = []
|
|
result: T | None = None
|
|
query_count = 0
|
|
for i in range(repeat):
|
|
with CaptureQueriesContext(connection) as ctx:
|
|
start = time.perf_counter()
|
|
result = fn()
|
|
all_seconds.append(time.perf_counter() - start)
|
|
if i == repeat - 1:
|
|
query_count = len(ctx.captured_queries)
|
|
assert result is not None # repeat >= 1 guarantees at least one assignment
|
|
return ProfileResult(
|
|
best_seconds=min(all_seconds),
|
|
all_seconds=tuple(all_seconds),
|
|
query_count=query_count,
|
|
result=result,
|
|
)
|
|
|
|
|
|
def capture_explain_analyze(queryset: QuerySet[Any]) -> str:
|
|
if connection.vendor != "postgresql":
|
|
raise RuntimeError("EXPLAIN ANALYZE capture is Postgres-only")
|
|
sql, params = queryset.query.sql_with_params()
|
|
with connection.cursor() as cur:
|
|
cur.execute(f"EXPLAIN ANALYZE {sql}", params)
|
|
return "\n".join(row[0] for row in cur.fetchall())
|