mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-11-22 05:14:22 +00:00
Perf/fix: stop building a full-library IN filter for unrestricted RAG context
get_context_for_document() always materialized every document id a user can see into a Python list, even for a superuser (or no user at all), passing it through as a SQL IN filter. For a superuser, that's the whole library: - Past ~32,763 documents, this crashes outright: sqlite3.OperationalError: too many SQL variables (SQLite's SQLITE_MAX_VARIABLE_NUMBER is 32766 by default, and the query already binds embedding + k + a NE self-exclusion clause alongside the ids). - Below that cliff, vec0's IN-list evaluation is a nested loop (strncmp per row per allowed id), so it's quadratic in library size for no reason -- the filter was never going to exclude anything. get_objects_for_user_owner_aware() already returns every Document for a superuser (guardian's own with_superuser shortcut), so skipping straight to document_ids=None changes nothing about which documents are considered, only how we get there. Also: - Drop the pointless sorted() in _document_id_filters(): a SQL IN clause doesn't care about order, so it was pure overhead on every call. - Add a hard guard in _build_where() so a future regression (or a legitimately huge permission-restricted user) fails closed -- no rows, a logged warning -- instead of a cryptic OperationalError. Since this filter scopes document access, failing closed rather than skipping the filter is the only safe way to handle an oversized list. First item from VECTOR_STORE_PERF_BACKLOG.md (an audit done alongside perf/13314-vecstore-point-delete, deferred to its own branch since that one was already large). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
2537e94f4e
commit
191a09b871
5 files changed
+191
-15
No files matched your search
@@ -96,19 +96,26 @@ def get_context_for_document(
|
|||||||
user: User | None = None,
|
user: User | None = None,
|
||||||
max_docs: int = 5,
|
max_docs: int = 5,
|
||||||
) -> str:
|
) -> str:
|
||||||
visible_documents = (
|
# None means "no restriction" to query_similar_documents. A superuser
|
||||||
get_objects_for_user_owner_aware(
|
# (like no user at all) can see every document, so skip materializing
|
||||||
user,
|
# every visible pk into a Python list and passing it through as a SQL
|
||||||
"view_document",
|
# IN filter: for a large library that is a wasted quadratic scan in the
|
||||||
Document,
|
# vector store at best, and past ~32,763 documents a hard
|
||||||
)
|
# sqlite3.OperationalError (SQLite's bound-parameter limit) at worst.
|
||||||
if user
|
# get_objects_for_user_owner_aware() would return every Document for a
|
||||||
else None
|
# superuser anyway (guardian's own with_superuser shortcut), so this
|
||||||
)
|
# changes nothing about which documents are considered -- only how we
|
||||||
|
# get there.
|
||||||
visible_document_ids = (
|
visible_document_ids = (
|
||||||
list(visible_documents.values_list("pk", flat=True))
|
None
|
||||||
if visible_documents is not None
|
if user is None or user.is_superuser
|
||||||
else None
|
else list(
|
||||||
|
get_objects_for_user_owner_aware(
|
||||||
|
user,
|
||||||
|
"view_document",
|
||||||
|
Document,
|
||||||
|
).values_list("pk", flat=True),
|
||||||
|
)
|
||||||
)
|
)
|
||||||
similar_docs = query_similar_documents(
|
similar_docs = query_similar_documents(
|
||||||
document=doc,
|
document=doc,
|
||||||
|
|||||||
@@ -397,7 +397,7 @@ def _document_id_filters(doc_ids):
|
|||||||
MetadataFilter(
|
MetadataFilter(
|
||||||
key="document_id",
|
key="document_id",
|
||||||
operator=FilterOperator.IN,
|
operator=FilterOperator.IN,
|
||||||
value=sorted(doc_ids),
|
value=list(doc_ids),
|
||||||
),
|
),
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -3,6 +3,8 @@ from unittest.mock import MagicMock
|
|||||||
from unittest.mock import patch
|
from unittest.mock import patch
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
import pytest_mock
|
||||||
|
from django.contrib.auth.models import User
|
||||||
from django.test import override_settings
|
from django.test import override_settings
|
||||||
|
|
||||||
from documents.models import Document
|
from documents.models import Document
|
||||||
@@ -278,3 +280,104 @@ def test_get_context_for_document_no_similar_docs(mock_document):
|
|||||||
with patch("paperless_ai.ai_classifier.query_similar_documents", return_value=[]):
|
with patch("paperless_ai.ai_classifier.query_similar_documents", return_value=[]):
|
||||||
result = get_context_for_document(mock_document)
|
result = get_context_for_document(mock_document)
|
||||||
assert result == ""
|
assert result == ""
|
||||||
|
|
||||||
|
|
||||||
|
class TestGetContextForDocumentVisibility:
|
||||||
|
"""get_context_for_document must not materialize every visible document
|
||||||
|
id for a user who can already see the whole library: a superuser (like
|
||||||
|
no user at all) gets document_ids=None (no restriction) straight
|
||||||
|
through to query_similar_documents(), instead of a full-library IN
|
||||||
|
filter that is wasteful at best and, past ~32,763 documents, a hard
|
||||||
|
sqlite3.OperationalError at worst (SQLite's bound-parameter limit).
|
||||||
|
"""
|
||||||
|
|
||||||
|
def test_skips_permission_lookup_for_superuser(
|
||||||
|
self,
|
||||||
|
mock_document: MagicMock,
|
||||||
|
mock_similar_documents: list[MagicMock],
|
||||||
|
mocker: pytest_mock.MockerFixture,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A superuser
|
||||||
|
WHEN:
|
||||||
|
- get_context_for_document() is called
|
||||||
|
THEN:
|
||||||
|
- get_objects_for_user_owner_aware() is never called, and
|
||||||
|
query_similar_documents() is called with document_ids=None
|
||||||
|
"""
|
||||||
|
mock_query = mocker.patch(
|
||||||
|
"paperless_ai.ai_classifier.query_similar_documents",
|
||||||
|
return_value=mock_similar_documents,
|
||||||
|
)
|
||||||
|
mock_get_objects = mocker.patch(
|
||||||
|
"paperless_ai.ai_classifier.get_objects_for_user_owner_aware",
|
||||||
|
)
|
||||||
|
user = mocker.MagicMock(spec=User)
|
||||||
|
user.is_superuser = True
|
||||||
|
|
||||||
|
get_context_for_document(mock_document, user, max_docs=2)
|
||||||
|
|
||||||
|
mock_get_objects.assert_not_called()
|
||||||
|
assert mock_query.call_args.kwargs["document_ids"] is None
|
||||||
|
|
||||||
|
def test_skips_permission_lookup_when_no_user(
|
||||||
|
self,
|
||||||
|
mock_document: MagicMock,
|
||||||
|
mock_similar_documents: list[MagicMock],
|
||||||
|
mocker: pytest_mock.MockerFixture,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- No user (user=None)
|
||||||
|
WHEN:
|
||||||
|
- get_context_for_document() is called
|
||||||
|
THEN:
|
||||||
|
- get_objects_for_user_owner_aware() is never called, and
|
||||||
|
query_similar_documents() is called with document_ids=None
|
||||||
|
"""
|
||||||
|
mock_query = mocker.patch(
|
||||||
|
"paperless_ai.ai_classifier.query_similar_documents",
|
||||||
|
return_value=mock_similar_documents,
|
||||||
|
)
|
||||||
|
mock_get_objects = mocker.patch(
|
||||||
|
"paperless_ai.ai_classifier.get_objects_for_user_owner_aware",
|
||||||
|
)
|
||||||
|
|
||||||
|
get_context_for_document(mock_document, None, max_docs=2)
|
||||||
|
|
||||||
|
mock_get_objects.assert_not_called()
|
||||||
|
assert mock_query.call_args.kwargs["document_ids"] is None
|
||||||
|
|
||||||
|
def test_restricts_to_visible_documents_for_non_superuser(
|
||||||
|
self,
|
||||||
|
mock_document: MagicMock,
|
||||||
|
mock_similar_documents: list[MagicMock],
|
||||||
|
mocker: pytest_mock.MockerFixture,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A non-superuser with a specific set of visible documents
|
||||||
|
WHEN:
|
||||||
|
- get_context_for_document() is called
|
||||||
|
THEN:
|
||||||
|
- query_similar_documents() is called with exactly that user's
|
||||||
|
visible document ids, unchanged from before this optimization
|
||||||
|
"""
|
||||||
|
mock_query = mocker.patch(
|
||||||
|
"paperless_ai.ai_classifier.query_similar_documents",
|
||||||
|
return_value=mock_similar_documents,
|
||||||
|
)
|
||||||
|
mock_queryset = mocker.MagicMock()
|
||||||
|
mock_queryset.values_list.return_value = [1, 2, 3]
|
||||||
|
mock_get_objects = mocker.patch(
|
||||||
|
"paperless_ai.ai_classifier.get_objects_for_user_owner_aware",
|
||||||
|
return_value=mock_queryset,
|
||||||
|
)
|
||||||
|
user = mocker.MagicMock(spec=User)
|
||||||
|
user.is_superuser = False
|
||||||
|
|
||||||
|
get_context_for_document(mock_document, user, max_docs=2)
|
||||||
|
|
||||||
|
mock_get_objects.assert_called_once_with(user, "view_document", Document)
|
||||||
|
assert mock_query.call_args.kwargs["document_ids"] == [1, 2, 3]
|
||||||
@@ -17,6 +17,7 @@ from paperless_ai.migrations import Migration
|
|||||||
from paperless_ai.migrations import m0001_v1_to_v2
|
from paperless_ai.migrations import m0001_v1_to_v2
|
||||||
from paperless_ai.tables import DocumentChunksTable
|
from paperless_ai.tables import DocumentChunksTable
|
||||||
from paperless_ai.tables import DocumentMetaTable
|
from paperless_ai.tables import DocumentMetaTable
|
||||||
|
from paperless_ai.vector_store import _MAX_IN_VALUES
|
||||||
from paperless_ai.vector_store import DB_FILENAME
|
from paperless_ai.vector_store import DB_FILENAME
|
||||||
from paperless_ai.vector_store import DEFAULT_TABLE_NAME
|
from paperless_ai.vector_store import DEFAULT_TABLE_NAME
|
||||||
from paperless_ai.vector_store import SCHEMA_VERSION
|
from paperless_ai.vector_store import SCHEMA_VERSION
|
||||||
@@ -296,8 +297,47 @@ class TestBuildWhere:
|
|||||||
assert where == "1 = 0"
|
assert where == "1 = 0"
|
||||||
assert params == []
|
assert params == []
|
||||||
|
|
||||||
def test_query_with_untranslatable_filter_returns_no_rows(self, store) -> None:
|
def test_fails_closed_when_in_filter_exceeds_max_values(
|
||||||
store.add([make_node("a1", 1), make_node("b1", 2)])
|
self,
|
||||||
|
caplog: pytest.LogCaptureFixture,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- An IN filter with more values than _MAX_IN_VALUES (SQLite's
|
||||||
|
own bound-parameter limit is 32766; this guard sits below
|
||||||
|
that with headroom for the query's other bound parameters)
|
||||||
|
WHEN:
|
||||||
|
- _build_where() translates it to SQL
|
||||||
|
THEN:
|
||||||
|
- It fails closed ("1 = 0", no params) instead of building an
|
||||||
|
IN clause SQLite would reject, and logs a warning -- this
|
||||||
|
filter scopes document access, so refusing to build it must
|
||||||
|
never widen the scope to "everything" by accident
|
||||||
|
"""
|
||||||
|
oversized = _in_filter([str(i) for i in range(_MAX_IN_VALUES + 1)])
|
||||||
|
|
||||||
|
with caplog.at_level("WARNING"):
|
||||||
|
where, params = _build_where(oversized)
|
||||||
|
|
||||||
|
assert where == "(1 = 0)"
|
||||||
|
assert params == []
|
||||||
|
assert "document_id" in caplog.text
|
||||||
|
|
||||||
|
def test_query_with_untranslatable_filter_returns_no_rows(
|
||||||
|
self,
|
||||||
|
store: PaperlessSqliteVecVectorStore,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A store with one chunk each for documents 1 and 2
|
||||||
|
WHEN:
|
||||||
|
- A query and a get_nodes() call are issued with a filter whose
|
||||||
|
only clause is an untranslatable nested MetadataFilters
|
||||||
|
THEN:
|
||||||
|
- Neither call raises, and both fail closed by returning no rows
|
||||||
|
(rather than emitting invalid or unfiltered SQL)
|
||||||
|
"""
|
||||||
|
store.add([make_node("a1", "1"), make_node("b1", "2")])
|
||||||
nested = MetadataFilters(
|
nested = MetadataFilters(
|
||||||
filters=[
|
filters=[
|
||||||
MetadataFilter(
|
MetadataFilter(
|
||||||
|
|||||||
@@ -75,6 +75,17 @@ class _Row(NamedTuple):
|
|||||||
embedding: bytes
|
embedding: bytes
|
||||||
|
|
||||||
|
|
||||||
|
# _build_where(): the largest IN value list translated into bound SQL
|
||||||
|
# parameters. SQLite's own hard limit (SQLITE_MAX_VARIABLE_NUMBER) is 32766
|
||||||
|
# by default; this leaves headroom below that for the query's other bound
|
||||||
|
# parameters (the embedding blob, k, and any NE clause) and for the limit
|
||||||
|
# itself to move. An IN filter this large should not happen in practice --
|
||||||
|
# callers are expected to pass None (no filter) rather than every id when
|
||||||
|
# the filter would not actually narrow anything -- so this is a guard
|
||||||
|
# against a future regression, not a normal code path.
|
||||||
|
_MAX_IN_VALUES = 32700
|
||||||
|
|
||||||
|
|
||||||
def _pack(embedding: Sequence[float]) -> bytes:
|
def _pack(embedding: Sequence[float]) -> bytes:
|
||||||
return struct.pack(f"{len(embedding)}f", *embedding)
|
return struct.pack(f"{len(embedding)}f", *embedding)
|
||||||
|
|
||||||
@@ -119,6 +130,21 @@ def _build_where(filters: MetadataFilters | None) -> tuple[str, list[int]]:
|
|||||||
if not values: # pragma: no cover
|
if not values: # pragma: no cover
|
||||||
clauses.append("1 = 0")
|
clauses.append("1 = 0")
|
||||||
continue
|
continue
|
||||||
|
if len(values) > _MAX_IN_VALUES:
|
||||||
|
# Fail closed (see the empty-clauses case below) rather than
|
||||||
|
# let SQLite raise "too many SQL variables" past its own
|
||||||
|
# limit: this filter scopes document access, so an IN list
|
||||||
|
# too large to safely bind must match no rows, never widen
|
||||||
|
# the scope to "everything" by accident.
|
||||||
|
logger.warning(
|
||||||
|
"Refusing to build an IN filter on %r with %d values "
|
||||||
|
"(over the %d-value safety limit); returning no rows.",
|
||||||
|
f.key,
|
||||||
|
len(values),
|
||||||
|
_MAX_IN_VALUES,
|
||||||
|
)
|
||||||
|
clauses.append("1 = 0")
|
||||||
|
continue
|
||||||
placeholders = ",".join("?" for _ in values)
|
placeholders = ",".join("?" for _ in values)
|
||||||
clauses.append(f"{f.key} IN ({placeholders})")
|
clauses.append(f"{f.key} IN ({placeholders})")
|
||||||
params.extend(values)
|
params.extend(values)
|
||||||
|
|||||||
Reference in new issue
Block a user