From 4eaf8032ad2bb1b400ed9d49455f97800339113c Mon Sep 17 00:00:00 2001 From: stumpylog <797416+stumpylog@users.noreply.github.com> Date: Tue, 22 Sep 2026 08:35:09 -0700 Subject: [PATCH] Add guarded httpx transports and client factories The transports install the outbound guard on httpcore's connection pool after checking its exact layout, and accept no proxy, uds or retries options. httpx, httpcore and anyio become declared dependencies, pinned narrowly where private attributes are relied on. Co-Authored-By: Claude Opus 5 --- pyproject.toml | 3 + src/paperless/network.py | 83 ++++++++++++++++ src/paperless/tests/test_network.py | 145 ++++++++++++++++++++++++++++ uv.lock | 6 ++ 4 files changed, 237 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 3aba1dcf8..12a06969d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ classifiers = [ # TODO: Move certain things to groups and then utilize that further # This will allow testing to not install a webserver, mysql, etc dependencies = [ + "anyio>=4.12", "azure-ai-documentintelligence>=1.0.2", "babel>=2.17", "bleach~=6.4.0", @@ -47,6 +48,8 @@ dependencies = [ "filelock~=3.32.0", "flower>=2.0.1,<2.2", "gotenberg-client[httpx]~=1.0", + "httpcore~=1.0.9", + "httpx~=0.28.1", "httpx-oauth~=0.17", "ijson>=3.5.1", "imap-tools>=1.14,<1.16", diff --git a/src/paperless/network.py b/src/paperless/network.py index 7448a4a00..431b6c59a 100644 --- a/src/paperless/network.py +++ b/src/paperless/network.py @@ -20,6 +20,9 @@ import anyio import httpcore import httpx +# Not exported by httpcore; the guard asserts it is still the async default. +from httpcore._backends.auto import AutoBackend + logger = logging.getLogger("paperless.network") # requires-python is >=3.11, so no PEP 695 `type` statement. @@ -415,6 +418,86 @@ class _GuardedAsyncBackend(httpcore.AsyncNetworkBackend): await self._inner.sleep(seconds) +_LAYOUT_ERROR = ( + "Unexpected httpx transport layout; refusing to create a transport " + "without the outbound connection guard" +) + + +class GuardedHTTPTransport(httpx.HTTPTransport): + """ + httpx transport whose connections pass through the outbound guard. + + Deliberately accepts no proxy, uds or retries options: a proxy would be + dialled instead of the destination, and a unix socket bypasses TCP + entirely. Adding an option here is a reviewed change, not a pass-through. + """ + + def __init__(self, *, allow_internal: bool) -> None: + super().__init__() + # httpx has no public hook for the network backend. Check the exact + # layout before swapping so an httpx or httpcore change fails loudly. + pool = self._pool + if ( + type(pool) is not httpcore.ConnectionPool + or type(pool._network_backend) is not httpcore.SyncBackend + ): + raise RuntimeError(_LAYOUT_ERROR) + pool._network_backend = _GuardedSyncBackend( + pool._network_backend, + allow_internal=allow_internal, + ) + + +class GuardedAsyncHTTPTransport(httpx.AsyncHTTPTransport): + """Async twin of GuardedHTTPTransport.""" + + def __init__(self, *, allow_internal: bool) -> None: + super().__init__() + pool = self._pool + if ( + type(pool) is not httpcore.AsyncConnectionPool + or type(pool._network_backend) is not AutoBackend + ): + raise RuntimeError(_LAYOUT_ERROR) + pool._network_backend = _GuardedAsyncBackend( + pool._network_backend, + allow_internal=allow_internal, + ) + + +def create_guarded_httpx_client( + url: str, + *, + allow_internal: bool, + timeout: float, +) -> httpx.Client: + """ + Validate ``url`` up front, then build a client that re-checks at connect + time. The up-front check turns static misconfiguration into a ValueError + before any retry layer sees it. + """ + validate_outbound_http_url(url, allow_internal=allow_internal) + return httpx.Client( + transport=GuardedHTTPTransport(allow_internal=allow_internal), + timeout=timeout, + ) + + +def create_guarded_async_httpx_client( + url: str, + *, + allow_internal: bool, + timeout: float, +) -> httpx.AsyncClient: + """Async twin of create_guarded_httpx_client.""" + validate_outbound_http_url(url, allow_internal=allow_internal) + return httpx.AsyncClient( + transport=GuardedAsyncHTTPTransport(allow_internal=allow_internal), + timeout=timeout, + ) + + def resolve_hostname_ips(hostname: str) -> list[str]: try: addr_info = socket.getaddrinfo(hostname, None) diff --git a/src/paperless/tests/test_network.py b/src/paperless/tests/test_network.py index 3bfa82233..bc9b21e12 100644 --- a/src/paperless/tests/test_network.py +++ b/src/paperless/tests/test_network.py @@ -10,12 +10,16 @@ from unittest.mock import MagicMock import anyio import httpcore import httpx +import ollama import pytest from celery.utils.serialization import get_pickleable_exception +from httpcore._backends.auto import AutoBackend from pytest_mock import MockerFixture from paperless.network import MAX_ADDRESSES_TRIED from paperless.network import BlockReason +from paperless.network import GuardedAsyncHTTPTransport +from paperless.network import GuardedHTTPTransport from paperless.network import HostResolutionError from paperless.network import OutboundRequestBlockedError from paperless.network import PinnedHostHTTPTransport @@ -23,6 +27,8 @@ from paperless.network import _GuardedAsyncBackend from paperless.network import _GuardedSyncBackend from paperless.network import aresolve_public_addresses from paperless.network import blocked_message +from paperless.network import create_guarded_async_httpx_client +from paperless.network import create_guarded_httpx_client from paperless.network import is_public_ip from paperless.network import resolve_public_addresses from paperless.network import validate_outbound_http_url @@ -1287,3 +1293,142 @@ class TestGuardedAsyncBackend: await _GuardedAsyncBackend(inner, allow_internal=False).sleep(0.5) assert inner.slept == [0.5] + + +class TestGuardedTransports: + def test_sync_transport_installs_guard(self) -> None: + """ + GIVEN: + - A guarded sync transport + WHEN: + - It is constructed + THEN: + - The httpcore pool dials through the guard wrapping the stock backend + """ + transport = GuardedHTTPTransport(allow_internal=False) + + assert type(transport._pool) is httpcore.ConnectionPool + backend = transport._pool._network_backend + assert isinstance(backend, _GuardedSyncBackend) + assert type(backend._inner) is httpcore.SyncBackend + assert backend._allow_internal is False + + def test_async_transport_installs_guard(self) -> None: + """ + GIVEN: + - A guarded async transport + WHEN: + - It is constructed + THEN: + - The httpcore pool dials through the guard wrapping the stock backend + """ + transport = GuardedAsyncHTTPTransport(allow_internal=True) + + assert type(transport._pool) is httpcore.AsyncConnectionPool + backend = transport._pool._network_backend + assert isinstance(backend, _GuardedAsyncBackend) + assert type(backend._inner) is AutoBackend + assert backend._allow_internal is True + + def test_sync_transport_refuses_unexpected_layout( + self, + mocker: MockerFixture, + ) -> None: + """ + GIVEN: + - An httpcore whose default sync backend is not the expected type + WHEN: + - A guarded transport is constructed + THEN: + - Construction fails instead of producing an unguarded transport + """ + mocker.patch.object(httpcore, "SyncBackend", type("OtherBackend", (), {})) + + with pytest.raises(RuntimeError, match="transport layout"): + GuardedHTTPTransport(allow_internal=False) + + def test_async_transport_refuses_unexpected_layout( + self, + mocker: MockerFixture, + ) -> None: + """ + GIVEN: + - An httpcore whose default async backend is not the expected type + WHEN: + - A guarded async transport is constructed + THEN: + - Construction fails instead of producing an unguarded transport + """ + mocker.patch("paperless.network.AutoBackend", type("OtherBackend", (), {})) + + with pytest.raises(RuntimeError, match="transport layout"): + GuardedAsyncHTTPTransport(allow_internal=False) + + def test_factory_validates_url_first(self) -> None: + """ + GIVEN: + - An internal endpoint with internal addresses disallowed + WHEN: + - A guarded client is requested + THEN: + - The up-front check raises ValueError + """ + with pytest.raises(ValueError, match="non-public address"): + create_guarded_httpx_client( + "http://127.0.0.1:8080", + allow_internal=False, + timeout=5.0, + ) + + def test_factory_builds_guarded_clients(self) -> None: + """ + GIVEN: + - A public endpoint + WHEN: + - Sync and async guarded clients are requested + THEN: + - Both use guarded transports and the requested timeout + """ + with create_guarded_httpx_client( + "http://93.184.216.34", + allow_internal=False, + timeout=5.0, + ) as client: + assert isinstance(client._transport, GuardedHTTPTransport) + assert client.timeout == httpx.Timeout(5.0) + + async_client = create_guarded_async_httpx_client( + "http://93.184.216.34", + allow_internal=False, + timeout=5.0, + ) + assert isinstance(async_client._transport, GuardedAsyncHTTPTransport) + assert async_client.timeout == httpx.Timeout(5.0) + + def test_environment_proxies_are_ignored( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """ + GIVEN: + - HTTP_PROXY and HTTPS_PROXY set in the environment + WHEN: + - A guarded client is built, directly or inside ollama.Client + THEN: + - No proxy transport is mounted, so requests go through the guard + """ + monkeypatch.setenv("HTTP_PROXY", "http://proxy.invalid:3128") + monkeypatch.setenv("HTTPS_PROXY", "http://proxy.invalid:3128") + + with create_guarded_httpx_client( + "http://93.184.216.34", + allow_internal=False, + timeout=5.0, + ) as client: + assert client._mounts == {} + + ollama_client = ollama.Client( + host="http://93.184.216.34:11434", + transport=GuardedHTTPTransport(allow_internal=False), + ) + assert ollama_client._client._mounts == {} diff --git a/uv.lock b/uv.lock index 94c3533c7..9879e6618 100644 --- a/uv.lock +++ b/uv.lock @@ -2874,6 +2874,7 @@ name = "paperless-ngx" version = "3.2.1" source = { virtual = "." } dependencies = [ + { name = "anyio" }, { name = "azure-ai-documentintelligence" }, { name = "babel" }, { name = "bleach" }, @@ -2902,6 +2903,8 @@ dependencies = [ { name = "filelock" }, { name = "flower" }, { name = "gotenberg-client", extra = ["httpx"] }, + { name = "httpcore" }, + { name = "httpx" }, { name = "httpx-oauth" }, { name = "ijson" }, { name = "imap-tools" }, @@ -3026,6 +3029,7 @@ typing = [ [package.metadata] requires-dist = [ + { name = "anyio", specifier = ">=4.12" }, { name = "azure-ai-documentintelligence", specifier = ">=1.0.2" }, { name = "babel", specifier = ">=2.17" }, { name = "bleach", specifier = "~=6.4.0" }, @@ -3055,6 +3059,8 @@ requires-dist = [ { name = "flower", specifier = ">=2.0.1,<2.2" }, { name = "gotenberg-client", extras = ["httpx"], specifier = "~=1.0" }, { name = "granian", extras = ["uvloop"], marker = "extra == 'webserver'", specifier = ">=2.7,<2.9" }, + { name = "httpcore", specifier = "~=1.0.9" }, + { name = "httpx", specifier = "~=0.28.1" }, { name = "httpx-oauth", specifier = "~=0.17" }, { name = "ijson", specifier = ">=3.5.1" }, { name = "imap-tools", specifier = ">=1.14,<1.16" },