diff --git a/pyproject.toml b/pyproject.toml index 3aba1dcf8..12a06969d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ classifiers = [ # TODO: Move certain things to groups and then utilize that further # This will allow testing to not install a webserver, mysql, etc dependencies = [ + "anyio>=4.12", "azure-ai-documentintelligence>=1.0.2", "babel>=2.17", "bleach~=6.4.0", @@ -47,6 +48,8 @@ dependencies = [ "filelock~=3.32.0", "flower>=2.0.1,<2.2", "gotenberg-client[httpx]~=1.0", + "httpcore~=1.0.9", + "httpx~=0.28.1", "httpx-oauth~=0.17", "ijson>=3.5.1", "imap-tools>=1.14,<1.16", diff --git a/src/paperless/network.py b/src/paperless/network.py index 7448a4a00..431b6c59a 100644 --- a/src/paperless/network.py +++ b/src/paperless/network.py @@ -20,6 +20,9 @@ import anyio import httpcore import httpx +# Not exported by httpcore; the guard asserts it is still the async default. +from httpcore._backends.auto import AutoBackend + logger = logging.getLogger("paperless.network") # requires-python is >=3.11, so no PEP 695 `type` statement. @@ -415,6 +418,86 @@ class _GuardedAsyncBackend(httpcore.AsyncNetworkBackend): await self._inner.sleep(seconds) +_LAYOUT_ERROR = ( + "Unexpected httpx transport layout; refusing to create a transport " + "without the outbound connection guard" +) + + +class GuardedHTTPTransport(httpx.HTTPTransport): + """ + httpx transport whose connections pass through the outbound guard. + + Deliberately accepts no proxy, uds or retries options: a proxy would be + dialled instead of the destination, and a unix socket bypasses TCP + entirely. Adding an option here is a reviewed change, not a pass-through. + """ + + def __init__(self, *, allow_internal: bool) -> None: + super().__init__() + # httpx has no public hook for the network backend. Check the exact + # layout before swapping so an httpx or httpcore change fails loudly. + pool = self._pool + if ( + type(pool) is not httpcore.ConnectionPool + or type(pool._network_backend) is not httpcore.SyncBackend + ): + raise RuntimeError(_LAYOUT_ERROR) + pool._network_backend = _GuardedSyncBackend( + pool._network_backend, + allow_internal=allow_internal, + ) + + +class GuardedAsyncHTTPTransport(httpx.AsyncHTTPTransport): + """Async twin of GuardedHTTPTransport.""" + + def __init__(self, *, allow_internal: bool) -> None: + super().__init__() + pool = self._pool + if ( + type(pool) is not httpcore.AsyncConnectionPool + or type(pool._network_backend) is not AutoBackend + ): + raise RuntimeError(_LAYOUT_ERROR) + pool._network_backend = _GuardedAsyncBackend( + pool._network_backend, + allow_internal=allow_internal, + ) + + +def create_guarded_httpx_client( + url: str, + *, + allow_internal: bool, + timeout: float, +) -> httpx.Client: + """ + Validate ``url`` up front, then build a client that re-checks at connect + time. The up-front check turns static misconfiguration into a ValueError + before any retry layer sees it. + """ + validate_outbound_http_url(url, allow_internal=allow_internal) + return httpx.Client( + transport=GuardedHTTPTransport(allow_internal=allow_internal), + timeout=timeout, + ) + + +def create_guarded_async_httpx_client( + url: str, + *, + allow_internal: bool, + timeout: float, +) -> httpx.AsyncClient: + """Async twin of create_guarded_httpx_client.""" + validate_outbound_http_url(url, allow_internal=allow_internal) + return httpx.AsyncClient( + transport=GuardedAsyncHTTPTransport(allow_internal=allow_internal), + timeout=timeout, + ) + + def resolve_hostname_ips(hostname: str) -> list[str]: try: addr_info = socket.getaddrinfo(hostname, None) diff --git a/src/paperless/tests/test_network.py b/src/paperless/tests/test_network.py index 3bfa82233..bc9b21e12 100644 --- a/src/paperless/tests/test_network.py +++ b/src/paperless/tests/test_network.py @@ -10,12 +10,16 @@ from unittest.mock import MagicMock import anyio import httpcore import httpx +import ollama import pytest from celery.utils.serialization import get_pickleable_exception +from httpcore._backends.auto import AutoBackend from pytest_mock import MockerFixture from paperless.network import MAX_ADDRESSES_TRIED from paperless.network import BlockReason +from paperless.network import GuardedAsyncHTTPTransport +from paperless.network import GuardedHTTPTransport from paperless.network import HostResolutionError from paperless.network import OutboundRequestBlockedError from paperless.network import PinnedHostHTTPTransport @@ -23,6 +27,8 @@ from paperless.network import _GuardedAsyncBackend from paperless.network import _GuardedSyncBackend from paperless.network import aresolve_public_addresses from paperless.network import blocked_message +from paperless.network import create_guarded_async_httpx_client +from paperless.network import create_guarded_httpx_client from paperless.network import is_public_ip from paperless.network import resolve_public_addresses from paperless.network import validate_outbound_http_url @@ -1287,3 +1293,142 @@ class TestGuardedAsyncBackend: await _GuardedAsyncBackend(inner, allow_internal=False).sleep(0.5) assert inner.slept == [0.5] + + +class TestGuardedTransports: + def test_sync_transport_installs_guard(self) -> None: + """ + GIVEN: + - A guarded sync transport + WHEN: + - It is constructed + THEN: + - The httpcore pool dials through the guard wrapping the stock backend + """ + transport = GuardedHTTPTransport(allow_internal=False) + + assert type(transport._pool) is httpcore.ConnectionPool + backend = transport._pool._network_backend + assert isinstance(backend, _GuardedSyncBackend) + assert type(backend._inner) is httpcore.SyncBackend + assert backend._allow_internal is False + + def test_async_transport_installs_guard(self) -> None: + """ + GIVEN: + - A guarded async transport + WHEN: + - It is constructed + THEN: + - The httpcore pool dials through the guard wrapping the stock backend + """ + transport = GuardedAsyncHTTPTransport(allow_internal=True) + + assert type(transport._pool) is httpcore.AsyncConnectionPool + backend = transport._pool._network_backend + assert isinstance(backend, _GuardedAsyncBackend) + assert type(backend._inner) is AutoBackend + assert backend._allow_internal is True + + def test_sync_transport_refuses_unexpected_layout( + self, + mocker: MockerFixture, + ) -> None: + """ + GIVEN: + - An httpcore whose default sync backend is not the expected type + WHEN: + - A guarded transport is constructed + THEN: + - Construction fails instead of producing an unguarded transport + """ + mocker.patch.object(httpcore, "SyncBackend", type("OtherBackend", (), {})) + + with pytest.raises(RuntimeError, match="transport layout"): + GuardedHTTPTransport(allow_internal=False) + + def test_async_transport_refuses_unexpected_layout( + self, + mocker: MockerFixture, + ) -> None: + """ + GIVEN: + - An httpcore whose default async backend is not the expected type + WHEN: + - A guarded async transport is constructed + THEN: + - Construction fails instead of producing an unguarded transport + """ + mocker.patch("paperless.network.AutoBackend", type("OtherBackend", (), {})) + + with pytest.raises(RuntimeError, match="transport layout"): + GuardedAsyncHTTPTransport(allow_internal=False) + + def test_factory_validates_url_first(self) -> None: + """ + GIVEN: + - An internal endpoint with internal addresses disallowed + WHEN: + - A guarded client is requested + THEN: + - The up-front check raises ValueError + """ + with pytest.raises(ValueError, match="non-public address"): + create_guarded_httpx_client( + "http://127.0.0.1:8080", + allow_internal=False, + timeout=5.0, + ) + + def test_factory_builds_guarded_clients(self) -> None: + """ + GIVEN: + - A public endpoint + WHEN: + - Sync and async guarded clients are requested + THEN: + - Both use guarded transports and the requested timeout + """ + with create_guarded_httpx_client( + "http://93.184.216.34", + allow_internal=False, + timeout=5.0, + ) as client: + assert isinstance(client._transport, GuardedHTTPTransport) + assert client.timeout == httpx.Timeout(5.0) + + async_client = create_guarded_async_httpx_client( + "http://93.184.216.34", + allow_internal=False, + timeout=5.0, + ) + assert isinstance(async_client._transport, GuardedAsyncHTTPTransport) + assert async_client.timeout == httpx.Timeout(5.0) + + def test_environment_proxies_are_ignored( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """ + GIVEN: + - HTTP_PROXY and HTTPS_PROXY set in the environment + WHEN: + - A guarded client is built, directly or inside ollama.Client + THEN: + - No proxy transport is mounted, so requests go through the guard + """ + monkeypatch.setenv("HTTP_PROXY", "http://proxy.invalid:3128") + monkeypatch.setenv("HTTPS_PROXY", "http://proxy.invalid:3128") + + with create_guarded_httpx_client( + "http://93.184.216.34", + allow_internal=False, + timeout=5.0, + ) as client: + assert client._mounts == {} + + ollama_client = ollama.Client( + host="http://93.184.216.34:11434", + transport=GuardedHTTPTransport(allow_internal=False), + ) + assert ollama_client._client._mounts == {} diff --git a/uv.lock b/uv.lock index 94c3533c7..9879e6618 100644 --- a/uv.lock +++ b/uv.lock @@ -2874,6 +2874,7 @@ name = "paperless-ngx" version = "3.2.1" source = { virtual = "." } dependencies = [ + { name = "anyio" }, { name = "azure-ai-documentintelligence" }, { name = "babel" }, { name = "bleach" }, @@ -2902,6 +2903,8 @@ dependencies = [ { name = "filelock" }, { name = "flower" }, { name = "gotenberg-client", extra = ["httpx"] }, + { name = "httpcore" }, + { name = "httpx" }, { name = "httpx-oauth" }, { name = "ijson" }, { name = "imap-tools" }, @@ -3026,6 +3029,7 @@ typing = [ [package.metadata] requires-dist = [ + { name = "anyio", specifier = ">=4.12" }, { name = "azure-ai-documentintelligence", specifier = ">=1.0.2" }, { name = "babel", specifier = ">=2.17" }, { name = "bleach", specifier = "~=6.4.0" }, @@ -3055,6 +3059,8 @@ requires-dist = [ { name = "flower", specifier = ">=2.0.1,<2.2" }, { name = "gotenberg-client", extras = ["httpx"], specifier = "~=1.0" }, { name = "granian", extras = ["uvloop"], marker = "extra == 'webserver'", specifier = ">=2.7,<2.9" }, + { name = "httpcore", specifier = "~=1.0.9" }, + { name = "httpx", specifier = "~=0.28.1" }, { name = "httpx-oauth", specifier = "~=0.17" }, { name = "ijson", specifier = ">=3.5.1" }, { name = "imap-tools", specifier = ">=1.14,<1.16" },