mirror of
https://github.com/domainaware/parsedmarc.git
synced 2026-11-07 09:19:22 +00:00
Merge commit from fork
* Stop logging credentials from configured URLs and the IPinfo token (GHSA-hjr3-rx4c-24cv) With --debug, the Splunk HEC, Elasticsearch and OpenSearch client-init lines and opensearch.set_hosts() logged configured URLs verbatim, so userinfo passwords reached the console and log_file (CWE-532). Triage found more paths of the same class: - the httpx "HTTP Request:" line, forwarded at --verbose/--debug, contains the full URL (str(httpx.URL) keeps userinfo and query), so webhook and HEC URL credentials and the IPinfo ?token= were logged on every request; - the PSL overrides, IP database and reverse DNS map downloaders logged their URL at DEBUG and str(HTTPStatusError), which embeds the full URL, at WARNING; - libpq echoes the DSN or a password fragment for some malformed PostgreSQL connection strings, logged as an output-client error. Add redact_url_credentials() (string methods only; urlsplit misparses scheme-less "user:pw@host:port") and use it at each log site; add a filter on the httpx logger that masks userinfo and the token query value; build download-failure warnings from the status code and the redacted request URL instead of the exception text; replace psycopg ProgrammingError detail from a connection_string with a fixed message. Docs now recommend separate user/password settings over URL credentials. Reported by helixnebulatech. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * Mask credentials in the Loki URL in the log and LokiError text The Loki output (#926) landed on master after this fix was written. Its url keeps userinfo (copy_with(path=...) preserves it), and the --debug "Initializing Loki client" line and both LokiError messages, which the CLI logs on a failed push, included it as-is: the same CWE-532 path as the Splunk HEC line. Mask it with redact_url_credentials() at all three sites. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * Withhold InvalidURL details, mask malformed URLs, and cover httpx2 and DoH nameservers Follow-up for GHSA-hjr3-rx4c-24cv (CWE-532): - httpx.InvalidURL quotes the component it rejected, so a password with an unencoded "/", "?" or "#" (RFC 3986 3.2: the authority ends there) leaked as "Invalid port: 'p4ssw0rd'", and its chained ValueError repeats it. Loki, Splunk HEC, the webhook and the three download loaders now report a fixed "details withheld" message raised from None. InvalidURL is not an httpx.HTTPError, so a malformed psl_overrides_url crashed startup; the loaders now treat it as a download failure. - redact_url_credentials() masks through the last "@" when the authority has no "@" but one follows it, so the --debug lines no longer print such passwords; the userinfo lookup is shared with the new nameserver masking. - dnspython 2.9 DoH logs its request line on the httpx2 logger, which was forwarded but not filtered; the filter now covers httpx2 and httpx2.URL. - [general] nameservers / --nameservers credentials are masked in the DNS pre-flight error, reverse DNS debug errors and per-report errors, and unusable https:// / tls:// ports are rejected before any query. - Docs recommend token/username settings for [loki] and [splunk_hec]; AGENTS.md gains rules for keeping credentials out of logs. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Sean Whalen <seanthegeek@users.noreply.github.com>
This commit is contained in:
17 files changed
+1595
-73
No files matched your search
@@ -3,6 +3,7 @@
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import traceback
|
||||
import unittest
|
||||
from typing import cast
|
||||
from unittest.mock import MagicMock, patch
|
||||
@@ -164,6 +165,44 @@ class TestLokiClientInit(unittest.TestCase):
|
||||
self.assertIsInstance(client.session.auth, httpx.BasicAuth)
|
||||
|
||||
|
||||
class TestLokiInvalidUrlDoesNotLeakPassword(unittest.TestCase):
|
||||
"""GHSA-hjr3-rx4c-24cv: a password with an unencoded "/", "?" or "#"
|
||||
ends the URL's authority early (RFC 3986 section 3.2), so httpx reads the
|
||||
text after the colon as the port and raises
|
||||
``InvalidURL("Invalid port: 'p4ssw0rd'")`` (httpx 0.28.1 _urlparse.py:
|
||||
``raise InvalidURL(f"Invalid port: {port!r}")``). Its implicit
|
||||
``__context__`` is a ValueError that repeats that part of the password
|
||||
too."""
|
||||
|
||||
MALFORMED = (
|
||||
("https://admin:p4ssw0rd/extra@loki.example.org:3100", "p4ssw0rd"),
|
||||
("https://admin:p4ss?w0rd@loki.example.org:3100", "p4ss"),
|
||||
("https://admin:p4ss#w0rd@loki.example.org:3100", "p4ss"),
|
||||
)
|
||||
|
||||
def test_invalid_url_raises_fixed_message_without_password(self):
|
||||
for url, secret in self.MALFORMED:
|
||||
with self.subTest(url=url):
|
||||
with self.assertRaises(ValueError) as ctx:
|
||||
LokiClient(url)
|
||||
self.assertEqual(
|
||||
str(ctx.exception),
|
||||
"Invalid Loki URL (details withheld because they may "
|
||||
"include credentials)",
|
||||
)
|
||||
self.assertIsNone(ctx.exception.__cause__)
|
||||
self.assertTrue(ctx.exception.__suppress_context__)
|
||||
trace = "".join(traceback.format_exception(ctx.exception))
|
||||
self.assertNotIn(secret, trace)
|
||||
|
||||
def test_valid_url_keeps_userinfo_for_requests(self):
|
||||
client = LokiClient("https://admin:p4ssw0rd@loki.example.org:3100")
|
||||
self.assertEqual(
|
||||
client.url,
|
||||
"https://admin:p4ssw0rd@loki.example.org:3100/loki/api/v1/push",
|
||||
)
|
||||
|
||||
|
||||
@patch(
|
||||
"parsedmarc.loki.parsed_aggregate_reports_to_csv_rows", return_value=AGGREGATE_ROWS
|
||||
)
|
||||
@@ -370,6 +409,28 @@ class TestLokiErrors(unittest.TestCase):
|
||||
client.save_aggregate_report_to_loki(AGGREGATE)
|
||||
self.assertIn("refused", str(ctx.exception))
|
||||
|
||||
def test_errors_mask_url_password(self, _rows):
|
||||
"""GHSA-hjr3-rx4c-24cv: the URL keeps its userinfo, and the CLI
|
||||
logs LokiError text, so the password must be masked in it."""
|
||||
secret = "s3cretPW"
|
||||
client = LokiClient(f"http://u:{secret}@loki.example.org:3100")
|
||||
client.session = MagicMock()
|
||||
_session(client).post.return_value = _ok_response(500)
|
||||
with self.assertRaises(LokiError) as status_ctx:
|
||||
client.save_aggregate_report_to_loki(AGGREGATE)
|
||||
_session(client).post.side_effect = httpx.ConnectError("refused")
|
||||
with self.assertRaises(LokiError) as network_ctx:
|
||||
client.save_aggregate_report_to_loki(AGGREGATE)
|
||||
for ctx in (status_ctx, network_ctx):
|
||||
with self.subTest(error=str(ctx.exception)):
|
||||
self.assertNotIn(secret, str(ctx.exception))
|
||||
self.assertIn("http://u:***@loki.example.org:3100", str(ctx.exception))
|
||||
# Redaction is message-only: the request still goes to the real URL
|
||||
self.assertEqual(
|
||||
_session(client).post.call_args.args[0],
|
||||
f"http://u:{secret}@loki.example.org:3100/loki/api/v1/push",
|
||||
)
|
||||
|
||||
def test_blocked_ingestion_status_raises(self, _rows):
|
||||
"""Loki returns 260 (block_ingestion_status_code default) and drops
|
||||
the logs when ingestion is blocked; that must not read as success."""
|
||||
|
||||
Reference in new issue
Block a user