mirror of
https://github.com/domainaware/parsedmarc.git
synced 2026-11-06 16:59:23 +00:00
* Stop logging credentials from configured URLs and the IPinfo token (GHSA-hjr3-rx4c-24cv) With --debug, the Splunk HEC, Elasticsearch and OpenSearch client-init lines and opensearch.set_hosts() logged configured URLs verbatim, so userinfo passwords reached the console and log_file (CWE-532). Triage found more paths of the same class: - the httpx "HTTP Request:" line, forwarded at --verbose/--debug, contains the full URL (str(httpx.URL) keeps userinfo and query), so webhook and HEC URL credentials and the IPinfo ?token= were logged on every request; - the PSL overrides, IP database and reverse DNS map downloaders logged their URL at DEBUG and str(HTTPStatusError), which embeds the full URL, at WARNING; - libpq echoes the DSN or a password fragment for some malformed PostgreSQL connection strings, logged as an output-client error. Add redact_url_credentials() (string methods only; urlsplit misparses scheme-less "user:pw@host:port") and use it at each log site; add a filter on the httpx logger that masks userinfo and the token query value; build download-failure warnings from the status code and the redacted request URL instead of the exception text; replace psycopg ProgrammingError detail from a connection_string with a fixed message. Docs now recommend separate user/password settings over URL credentials. Reported by helixnebulatech. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * Mask credentials in the Loki URL in the log and LokiError text The Loki output (#926) landed on master after this fix was written. Its url keeps userinfo (copy_with(path=...) preserves it), and the --debug "Initializing Loki client" line and both LokiError messages, which the CLI logs on a failed push, included it as-is: the same CWE-532 path as the Splunk HEC line. Mask it with redact_url_credentials() at all three sites. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * Withhold InvalidURL details, mask malformed URLs, and cover httpx2 and DoH nameservers Follow-up for GHSA-hjr3-rx4c-24cv (CWE-532): - httpx.InvalidURL quotes the component it rejected, so a password with an unencoded "/", "?" or "#" (RFC 3986 3.2: the authority ends there) leaked as "Invalid port: 'p4ssw0rd'", and its chained ValueError repeats it. Loki, Splunk HEC, the webhook and the three download loaders now report a fixed "details withheld" message raised from None. InvalidURL is not an httpx.HTTPError, so a malformed psl_overrides_url crashed startup; the loaders now treat it as a download failure. - redact_url_credentials() masks through the last "@" when the authority has no "@" but one follows it, so the --debug lines no longer print such passwords; the userinfo lookup is shared with the new nameserver masking. - dnspython 2.9 DoH logs its request line on the httpx2 logger, which was forwarded but not filtered; the filter now covers httpx2 and httpx2.URL. - [general] nameservers / --nameservers credentials are masked in the DNS pre-flight error, reverse DNS debug errors and per-report errors, and unusable https:// / tls:// ports are rejected before any query. - Docs recommend token/username settings for [loki] and [splunk_hec]; AGENTS.md gains rules for keeping credentials out of logs. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Sean Whalen <seanthegeek@users.noreply.github.com>
180 lines
7.0 KiB
Python
180 lines
7.0 KiB
Python
"""Tests for parsedmarc.webhook"""
|
|
|
|
import unittest
|
|
from unittest.mock import MagicMock
|
|
|
|
from parsedmarc.webhook import WebhookClient
|
|
|
|
|
|
def _client():
|
|
return WebhookClient(
|
|
aggregate_url="http://agg.example.com",
|
|
failure_url="http://fail.example.com",
|
|
smtp_tls_url="http://tls.example.com",
|
|
)
|
|
|
|
|
|
class TestWebhookClientInit(unittest.TestCase):
|
|
"""The constructor stores URLs per report type. A mix-up here
|
|
would route reports to the wrong endpoint silently."""
|
|
|
|
def test_urls_and_timeout_stored(self):
|
|
client = _client()
|
|
self.assertEqual(client.aggregate_url, "http://agg.example.com")
|
|
self.assertEqual(client.failure_url, "http://fail.example.com")
|
|
self.assertEqual(client.smtp_tls_url, "http://tls.example.com")
|
|
self.assertEqual(client.timeout, 60)
|
|
|
|
def test_custom_timeout_respected(self):
|
|
client = WebhookClient(
|
|
aggregate_url="a", failure_url="f", smtp_tls_url="t", timeout=120
|
|
)
|
|
self.assertEqual(client.timeout, 120)
|
|
|
|
def test_session_headers_set(self):
|
|
"""The Content-Type is required by virtually every webhook
|
|
receiver to know how to deserialize the body."""
|
|
client = _client()
|
|
self.assertEqual(client.session.headers["Content-Type"], "application/json")
|
|
self.assertIn("parsedmarc", client.session.headers["User-Agent"])
|
|
|
|
|
|
class TestWebhookClientSaveMethods(unittest.TestCase):
|
|
"""Each save_* sends the payload to the URL configured for that
|
|
report type. A typo on which URL each method uses would
|
|
permanently mis-route reports of that type."""
|
|
|
|
def test_aggregate_posts_to_aggregate_url(self):
|
|
client = _client()
|
|
client.session = MagicMock()
|
|
client.save_aggregate_report_to_webhook('{"agg": 1}')
|
|
client.session.post.assert_called_once_with(
|
|
"http://agg.example.com", content='{"agg": 1}', timeout=60
|
|
)
|
|
|
|
def test_failure_posts_to_failure_url(self):
|
|
client = _client()
|
|
client.session = MagicMock()
|
|
client.save_failure_report_to_webhook('{"fail": 1}')
|
|
client.session.post.assert_called_once_with(
|
|
"http://fail.example.com", content='{"fail": 1}', timeout=60
|
|
)
|
|
|
|
def test_smtp_tls_posts_to_smtp_tls_url(self):
|
|
client = _client()
|
|
client.session = MagicMock()
|
|
client.save_smtp_tls_report_to_webhook('{"tls": 1}')
|
|
client.session.post.assert_called_once_with(
|
|
"http://tls.example.com", content='{"tls": 1}', timeout=60
|
|
)
|
|
|
|
|
|
class TestWebhookClientDictPayload(unittest.TestCase):
|
|
"""``_send_to_webhook`` accepts ``bytes | str | dict``. A dict is
|
|
form-encoded via httpx's ``data=``; string/bytes payloads must use
|
|
``content=`` since httpx's ``data=`` is form-encoding only."""
|
|
|
|
def test_dict_payload_uses_data_kwarg(self):
|
|
client = _client()
|
|
client.session = MagicMock()
|
|
client._send_to_webhook("http://agg.example.com", {"agg": 1})
|
|
client.session.post.assert_called_once_with(
|
|
"http://agg.example.com", data={"agg": 1}, timeout=60
|
|
)
|
|
|
|
|
|
class TestWebhookErrorHandling(unittest.TestCase):
|
|
"""HTTP / network failures from the webhook receiver must NOT
|
|
abort the surrounding parse-and-output batch — they're logged
|
|
and swallowed. Misbehaving webhooks shouldn't take down DMARC
|
|
processing."""
|
|
|
|
def test_network_error_is_logged_and_swallowed(self):
|
|
client = _client()
|
|
client.session = MagicMock()
|
|
client.session.post.side_effect = OSError("connection refused")
|
|
with self.assertLogs("parsedmarc.log", level="ERROR") as cm:
|
|
# Should NOT raise.
|
|
client.save_aggregate_report_to_webhook('{"a": 1}')
|
|
self.assertTrue(any("Webhook Error" in m for m in cm.output))
|
|
self.assertTrue(any("connection refused" in m for m in cm.output))
|
|
|
|
def test_error_in_failure_save_is_swallowed(self):
|
|
client = _client()
|
|
client.session = MagicMock()
|
|
client.session.post.side_effect = RuntimeError("timeout")
|
|
with self.assertLogs("parsedmarc.log", level="ERROR"):
|
|
client.save_failure_report_to_webhook('{"f": 1}')
|
|
|
|
def test_error_in_smtp_tls_save_is_swallowed(self):
|
|
client = _client()
|
|
client.session = MagicMock()
|
|
client.session.post.side_effect = RuntimeError("boom")
|
|
with self.assertLogs("parsedmarc.log", level="ERROR"):
|
|
client.save_smtp_tls_report_to_webhook('{"t": 1}')
|
|
|
|
|
|
class TestWebhookInvalidUrlDoesNotLeakPassword(unittest.TestCase):
|
|
"""GHSA-hjr3-rx4c-24cv: a password with an unencoded "/", "?" or "#"
|
|
ends the URL's authority early (RFC 3986 section 3.2), so httpx reads the
|
|
text after the colon as the port and raises
|
|
``InvalidURL("Invalid port: 'p4ssw0rd'")`` when posting (httpx 0.28.1
|
|
_urlparse.py: ``raise InvalidURL(f"Invalid port: {port!r}")``), before
|
|
any network I/O. The error is logged on every report."""
|
|
|
|
MALFORMED = (
|
|
("https://admin:p4ssw0rd/extra@hooks.example.com/x", "p4ssw0rd"),
|
|
("https://admin:p4ss?w0rd@hooks.example.com/x", "p4ss"),
|
|
("https://admin:p4ss#w0rd@hooks.example.com/x", "p4ss"),
|
|
)
|
|
|
|
def test_invalid_url_is_logged_without_password(self):
|
|
saves = (
|
|
"save_aggregate_report_to_webhook",
|
|
"save_failure_report_to_webhook",
|
|
"save_smtp_tls_report_to_webhook",
|
|
)
|
|
for url, secret in self.MALFORMED:
|
|
client = WebhookClient(url, url, url)
|
|
for method in saves:
|
|
with self.subTest(url=url, method=method):
|
|
with self.assertLogs("parsedmarc.log", level="ERROR") as cm:
|
|
getattr(client, method)('{"a": 1}') # must not raise
|
|
self.assertEqual(
|
|
cm.output,
|
|
[
|
|
"ERROR:parsedmarc.log:Webhook Error: invalid webhook "
|
|
"URL (details withheld because they may include "
|
|
"credentials)"
|
|
],
|
|
)
|
|
self.assertNotIn(secret, "\n".join(cm.output))
|
|
|
|
def test_valid_url_is_the_one_posted(self):
|
|
url = "https://admin:p4ssw0rd@hooks.example.com/x"
|
|
client = WebhookClient(url, url, url)
|
|
client.session = MagicMock()
|
|
client.save_aggregate_report_to_webhook('{"a": 1}')
|
|
client.session.post.assert_called_once_with(url, content='{"a": 1}', timeout=60)
|
|
|
|
|
|
class TestWebhookClientClose(unittest.TestCase):
|
|
def test_close_closes_session(self):
|
|
client = _client()
|
|
mock_close = MagicMock()
|
|
client.session.close = mock_close
|
|
client.close()
|
|
mock_close.assert_called_once()
|
|
|
|
|
|
class TestWebhookBackwardCompatAlias(unittest.TestCase):
|
|
def test_forensic_alias_points_to_failure_method(self):
|
|
self.assertIs(
|
|
WebhookClient.save_forensic_report_to_webhook,
|
|
WebhookClient.save_failure_report_to_webhook,
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main(verbosity=2)
|