mirror of
https://github.com/domainaware/parsedmarc.git
synced 2026-11-06 16:59:23 +00:00
Merge commit from fork
* Stop logging credentials from configured URLs and the IPinfo token (GHSA-hjr3-rx4c-24cv) With --debug, the Splunk HEC, Elasticsearch and OpenSearch client-init lines and opensearch.set_hosts() logged configured URLs verbatim, so userinfo passwords reached the console and log_file (CWE-532). Triage found more paths of the same class: - the httpx "HTTP Request:" line, forwarded at --verbose/--debug, contains the full URL (str(httpx.URL) keeps userinfo and query), so webhook and HEC URL credentials and the IPinfo ?token= were logged on every request; - the PSL overrides, IP database and reverse DNS map downloaders logged their URL at DEBUG and str(HTTPStatusError), which embeds the full URL, at WARNING; - libpq echoes the DSN or a password fragment for some malformed PostgreSQL connection strings, logged as an output-client error. Add redact_url_credentials() (string methods only; urlsplit misparses scheme-less "user:pw@host:port") and use it at each log site; add a filter on the httpx logger that masks userinfo and the token query value; build download-failure warnings from the status code and the redacted request URL instead of the exception text; replace psycopg ProgrammingError detail from a connection_string with a fixed message. Docs now recommend separate user/password settings over URL credentials. Reported by helixnebulatech. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * Mask credentials in the Loki URL in the log and LokiError text The Loki output (#926) landed on master after this fix was written. Its url keeps userinfo (copy_with(path=...) preserves it), and the --debug "Initializing Loki client" line and both LokiError messages, which the CLI logs on a failed push, included it as-is: the same CWE-532 path as the Splunk HEC line. Mask it with redact_url_credentials() at all three sites. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * Withhold InvalidURL details, mask malformed URLs, and cover httpx2 and DoH nameservers Follow-up for GHSA-hjr3-rx4c-24cv (CWE-532): - httpx.InvalidURL quotes the component it rejected, so a password with an unencoded "/", "?" or "#" (RFC 3986 3.2: the authority ends there) leaked as "Invalid port: 'p4ssw0rd'", and its chained ValueError repeats it. Loki, Splunk HEC, the webhook and the three download loaders now report a fixed "details withheld" message raised from None. InvalidURL is not an httpx.HTTPError, so a malformed psl_overrides_url crashed startup; the loaders now treat it as a download failure. - redact_url_credentials() masks through the last "@" when the authority has no "@" but one follows it, so the --debug lines no longer print such passwords; the userinfo lookup is shared with the new nameserver masking. - dnspython 2.9 DoH logs its request line on the httpx2 logger, which was forwarded but not filtered; the filter now covers httpx2 and httpx2.URL. - [general] nameservers / --nameservers credentials are masked in the DNS pre-flight error, reverse DNS debug errors and per-report errors, and unusable https:// / tls:// ports are rejected before any query. - Docs recommend token/username settings for [loki] and [splunk_hec]; AGENTS.md gains rules for keeping credentials out of logs. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Sean Whalen <seanthegeek@users.noreply.github.com>
This commit is contained in:
17 files changed
+1595
-73
No files matched your search
@@ -311,11 +311,74 @@ class TestPostgreSQLConstruction(unittest.TestCase):
|
||||
"""A driver-level connection error is wrapped in PostgreSQLError."""
|
||||
with patch("parsedmarc.postgres.psycopg") as mock_psycopg:
|
||||
mock_psycopg.Error = Exception
|
||||
mock_psycopg.ProgrammingError = type("ProgrammingError", (Exception,), {})
|
||||
mock_psycopg.connect.side_effect = mock_psycopg.Error("refused")
|
||||
with self.assertRaises(PostgreSQLError) as ctx:
|
||||
PostgreSQLClient(host="localhost")
|
||||
self.assertIn("refused", str(ctx.exception))
|
||||
|
||||
def test_connect_failure_detail_is_url_redacted(self):
|
||||
"""A non-parse driver error keeps its detail, with URL credentials
|
||||
masked."""
|
||||
programming_error = type("ProgrammingError", (Exception,), {})
|
||||
with patch("parsedmarc.postgres.psycopg") as mock_psycopg:
|
||||
mock_psycopg.Error = Exception
|
||||
mock_psycopg.ProgrammingError = programming_error
|
||||
mock_psycopg.connect.side_effect = mock_psycopg.Error(
|
||||
'connection to "postgresql://u:PGSECRET@db.example/x" failed'
|
||||
)
|
||||
with self.assertRaises(PostgreSQLError) as ctx:
|
||||
PostgreSQLClient(connection_string="postgresql://u:PGSECRET@h/db")
|
||||
message = str(ctx.exception)
|
||||
self.assertNotIn("PGSECRET", message)
|
||||
self.assertIn("u:***@db.example/x", message)
|
||||
|
||||
def test_malformed_connection_string_detail_withheld(self):
|
||||
"""A ProgrammingError from parsing the connection string (which libpq
|
||||
words with the whole string or a password fragment) is replaced with
|
||||
a fixed message that still identifies the problem."""
|
||||
programming_error = type("ProgrammingError", (Exception,), {})
|
||||
for dsn, driver_message in (
|
||||
(
|
||||
"postgresql://u:PGSECRET@[::1/db",
|
||||
'end of string reached when looking for matching "]" in IPv6 '
|
||||
'host address in URI: "postgresql://u:PGSECRET@[::1/db"',
|
||||
),
|
||||
(
|
||||
"postgresql://u:PG%ZZSECRET@h/db",
|
||||
'invalid percent-encoded token: "PG%ZZSECRET"',
|
||||
),
|
||||
):
|
||||
with self.subTest(dsn=dsn):
|
||||
with patch("parsedmarc.postgres.psycopg") as mock_psycopg:
|
||||
mock_psycopg.Error = Exception
|
||||
mock_psycopg.ProgrammingError = programming_error
|
||||
mock_psycopg.connect.side_effect = programming_error(driver_message)
|
||||
with self.assertRaises(PostgreSQLError) as ctx:
|
||||
PostgreSQLClient(connection_string=dsn)
|
||||
message = str(ctx.exception)
|
||||
self.assertNotIn("PGSECRET", message)
|
||||
self.assertNotIn("SECRET", message)
|
||||
self.assertIn("Invalid PostgreSQL connection_string", message)
|
||||
|
||||
@unittest.skipIf(
|
||||
importlib.util.find_spec("psycopg") is None, "psycopg is not installed"
|
||||
)
|
||||
def test_real_psycopg_malformed_connection_string_withheld(self):
|
||||
"""With the real driver, both malformed strings fail in parsing (no
|
||||
network I/O) and the raised message carries no part of the password."""
|
||||
for dsn in (
|
||||
"postgresql://u:PGSECRET@[::1/db",
|
||||
"postgresql://u:PG%ZZSECRET@h/db",
|
||||
):
|
||||
with self.subTest(dsn=dsn):
|
||||
with self.assertRaises(PostgreSQLError) as ctx:
|
||||
PostgreSQLClient(connection_string=dsn)
|
||||
self.assertNotIn("SECRET", str(ctx.exception))
|
||||
self.assertIn(
|
||||
"Invalid PostgreSQL connection_string", str(ctx.exception)
|
||||
)
|
||||
|
||||
def test_create_tables_executes_all_ddl(self):
|
||||
"""create_tables issues CREATE TABLE for every table and the indexes."""
|
||||
client, mock_conn = _make_client()
|
||||
|
||||
Reference in new issue
Block a user