mirror of
https://github.com/domainaware/parsedmarc.git
synced 2026-11-11 03:11:33 +00:00
48ea47adee
* Stop logging credentials from configured URLs and the IPinfo token (GHSA-hjr3-rx4c-24cv) With --debug, the Splunk HEC, Elasticsearch and OpenSearch client-init lines and opensearch.set_hosts() logged configured URLs verbatim, so userinfo passwords reached the console and log_file (CWE-532). Triage found more paths of the same class: - the httpx "HTTP Request:" line, forwarded at --verbose/--debug, contains the full URL (str(httpx.URL) keeps userinfo and query), so webhook and HEC URL credentials and the IPinfo ?token= were logged on every request; - the PSL overrides, IP database and reverse DNS map downloaders logged their URL at DEBUG and str(HTTPStatusError), which embeds the full URL, at WARNING; - libpq echoes the DSN or a password fragment for some malformed PostgreSQL connection strings, logged as an output-client error. Add redact_url_credentials() (string methods only; urlsplit misparses scheme-less "user:pw@host:port") and use it at each log site; add a filter on the httpx logger that masks userinfo and the token query value; build download-failure warnings from the status code and the redacted request URL instead of the exception text; replace psycopg ProgrammingError detail from a connection_string with a fixed message. Docs now recommend separate user/password settings over URL credentials. Reported by helixnebulatech. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * Mask credentials in the Loki URL in the log and LokiError text The Loki output (#926) landed on master after this fix was written. Its url keeps userinfo (copy_with(path=...) preserves it), and the --debug "Initializing Loki client" line and both LokiError messages, which the CLI logs on a failed push, included it as-is: the same CWE-532 path as the Splunk HEC line. Mask it with redact_url_credentials() at all three sites. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * Withhold InvalidURL details, mask malformed URLs, and cover httpx2 and DoH nameservers Follow-up for GHSA-hjr3-rx4c-24cv (CWE-532): - httpx.InvalidURL quotes the component it rejected, so a password with an unencoded "/", "?" or "#" (RFC 3986 3.2: the authority ends there) leaked as "Invalid port: 'p4ssw0rd'", and its chained ValueError repeats it. Loki, Splunk HEC, the webhook and the three download loaders now report a fixed "details withheld" message raised from None. InvalidURL is not an httpx.HTTPError, so a malformed psl_overrides_url crashed startup; the loaders now treat it as a download failure. - redact_url_credentials() masks through the last "@" when the authority has no "@" but one follows it, so the --debug lines no longer print such passwords; the userinfo lookup is shared with the new nameserver masking. - dnspython 2.9 DoH logs its request line on the httpx2 logger, which was forwarded but not filtered; the filter now covers httpx2 and httpx2.URL. - [general] nameservers / --nameservers credentials are masked in the DNS pre-flight error, reverse DNS debug errors and per-report errors, and unusable https:// / tls:// ports are rejected before any query. - Docs recommend token/username settings for [loki] and [splunk_hec]; AGENTS.md gains rules for keeping credentials out of logs. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Sean Whalen <seanthegeek@users.noreply.github.com>
80 lines
2.7 KiB
Python
80 lines
2.7 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
|
|
import httpx
|
|
|
|
from parsedmarc import logger
|
|
from parsedmarc.constants import USER_AGENT
|
|
|
|
|
|
class WebhookClient(object):
|
|
"""A client for webhooks"""
|
|
|
|
def __init__(
|
|
self,
|
|
aggregate_url: str,
|
|
failure_url: str,
|
|
smtp_tls_url: str,
|
|
timeout: int | None = 60,
|
|
):
|
|
"""
|
|
Initializes the WebhookClient
|
|
Args:
|
|
aggregate_url (str): The aggregate report webhook url
|
|
failure_url (str): The failure report webhook url
|
|
smtp_tls_url (str): The smtp_tls report webhook url
|
|
timeout (int): The timeout to use when calling the webhooks
|
|
"""
|
|
self.aggregate_url = aggregate_url
|
|
self.failure_url = failure_url
|
|
self.smtp_tls_url = smtp_tls_url
|
|
self.timeout = timeout
|
|
self.session = httpx.Client(
|
|
headers={
|
|
"User-Agent": USER_AGENT,
|
|
"Content-Type": "application/json",
|
|
},
|
|
follow_redirects=True,
|
|
)
|
|
|
|
def save_failure_report_to_webhook(self, report: str):
|
|
self._send_to_webhook(self.failure_url, report)
|
|
|
|
def save_smtp_tls_report_to_webhook(self, report: str):
|
|
self._send_to_webhook(self.smtp_tls_url, report)
|
|
|
|
def save_aggregate_report_to_webhook(self, report: str):
|
|
self._send_to_webhook(self.aggregate_url, report)
|
|
|
|
def _send_to_webhook(self, webhook_url: str, payload: bytes | str | dict[str, Any]):
|
|
# All HTTP / network errors are swallowed and logged: a failing
|
|
# webhook should never abort the surrounding parse-and-output
|
|
# batch. The outer save_* methods previously wrapped this in a
|
|
# redundant try/except — removed because _send_to_webhook
|
|
# already catches every Exception itself.
|
|
try:
|
|
if isinstance(payload, dict):
|
|
# requests form-encoded dict payloads via data=; httpx does
|
|
# the same only via data=
|
|
self.session.post(webhook_url, data=payload, timeout=self.timeout)
|
|
else:
|
|
self.session.post(webhook_url, content=payload, timeout=self.timeout)
|
|
except httpx.InvalidURL:
|
|
# httpx quotes part of the URL, which can be part of a password
|
|
logger.error(
|
|
"Webhook Error: invalid webhook URL (details withheld because "
|
|
"they may include credentials)"
|
|
)
|
|
except Exception as error_:
|
|
logger.error(f"Webhook Error: {error_.__str__()}")
|
|
|
|
def close(self):
|
|
"""Close the underlying HTTP session."""
|
|
self.session.close()
|
|
|
|
# Backward-compatible alias
|
|
save_forensic_report_to_webhook = save_failure_report_to_webhook
|