Post-review follow-ups for Graph send (#825/#826) and requests-to-httpx migration (#827)

Follow-ups from the review of PR #825 (whose implementation had already
landed on master via #826's stacked merge):

- Honor the documented [smtp] attachment and [smtp] message options.
  Both were parsed into opts but never passed to either summary-email
  transport (also broken in released 10.2.2), so a configured custom
  attachment filename or message body was silently ignored. Both the
  SMTP and Microsoft Graph transports now receive them, and the missing
  smtp_attachment Namespace default is added (also covers SIGHUP
  reload, which rebuilds opts from the CLI Namespace).
- Don't mislabel non-Graph mailbox errors as Microsoft Graph failures:
  the shared mailbox-fetch and watch handlers now log a generic
  "Mailbox Error" with traceback when the connection isn't Graph.
- Declare microsoft-kiota-abstractions as a direct dependency (imported
  directly in cli.py for Graph error handling; previously transitive).

Migrate all runtime HTTP from requests to httpx (webhook client, Splunk
HEC client, and the PSL-overrides / IP-database / reverse-DNS-map /
IPinfo-API fetches in utils.py):

- follow_redirects=True everywhere to preserve requests' default
  redirect-following; httpx does not follow redirects by default.
- The PSL-overrides and reverse-DNS-map fetches gain a 60s timeout
  (previously none), matching the IP-database fetch.
- response.ok -> response.is_success; requests.RequestException ->
  httpx.HTTPError; raw string bodies use content= (httpx's data= is
  form-encoding only); Splunk HEC verification moves to client
  construction (httpx has no per-request verify).
- requests drops out of [project] dependencies and moves to the [build]
  extra for the out-of-wheel maintainer script collect_domain_info.py,
  which deliberately stays on requests/urllib3 for its permissive-TLS
  adapter.
- Remove the requests-era module-level
  urllib3.disable_warnings(InsecureRequestWarning) in splunk.py; httpx
  doesn't route through urllib3, so its only remaining effect was
  globally silencing insecure-TLS warnings from other urllib3-based
  components as an import side effect. Nothing imports urllib3 directly
  anymore, so it also leaves [project] dependencies.

Tests: config-to-transport wiring for attachment/message on both
transports (including defaults), non-Graph errors keep the generic log
line, webhook/Splunk payload assertions moved to content=, and Splunk
verify asserted at httpx.Client construction. 736 passed; ruff and
pyright clean.

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Sean Whalen
2026-07-14 16:11:47 -04:00
committed by GitHub
co-authored by Claude Fable 5
parent 31c928d6fc
commit df9bf82e04
11 changed files with 335 additions and 93 deletions
+36 -11
View File
@@ -3,7 +3,7 @@
import json
import time
import unittest
from unittest.mock import MagicMock
from unittest.mock import MagicMock, patch
from parsedmarc.splunk import HECClient, SplunkError
from tests.tzutil import force_tz
@@ -211,7 +211,7 @@ class TestSaveAggregateReportsToSplunk(unittest.TestCase):
client.session = MagicMock()
client.session.post.return_value = _ok_response()
client.save_aggregate_reports_to_splunk(report)
body = client.session.post.call_args.kwargs["data"]
body = client.session.post.call_args.kwargs["content"]
events = [json.loads(line) for line in body.strip().split("\n")]
self.assertEqual(len(events), 2)
for event in events:
@@ -225,7 +225,7 @@ class TestSaveAggregateReportsToSplunk(unittest.TestCase):
client.session = MagicMock()
client.session.post.return_value = _ok_response()
client.save_aggregate_reports_to_splunk(_aggregate_report())
body = client.session.post.call_args.kwargs["data"]
body = client.session.post.call_args.kwargs["content"]
event = json.loads(body.strip())["event"]
self.assertEqual(event["source_ip_address"], "192.0.2.1")
self.assertEqual(event["header_from"], "example.com")
@@ -238,7 +238,7 @@ class TestSaveAggregateReportsToSplunk(unittest.TestCase):
client.session = MagicMock()
client.session.post.return_value = _ok_response()
client.save_aggregate_reports_to_splunk(_aggregate_report())
event = json.loads(client.session.post.call_args.kwargs["data"].strip())[
event = json.loads(client.session.post.call_args.kwargs["content"].strip())[
"event"
]
self.assertEqual(
@@ -258,7 +258,10 @@ class TestSaveAggregateReportsToSplunk(unittest.TestCase):
client.save_aggregate_reports_to_splunk([])
client.session.post.assert_not_called()
def test_post_uses_session_verify_and_timeout(self):
def test_post_uses_session_timeout(self):
"""httpx has no per-request verify= kwarg — verification is set
at client construction (see test_client_constructed_with_verify_
false below), so only the per-request timeout is asserted here."""
client = HECClient(
url="https://h:8088",
access_token="t",
@@ -270,9 +273,25 @@ class TestSaveAggregateReportsToSplunk(unittest.TestCase):
client.session.post.return_value = _ok_response()
client.save_aggregate_reports_to_splunk(_aggregate_report())
kwargs = client.session.post.call_args.kwargs
self.assertEqual(kwargs["verify"], False)
self.assertNotIn("verify", kwargs)
self.assertEqual(kwargs["timeout"], 15)
def test_client_constructed_with_verify_false(self):
"""verify=False must disable TLS verification on the underlying
httpx.Client at construction time, since httpx.Client does not
accept a per-request verify= kwarg like requests.Session did.
Mocked at the httpx SDK boundary (httpx.Client itself)."""
with patch("parsedmarc.splunk.httpx.Client") as mock_client_cls:
HECClient(
url="https://h:8088",
access_token="t",
index="dmarc",
verify=False,
timeout=15,
)
_, kwargs = mock_client_cls.call_args
self.assertEqual(kwargs["verify"], False)
def test_non_zero_response_code_raises_splunk_error(self):
"""HEC returns code=0 on success and non-zero codes for
token/index/format errors. The error text from HEC carries
@@ -306,7 +325,9 @@ class TestSaveAggregateReportsToSplunk(unittest.TestCase):
client.session = MagicMock()
client.session.post.return_value = _ok_response()
client.save_aggregate_reports_to_splunk(_aggregate_report())
event_wrapper = json.loads(client.session.post.call_args.kwargs["data"].strip())
event_wrapper = json.loads(
client.session.post.call_args.kwargs["content"].strip()
)
self.assertEqual(event_wrapper["time"], 1704067200)
@@ -318,7 +339,9 @@ class TestSaveFailureReportsToSplunk(unittest.TestCase):
client.save_failure_reports_to_splunk([_failure_report(), _failure_report()])
events = [
json.loads(line)
for line in client.session.post.call_args.kwargs["data"].strip().split("\n")
for line in client.session.post.call_args.kwargs["content"]
.strip()
.split("\n")
]
self.assertEqual(len(events), 2)
for event in events:
@@ -329,7 +352,7 @@ class TestSaveFailureReportsToSplunk(unittest.TestCase):
client.session = MagicMock()
client.session.post.return_value = _ok_response()
client.save_failure_reports_to_splunk(_failure_report())
event = json.loads(client.session.post.call_args.kwargs["data"].strip())[
event = json.loads(client.session.post.call_args.kwargs["content"].strip())[
"event"
]
self.assertEqual(event["reported_domain"], "example.com")
@@ -354,7 +377,7 @@ class TestSaveFailureReportsToSplunk(unittest.TestCase):
client.session = MagicMock()
client.session.post.return_value = _ok_response()
client.save_failure_reports_to_splunk(_failure_report())
event = json.loads(client.session.post.call_args.kwargs["data"].strip())
event = json.loads(client.session.post.call_args.kwargs["content"].strip())
# Fixture arrival_date_utc is 2024-01-01 00:00:00 UTC.
self.assertEqual(event["time"], 1704067200)
@@ -397,7 +420,9 @@ class TestSaveSmtpTlsReportsToSplunk(unittest.TestCase):
client.save_smtp_tls_reports_to_splunk([_smtp_tls_report()])
events = [
json.loads(line)
for line in client.session.post.call_args.kwargs["data"].strip().split("\n")
for line in client.session.post.call_args.kwargs["content"]
.strip()
.split("\n")
]
self.assertEqual(len(events), 1)
self.assertEqual(events[0]["sourcetype"], "smtp:tls")