mirror of
https://github.com/domainaware/parsedmarc.git
synced 2026-10-24 19:48:56 +00:00
Warn on unmatched CLI paths; send Kafka failure/SMTP TLS reports per-report (#890)
Fixes the two remaining bugs and the nits uncovered during the review session: - The CLI logs a warning for each file_path argument that matches no files, instead of silently succeeding with empty results — a typo'd path in a cron job previously 'worked' forever while processing nothing (_expand_file_path_args dropped it as a zero-match glob). Deliberately a warning, not an error: a glob legitimately matching nothing must not break existing workflows, and mailbox-only runs stay silent. - save_failure_reports_to_kafka and save_smtp_tls_reports_to_kafka now send one message per report, mirroring the aggregate saver's per-slice sends. Every released version documented per-record sends, but the code sent the whole list as one message, which a large batch — failure reports carry message samples — could push past Kafka's default 1MB message limit. Consumer-visible: these topics now carry individual report objects, not one JSON array per batch. - The CLI module docstring and argparse description now mention SMTP TLS reports; usage.md's CLI-help mirror regenerated. - The elastic/opensearch to_header display-name tests now assert the joined 'RT <rcpt@example.com>' string reaches the saved document (autospec save), plus the exactly-once call, instead of only that save ran. Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
2d76de9ca6
commit
07b00a931b
8 files changed
+180
-43
No files matched your search
+35
-12
@@ -186,22 +186,33 @@ class TestSaveFailureReportsToKafka(unittest.TestCase):
|
||||
with patch("parsedmarc.kafkaclient.KafkaProducer"):
|
||||
return KafkaClient(kafka_hosts=["b:9092"])
|
||||
|
||||
def test_sends_full_list_in_one_message(self):
|
||||
"""Failure reports are sent as one Kafka message carrying the
|
||||
whole list — unlike aggregate records, which are sent as
|
||||
individual slices to stay under Kafka's default 1MB message
|
||||
cap."""
|
||||
def test_sends_one_message_per_report(self):
|
||||
"""Failure reports are sent as one Kafka message per report,
|
||||
mirroring the aggregate saver's per-slice sends, so a large batch
|
||||
is far less likely to exceed Kafka's default 1MB message limit
|
||||
— failure
|
||||
reports carry message samples, so a whole-list send is
|
||||
particularly likely to blow past that cap."""
|
||||
client = self._client()
|
||||
reports = [{"id": "f1"}, {"id": "f2"}]
|
||||
client.save_failure_reports_to_kafka(reports, "dmarc-failure")
|
||||
_producer(client).send.assert_called_once_with("dmarc-failure", reports)
|
||||
producer = _producer(client)
|
||||
self.assertEqual(producer.send.call_count, 2)
|
||||
sent = [call.args for call in producer.send.call_args_list]
|
||||
self.assertEqual(
|
||||
sent, [("dmarc-failure", reports[0]), ("dmarc-failure", reports[1])]
|
||||
)
|
||||
self.assertEqual(producer.flush.call_count, 2)
|
||||
|
||||
def test_dict_input_normalized_to_list(self):
|
||||
"""A single-report dict is wrapped to a one-element list
|
||||
internally, then sent as that one unwrapped report (not the
|
||||
list itself)."""
|
||||
client = self._client()
|
||||
client.save_failure_reports_to_kafka({"id": "single"}, "topic")
|
||||
# The send payload is wrapped to a single-element list.
|
||||
self.assertEqual(_producer(client).send.call_count, 1)
|
||||
args = _producer(client).send.call_args.args
|
||||
self.assertEqual(args[1], [{"id": "single"}])
|
||||
self.assertEqual(args[1], {"id": "single"})
|
||||
|
||||
def test_empty_list_is_a_noop(self):
|
||||
client = self._client()
|
||||
@@ -232,17 +243,29 @@ class TestSaveSmtpTlsReportsToKafka(unittest.TestCase):
|
||||
with patch("parsedmarc.kafkaclient.KafkaProducer"):
|
||||
return KafkaClient(kafka_hosts=["b:9092"])
|
||||
|
||||
def test_sends_full_list_in_one_message(self):
|
||||
def test_sends_one_message_per_report(self):
|
||||
"""SMTP TLS reports are sent as one Kafka message per report,
|
||||
mirroring the aggregate saver's per-slice sends, so a large batch
|
||||
is far less likely to exceed Kafka's default 1MB message
|
||||
limit."""
|
||||
client = self._client()
|
||||
reports = [{"organization_name": "x"}]
|
||||
reports = [{"organization_name": "x"}, {"organization_name": "y"}]
|
||||
client.save_smtp_tls_reports_to_kafka(reports, "smtp-tls")
|
||||
_producer(client).send.assert_called_once_with("smtp-tls", reports)
|
||||
producer = _producer(client)
|
||||
self.assertEqual(producer.send.call_count, 2)
|
||||
sent = [call.args for call in producer.send.call_args_list]
|
||||
self.assertEqual(sent, [("smtp-tls", reports[0]), ("smtp-tls", reports[1])])
|
||||
self.assertEqual(producer.flush.call_count, 2)
|
||||
|
||||
def test_dict_input_normalized_to_list(self):
|
||||
"""A single-report dict is wrapped to a one-element list
|
||||
internally, then sent as that one unwrapped report (not the
|
||||
list itself)."""
|
||||
client = self._client()
|
||||
client.save_smtp_tls_reports_to_kafka({"organization_name": "x"}, "topic")
|
||||
self.assertEqual(_producer(client).send.call_count, 1)
|
||||
args = _producer(client).send.call_args.args
|
||||
self.assertEqual(args[1], [{"organization_name": "x"}])
|
||||
self.assertEqual(args[1], {"organization_name": "x"})
|
||||
|
||||
def test_empty_list_is_a_noop(self):
|
||||
client = self._client()
|
||||
|
||||
Reference in new issue
Block a user