From 625089a12c9494967d9a784cd98638b703dd0a7d Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Sat, 29 Sep 2018 13:25:27 -0400 Subject: [PATCH] 4.1.1 --- CHANGELOG.md | 8 ++++++ docs/index.rst | 20 ++++++++++++-- parsedmarc/__init__.py | 40 ++++++++++++++++++++++++---- parsedmarc/cli.py | 5 ++-- setup.py | 2 +- splunk/dmarc_aggregate_dashboard.xml | 2 +- 6 files changed, 66 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3667377..33bd806 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ + +4.1.1 +----- + +- Add splunk instructions +- Change default logging level to `ERROR` +- Reconnect reset IMAP connections when watching a folder + 4.1.0 ----- diff --git a/docs/index.rst b/docs/index.rst index 2de74fa..60f0f67 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -276,7 +276,7 @@ Or, install the latest development release directly from GitHub: .. note:: - On Windows, ``pip3`` is ``pip``, even with Python 3. So on Windows, simply + On Windows, ``pip3`` is ``pip``, even with Python 3. So on Windows, substitute ``pip`` as an administrator in place of ``sudo pip3``, in the above commands. @@ -365,11 +365,16 @@ Testing multiple report analyzers If you would like to test parsedmarc and another report processing solution at the same time, you can have up to two mailto URIs each in the rua and ruf -tags tgs in your DMARC record, separated by commas. +tags in your DMARC record, separated by commas. Elasticsearch and Kibana ------------------------ +.. note:: + + Splunk is also supported starting with ``parsedmarc`` 4.1.1 + + To set up visual dashboards of DMARC data, install Elasticsearch and Kibana. .. note:: @@ -595,6 +600,17 @@ select ``dmarc_aggregate`` for the other saved objects, as shown below. :align: center :target: _static/screenshots/index-pattern-conflicts.png + +Splunk +------ + +Starting in version 4.1.1 ``parsedmarc`` supports sending aggregate and/or +forensic DMARC data to a Splunk HTTP Events collector (HEC). Simply use the +following command line options, along with ``--save-aggregate`` or +``save-forensic``: + + + Running parsedmarc as a systemd service --------------------------------------- diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index 3baf515..9f0fdc8 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -44,10 +44,10 @@ import imapclient.exceptions import dateparser import mailparser -__version__ = "4.1.0" +__version__ = "4.1." logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) +logger.setLevel(logging.ERROR) feedback_report_regex = re.compile(r"^([\w\-]+): (.+)$", re.MULTILINE) @@ -1528,7 +1528,7 @@ def watch_inbox(host, username, password, callback, reports_folder="INBOX", except imapclient.exceptions.IMAPClientError as error: error = error.__str__().replace("b'", "").replace("'", "") # Workaround for random Exchange/Office365 IMAP errors - if "Server Unavailable. 15" in error: + if "Server Unavailable. 15" in error or "BAD" in error: logger.debug("IMAP error: {0}".format(error)) logger.debug("Reconnecting watcher") server = imapclient.IMAPClient(host) @@ -1553,7 +1553,23 @@ def watch_inbox(host, username, password, callback, reports_folder="INBOX", except ConnectionRefusedError: raise IMAPError("Connection refused") except ConnectionResetError: - raise IMAPError("Connection reset") + logger.debug("IMAP error: Connection reset") + logger.debug("Reconnecting watcher") + server = imapclient.IMAPClient(host) + server.login(username, password) + server.select_folder(rf) + idle_start_time = time.monotonic() + ms = "MOVE" in get_imap_capabilities(server) + res = get_dmarc_reports_from_inbox(connection=server, + move_supported=ms, + reports_folder=rf, + archive_folder=af, + delete=delete, + test=test, + nameservers=ns, + dns_timeout=dt) + callback(res) + server.idle() except ConnectionAbortedError: raise IMAPError("Connection aborted") except TimeoutError: @@ -1614,7 +1630,21 @@ def watch_inbox(host, username, password, callback, reports_folder="INBOX", except ConnectionRefusedError: raise IMAPError("Connection refused") except ConnectionResetError: - raise IMAPError("Connection reset") + logger.debug("IMAP error: Connection reset") + logger.debug("Reconnecting watcher") + server = imapclient.IMAPClient(host) + server.login(username, password) + server.select_folder(rf) + idle_start_time = time.monotonic() + ms = "MOVE" in get_imap_capabilities(server) + res = get_dmarc_reports_from_inbox(connection=server, + move_supported=ms, + reports_folder=rf, + archive_folder=af, + delete=delete, + test=test, + nameservers=ns, + dns_timeout=dt) except ConnectionAbortedError: raise IMAPError("Connection aborted") except TimeoutError: diff --git a/parsedmarc/cli.py b/parsedmarc/cli.py index 236406b..ae5c32e 100644 --- a/parsedmarc/cli.py +++ b/parsedmarc/cli.py @@ -108,10 +108,11 @@ def _main(): "Collector (HEC)") arg_parser.add_argument("--hec-token", help="The authorization token for " "a Splunk " - "HTTP event collector (HEC)") + "HTTP Event Collector (HEC)") arg_parser.add_argument("--hec-index", help="The index to use when " "sending events to the " - "Splunk HTTP Events") + "Splunk HTTP Event Collector " + "(HEC)") arg_parser.add_argument("--hec-skip-certificate-verification", action="store_true", default=False, diff --git a/setup.py b/setup.py index 923e8af..12da2da 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ from setuptools import setup from codecs import open from os import path -__version__ = "4.1.0" +__version__ = "4.1.1" description = "A Python package and CLI for parsing aggregate and " \ "forensic DMARC reports" diff --git a/splunk/dmarc_aggregate_dashboard.xml b/splunk/dmarc_aggregate_dashboard.xml index 96af964..d18878f 100644 --- a/splunk/dmarc_aggregate_dashboard.xml +++ b/splunk/dmarc_aggregate_dashboard.xml @@ -107,7 +107,7 @@ Reporting organizations - index="email" sourcetype="dmarc:aggregate" | chart sum(message_count) by org_name | sort -sum(message_count) + index="email" sourcetype="dmarc:aggregate" sourcetype="dmarc:aggregate" spf_aligned=$spf_aligned$ dkim_aligned=$dkim_aligned$ passed_dmarc=$passed_dmarc$ org_name=$org_name$ source_reverse_dns=$source_reverse_dns$ header_from=$header_from$ envelope_from=$envelope_from$ disposition=$disposition$ source_ip_address=$source_ip_address$ source_country=$source_country$ | chart sum(message_count) by org_name | sort -sum(message_count) $time_range.earliest$ $time_range.latest$