From 20433cd2b6aaccf85bea0226050663c112c00b30 Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Tue, 9 Oct 2018 10:34:42 -0400 Subject: [PATCH 01/14] Logging and packaging fixes --- CHANGELOG.md | 4 ++++ parsedmarc/__init__.py | 8 +++----- parsedmarc/cli.py | 12 ++++++++---- parsedmarc/splunk.py | 6 ++++++ requirements.txt | 1 + setup.py | 4 ++-- 6 files changed, 24 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1176ee2..46db8a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ------ - Save each aggregate report record as a separate Splunk event +- Fix IAMP delete action +- Suppress Splunk SSL validation warnings +- Change default logging level to `INFO` + 4.1.9 ----- diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index cc075e3..aeaa626 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -47,7 +47,7 @@ import mailparser __version__ = "4.2.0" logger = logging.getLogger(__name__) -logger.setLevel(logging.ERROR) +logger.setLevel(logging.INFO) feedback_report_regex = re.compile(r"^([\w\-]+): (.+)$", re.MULTILINE) xml_header_regex = re.compile(r"^<\?xml .*$", re.MULTILINE) @@ -1249,10 +1249,8 @@ def get_dmarc_reports_from_inbox(host=None, if type(msg_uids) == str: msg_uids = [msg_uids] - for chunk in chunks(msg_uids, 100): - server.add_flags(chunk, [imapclient.DELETED]) - - server.expunge() + server.delete_messages(msg_uids, silent=True) + server.expunge(msg_uids) def move_messages(msg_uids, folder): if type(msg_uids) == str: diff --git a/parsedmarc/cli.py b/parsedmarc/cli.py index fcf3d62..8405689 100644 --- a/parsedmarc/cli.py +++ b/parsedmarc/cli.py @@ -83,7 +83,7 @@ def _main(): arg_parser.add_argument("--imap-port", default=None, help="IMAP port") arg_parser.add_argument("--imap-no-ssl", action="store_true", default=False, - help="Do not use SSL when connecting to IMAP") + help="Do not use SSL/TLS when connecting to IMAP") arg_parser.add_argument("-r", "--reports-folder", default="INBOX", help="The IMAP folder containing the reports\n" "Default: INBOX") @@ -157,7 +157,7 @@ def _main(): help="Do not move or delete IMAP messages", action="store_true", default=False) arg_parser.add_argument("-s", "--silent", action="store_true", - help="Only print errors") + help="Only print errors and warnings") arg_parser.add_argument("--debug", action="store_true", help="Print debugging information") arg_parser.add_argument("-v", "--version", action="version", @@ -168,8 +168,12 @@ def _main(): args = arg_parser.parse_args() - logging.basicConfig(level=logging.ERROR) - logger.setLevel(logging.ERROR) + logging.basicConfig(level=logging.INFO) + logger.setLevel(logging.INFO) + + if args.silent: + logging.basicConfig(level=logging.WARNING) + logger.setLevel(logging.WARNING) if args.debug: logging.basicConfig(level=logging.DEBUG) logger.setLevel(logging.DEBUG) diff --git a/parsedmarc/splunk.py b/parsedmarc/splunk.py index 8448d15..714f2d4 100644 --- a/parsedmarc/splunk.py +++ b/parsedmarc/splunk.py @@ -1,11 +1,14 @@ from urllib.parse import urlparse import socket import json +import urllib3 import requests from parsedmarc import __version__, human_timestamp_to_timestamp +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + class SplunkError(RuntimeError): """Raised when a Splunk API error occurs""" @@ -136,3 +139,6 @@ class HECClient(object): raise SplunkError(e.__str__()) if response["code"] != 0: raise SplunkError(response["text"]) + + + diff --git a/requirements.txt b/requirements.txt index b10e457..aff41e8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ dnspython +urllib3 requests publicsuffix xmltodict diff --git a/setup.py b/setup.py index fe80350..b94a56c 100644 --- a/setup.py +++ b/setup.py @@ -92,8 +92,8 @@ setup( # requirements files see: # https://packaging.python.org/en/latest/requirements.html install_requires=['dnspython', 'publicsuffix', 'xmltodict', 'geoip2', - 'dnspython', 'imapclient', 'mail-parser', 'dateparser', - 'elasticsearch>=6.3.0,<7.0.0', + 'urllib3', 'requests', 'imapclient', 'mail-parser', + 'dateparser', 'elasticsearch>=6.3.0,<7.0.0', 'elasticsearch-dsl>=6.2.1,<7.0.0' ], From e2ec3bc2dad1e59f20bb682b94e85900f44446b7 Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Tue, 9 Oct 2018 10:55:40 -0400 Subject: [PATCH 02/14] Change default logging level to `WARNING` --- CHANGELOG.md | 2 +- parsedmarc/__init__.py | 2 +- parsedmarc/cli.py | 7 ++----- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 46db8a1..1a716a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ - Save each aggregate report record as a separate Splunk event - Fix IAMP delete action - Suppress Splunk SSL validation warnings -- Change default logging level to `INFO` +- Change default logging level to `WARNING` 4.1.9 diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index aeaa626..7e65715 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -47,7 +47,7 @@ import mailparser __version__ = "4.2.0" logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) +logger.setLevel(logging.WARNING) feedback_report_regex = re.compile(r"^([\w\-]+): (.+)$", re.MULTILINE) xml_header_regex = re.compile(r"^<\?xml .*$", re.MULTILINE) diff --git a/parsedmarc/cli.py b/parsedmarc/cli.py index 8405689..7be74b5 100644 --- a/parsedmarc/cli.py +++ b/parsedmarc/cli.py @@ -168,12 +168,9 @@ def _main(): args = arg_parser.parse_args() - logging.basicConfig(level=logging.INFO) - logger.setLevel(logging.INFO) + logging.basicConfig(level=logging.WARNING) + logger.setLevel(logging.WARNING) - if args.silent: - logging.basicConfig(level=logging.WARNING) - logger.setLevel(logging.WARNING) if args.debug: logging.basicConfig(level=logging.DEBUG) logger.setLevel(logging.DEBUG) From df1c2bac5fa39467b6bee9a6b1dd68d00f6737ae Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Tue, 9 Oct 2018 11:12:00 -0400 Subject: [PATCH 03/14] Fix Splunk forensic dashboard sorting --- splunk/dmarc_forensic_dashboard.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/splunk/dmarc_forensic_dashboard.xml b/splunk/dmarc_forensic_dashboard.xml index a4d611b..f501b76 100644 --- a/splunk/dmarc_forensic_dashboard.xml +++ b/splunk/dmarc_forensic_dashboard.xml @@ -38,7 +38,7 @@ Forensic samples - index="email" sourcetype="dmarc:forensic" parsed_sample.headers.From=$header_from$ parsed_sample.headers.To=$header_to$ parsed_sample.headers.Subject=$header_subject$ source.ip_address=$source_ip_address$ source.reverse_dns=$source_reverse_dns$ source.country=$source_country$ | fillnull value="none" | stats count by _time,parsed_sample.headers.From,parsed_sample.headers.To,parsed_sample.headers.Reply-To,parsed_sample.headers.Subject | sort -_time + index="email" sourcetype="dmarc:forensic" parsed_sample.headers.From=$header_from$ parsed_sample.headers.To=$header_to$ parsed_sample.headers.Subject=$header_subject$ source.ip_address=$source_ip_address$ source.reverse_dns=$source_reverse_dns$ source.country=$source_country$ | fillnull value="none" | stats count by arrival_date_utc,parsed_sample.headers.From,parsed_sample.headers.To,parsed_sample.headers.Reply-To,parsed_sample.headers.Subject | sort -arrival_date_utc $time_range.earliest$ $time_range.latest$ @@ -95,4 +95,4 @@
- + \ No newline at end of file From fd2d5093a900af512d6f4e29824df023c22565f3 Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Tue, 9 Oct 2018 11:48:31 -0400 Subject: [PATCH 04/14] More logging when --debug is used --- parsedmarc/__init__.py | 1 + parsedmarc/elastic.py | 7 +++++++ parsedmarc/splunk.py | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index 7e65715..04cd65c 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -1587,6 +1587,7 @@ def email_results(results, host, mail_from, mail_to, port=0, message: Override the default plain text body ssl_context: SSL context options """ + logging.debug("Emailing report to: {0}".format(",".join(mail_to))) date_string = datetime.now().strftime("%Y-%m-%d") if attachment_filename: if not attachment_filename.lower().endswith(".zip"): diff --git a/parsedmarc/elastic.py b/parsedmarc/elastic.py index c426755..1a781ed 100644 --- a/parsedmarc/elastic.py +++ b/parsedmarc/elastic.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- +import logging from collections import OrderedDict import parsedmarc @@ -7,6 +8,9 @@ from elasticsearch_dsl.search import Q from elasticsearch_dsl import connections, Object, Document, Index, Nested, \ InnerDoc, Integer, Text, Boolean, DateRange, Ip, Date +logger = logging.getLogger(__name__) +logger.setLevel(logging.WARNING) + class _PolicyOverride(InnerDoc): type = Text() @@ -179,6 +183,7 @@ def create_indexes(names=None, settings=None): settings (dict): Index settings """ + logger.debug("Creating elasticsearch indexes") if names is None: names = ["dmarc_aggregate", "dmarc_forensic"] for name in names: @@ -201,6 +206,7 @@ def save_aggregate_report_to_elasticsearch(aggregate_report, Raises: AlreadySaved """ + logger.debug("Saving aggregate report to Elasticsearch") aggregate_report = aggregate_report.copy() metadata = aggregate_report["report_metadata"] org_name = metadata["org_name"] @@ -299,6 +305,7 @@ def save_forensic_report_to_elasticsearch(forensic_report, AlreadySaved """ + logger.debug("Saving forensic report to Elasticsearch") forensic_report = forensic_report.copy() sample_date = forensic_report["parsed_sample"]["date"] sample_date = parsedmarc.human_timestamp_to_datetime(sample_date) diff --git a/parsedmarc/splunk.py b/parsedmarc/splunk.py index 714f2d4..5c1bd1a 100644 --- a/parsedmarc/splunk.py +++ b/parsedmarc/splunk.py @@ -1,3 +1,4 @@ +import logging from urllib.parse import urlparse import socket import json @@ -9,6 +10,9 @@ from parsedmarc import __version__, human_timestamp_to_timestamp urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) +logger = logging.getLogger(__name__) +logger.setLevel(logging.WARNING) + class SplunkError(RuntimeError): """Raised when a Splunk API error occurs""" From 8ea0e62bdd0f1b95e6ac0a166da884f8a500d1de Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Tue, 9 Oct 2018 20:21:17 -0400 Subject: [PATCH 05/14] Add more debug logging details --- parsedmarc/__init__.py | 6 +++--- parsedmarc/splunk.py | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index 04cd65c..65a06c2 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -1246,6 +1246,7 @@ def get_dmarc_reports_from_inbox(host=None, move_supported = "MOVE" in server_capabilities def delete_messages(msg_uids): + logger.debug("Deleting messages") if type(msg_uids) == str: msg_uids = [msg_uids] @@ -1284,9 +1285,8 @@ def get_dmarc_reports_from_inbox(host=None, server.create_folder(invalid_reports_folder) server.select_folder(reports_folder) messages = server.search() - logger.debug("Found {0} messages in IMAP folder".format(len(messages), - reports_folder - )) + logger.debug("Found {0} messages in IMAP folder " + "{1}".format(len(messages), reports_folder)) for i in range(len(messages)): number_of_messages = len(messages) message_uid = messages[i] diff --git a/parsedmarc/splunk.py b/parsedmarc/splunk.py index 5c1bd1a..1861897 100644 --- a/parsedmarc/splunk.py +++ b/parsedmarc/splunk.py @@ -61,6 +61,7 @@ class HECClient(object): to save in Splunk """ + logger.debug("Saving aggregate reports to Splunk") if type(aggregate_reports) == dict: aggregate_reports = [aggregate_reports] @@ -122,6 +123,7 @@ class HECClient(object): to save in Splunk """ + logger.debug("Saving forensic reports to Splunk") if type(forensic_reports) == dict: forensic_reports = [forensic_reports] From 21af33687c80dacab3df31d43e0c0ccab81b5ec4 Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Tue, 9 Oct 2018 20:35:45 -0400 Subject: [PATCH 06/14] Fix module logging --- parsedmarc/cli.py | 5 ++++- parsedmarc/elastic.py | 1 - parsedmarc/splunk.py | 1 - 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/parsedmarc/cli.py b/parsedmarc/cli.py index 7be74b5..f7b0086 100644 --- a/parsedmarc/cli.py +++ b/parsedmarc/cli.py @@ -12,10 +12,13 @@ import json from elasticsearch.exceptions import ElasticsearchException -from parsedmarc import logger, IMAPError, get_dmarc_reports_from_inbox, \ +from parsedmarc import IMAPError, get_dmarc_reports_from_inbox, \ parse_report_file, elastic, splunk, save_output, watch_inbox, \ email_results, SMTPError, ParserError, __version__ +logger = logging.getLogger(__name__) +logger.setLevel(logging.WARNING) + def _main(): """Called when the module is executed""" diff --git a/parsedmarc/elastic.py b/parsedmarc/elastic.py index 1a781ed..462cd22 100644 --- a/parsedmarc/elastic.py +++ b/parsedmarc/elastic.py @@ -9,7 +9,6 @@ from elasticsearch_dsl import connections, Object, Document, Index, Nested, \ InnerDoc, Integer, Text, Boolean, DateRange, Ip, Date logger = logging.getLogger(__name__) -logger.setLevel(logging.WARNING) class _PolicyOverride(InnerDoc): diff --git a/parsedmarc/splunk.py b/parsedmarc/splunk.py index 1861897..2c00f74 100644 --- a/parsedmarc/splunk.py +++ b/parsedmarc/splunk.py @@ -11,7 +11,6 @@ from parsedmarc import __version__, human_timestamp_to_timestamp urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) logger = logging.getLogger(__name__) -logger.setLevel(logging.WARNING) class SplunkError(RuntimeError): From 16dcb2edc70b62251d2baef6090cf60b5d3dc7a4 Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Tue, 9 Oct 2018 20:48:31 -0400 Subject: [PATCH 07/14] Fix logging --- parsedmarc/cli.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/parsedmarc/cli.py b/parsedmarc/cli.py index 7be74b5..ee54d89 100644 --- a/parsedmarc/cli.py +++ b/parsedmarc/cli.py @@ -12,11 +12,15 @@ import json from elasticsearch.exceptions import ElasticsearchException -from parsedmarc import logger, IMAPError, get_dmarc_reports_from_inbox, \ +from parsedmarc import IMAPError, get_dmarc_reports_from_inbox, \ parse_report_file, elastic, splunk, save_output, watch_inbox, \ email_results, SMTPError, ParserError, __version__ +logger = logging.getLogger(__name__) +logger.setLevel(logging.WARNING) + + def _main(): """Called when the module is executed""" def process_reports(reports_): From 0d1a4786e1e4aa75364f1e2e1308d829e76feb4c Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Tue, 9 Oct 2018 20:49:12 -0400 Subject: [PATCH 08/14] Fix logging --- parsedmarc/cli.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/parsedmarc/cli.py b/parsedmarc/cli.py index c9f2b72..f7b0086 100644 --- a/parsedmarc/cli.py +++ b/parsedmarc/cli.py @@ -20,10 +20,6 @@ logger = logging.getLogger(__name__) logger.setLevel(logging.WARNING) -logger = logging.getLogger(__name__) -logger.setLevel(logging.WARNING) - - def _main(): """Called when the module is executed""" def process_reports(reports_): From c944264760d20712f1a852d47dabadea1be8a081 Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Tue, 9 Oct 2018 23:00:28 -0400 Subject: [PATCH 09/14] Fix logging --- parsedmarc/__init__.py | 3 +-- parsedmarc/cli.py | 3 +-- parsedmarc/elastic.py | 2 +- parsedmarc/splunk.py | 2 +- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index 65a06c2..4028667 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -46,8 +46,7 @@ import mailparser __version__ = "4.2.0" -logger = logging.getLogger(__name__) -logger.setLevel(logging.WARNING) +logger = logging.getLogger("parsedmarc") feedback_report_regex = re.compile(r"^([\w\-]+): (.+)$", re.MULTILINE) xml_header_regex = re.compile(r"^<\?xml .*$", re.MULTILINE) diff --git a/parsedmarc/cli.py b/parsedmarc/cli.py index f7b0086..0f978ee 100644 --- a/parsedmarc/cli.py +++ b/parsedmarc/cli.py @@ -16,8 +16,7 @@ from parsedmarc import IMAPError, get_dmarc_reports_from_inbox, \ parse_report_file, elastic, splunk, save_output, watch_inbox, \ email_results, SMTPError, ParserError, __version__ -logger = logging.getLogger(__name__) -logger.setLevel(logging.WARNING) +logger = logging.getLogger("parsedmarc") def _main(): diff --git a/parsedmarc/elastic.py b/parsedmarc/elastic.py index 462cd22..e9fd23f 100644 --- a/parsedmarc/elastic.py +++ b/parsedmarc/elastic.py @@ -8,7 +8,7 @@ from elasticsearch_dsl.search import Q from elasticsearch_dsl import connections, Object, Document, Index, Nested, \ InnerDoc, Integer, Text, Boolean, DateRange, Ip, Date -logger = logging.getLogger(__name__) +logger = logging.getLogger("parsedmarc") class _PolicyOverride(InnerDoc): diff --git a/parsedmarc/splunk.py b/parsedmarc/splunk.py index 2c00f74..42403e6 100644 --- a/parsedmarc/splunk.py +++ b/parsedmarc/splunk.py @@ -10,7 +10,7 @@ from parsedmarc import __version__, human_timestamp_to_timestamp urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) -logger = logging.getLogger(__name__) +logger = logging.getLogger("parsedmarc") class SplunkError(RuntimeError): From d25d01a230d2a24b1f274eb8c35bb9ee42079f51 Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Tue, 9 Oct 2018 23:28:55 -0400 Subject: [PATCH 10/14] More logging fixes --- parsedmarc/__init__.py | 1 - parsedmarc/cli.py | 10 ++++++---- parsedmarc/elastic.py | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index 4028667..75085e2 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -1343,7 +1343,6 @@ def get_dmarc_reports_from_inbox(host=None, forensic_report_msg_uids number_of_msgs = len(processed_messages) - logger.debug("Deleting messages") for i in range(number_of_msgs): msg_uid = processed_messages[i] logger.debug("Deleting message {0} of {1}: " diff --git a/parsedmarc/cli.py b/parsedmarc/cli.py index 0f978ee..870d70f 100644 --- a/parsedmarc/cli.py +++ b/parsedmarc/cli.py @@ -42,8 +42,9 @@ def _main(): if args.hec: try: aggregate_reports_ = reports_["aggregate_reports"] - hec_client.save_aggregate_reports_to_splunk( - aggregate_reports_) + if len(aggregate_reports_) > 0: + hec_client.save_aggregate_reports_to_splunk( + aggregate_reports_) except splunk.SplunkError as e: logger.error("Splunk HEC error: {0}".format(e.__str__())) if args.save_forensic: @@ -60,8 +61,9 @@ def _main(): if args.hec: try: forensic_reports_ = reports_["forensic_reports"] - hec_client.save_forensic_reports_to_splunk( - forensic_reports_) + if len(forensic_reports_) > 0: + hec_client.save_forensic_reports_to_splunk( + forensic_reports_) except splunk.SplunkError as e: logger.error("Splunk HEC error: {0}".format(e.__str__())) diff --git a/parsedmarc/elastic.py b/parsedmarc/elastic.py index e9fd23f..4649f78 100644 --- a/parsedmarc/elastic.py +++ b/parsedmarc/elastic.py @@ -182,7 +182,7 @@ def create_indexes(names=None, settings=None): settings (dict): Index settings """ - logger.debug("Creating elasticsearch indexes") + logger.debug("Creating Elasticsearch indexes") if names is None: names = ["dmarc_aggregate", "dmarc_forensic"] for name in names: From ee97a76654cc7463523a60d9510c3e6b00c13d53 Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Wed, 10 Oct 2018 08:25:32 -0400 Subject: [PATCH 11/14] More logging fixes --- parsedmarc/elastic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parsedmarc/elastic.py b/parsedmarc/elastic.py index 4649f78..c14f012 100644 --- a/parsedmarc/elastic.py +++ b/parsedmarc/elastic.py @@ -182,12 +182,12 @@ def create_indexes(names=None, settings=None): settings (dict): Index settings """ - logger.debug("Creating Elasticsearch indexes") if names is None: names = ["dmarc_aggregate", "dmarc_forensic"] for name in names: index = Index(name) if not index.exists(): + logger.debug("Creating Elasticsearch index: {0{".format(name)) if settings: index.put_settings(settings) index.create() From 35e6a72691e256c77c56db3aa68249b99b1c1fda Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Wed, 10 Oct 2018 08:27:28 -0400 Subject: [PATCH 12/14] Remove duplicate logging message --- parsedmarc/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index 75085e2..8892756 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -1245,7 +1245,6 @@ def get_dmarc_reports_from_inbox(host=None, move_supported = "MOVE" in server_capabilities def delete_messages(msg_uids): - logger.debug("Deleting messages") if type(msg_uids) == str: msg_uids = [msg_uids] From 20afbba7e2ee52ba9e2d7a10963ca3887af61082 Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Wed, 10 Oct 2018 08:29:11 -0400 Subject: [PATCH 13/14] PEP 8 fix --- parsedmarc/splunk.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/parsedmarc/splunk.py b/parsedmarc/splunk.py index 42403e6..1765d3f 100644 --- a/parsedmarc/splunk.py +++ b/parsedmarc/splunk.py @@ -144,6 +144,3 @@ class HECClient(object): raise SplunkError(e.__str__()) if response["code"] != 0: raise SplunkError(response["text"]) - - - From 88e3a5e0d689672ae07236c8c79d3c15e21fa728 Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Wed, 10 Oct 2018 09:40:35 -0400 Subject: [PATCH 14/14] 4.2.0 Release --- CHANGELOG.md | 2 +- README.rst | 4 +- docs/index.rst | 42 ++++++++++------- parsedmarc/__init__.py | 103 +++++++++++++++++++++++------------------ 4 files changed, 84 insertions(+), 67 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a716a3..d68c0ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ------ - Save each aggregate report record as a separate Splunk event -- Fix IAMP delete action +- Fix IMAP delete action (issue # 20) - Suppress Splunk SSL validation warnings - Change default logging level to `WARNING` diff --git a/README.rst b/README.rst index d74f820..db3ed73 100644 --- a/README.rst +++ b/README.rst @@ -76,7 +76,7 @@ CLI help IMAP password --imap-port IMAP_PORT IMAP port - --imap-no-ssl Do not use SSL when connecting to IMAP + --imap-no-ssl Do not use SSL/TLS when connecting to IMAP -r REPORTS_FOLDER, --reports-folder REPORTS_FOLDER The IMAP folder containing the reports Default: INBOX -a ARCHIVE_FOLDER, --archive-folder ARCHIVE_FOLDER @@ -127,7 +127,7 @@ CLI help -w, --watch Use an IMAP IDLE connection to process reports as they arrive in the inbox --test Do not move or delete IMAP messages - -s, --silent Only print errors + -s, --silent Only print errors and warnings --debug Print debugging information -v, --version show program's version number and exit diff --git a/docs/index.rst b/docs/index.rst index 72ae0f6..fe63a57 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -45,22 +45,22 @@ CLI help :: - usage: parsedmarc [-h] [-o OUTPUT] [-n NAMESERVERS [NAMESERVERS ...]] - [-t TIMEOUT] [-H HOST] [-u USER] [-p PASSWORD] - [--imap-port IMAP_PORT] [--imap-no-ssl] [-r REPORTS_FOLDER] - [-a ARCHIVE_FOLDER] [-d] - [-E [ELASTICSEARCH_HOST [ELASTICSEARCH_HOST ...]]] - [--elasticsearch-index-prefix ELASTICSEARCH_INDEX_PREFIX] - [--elasticsearch-index-suffix ELASTICSEARCH_INDEX_SUFFIX] - [--hec HEC] [--hec-token HEC_TOKEN] [--hec-index HEC_INDEX] - [--hec-skip-certificate-verification] [--save-aggregate] - [--save-forensic] [-O OUTGOING_HOST] [-U OUTGOING_USER] - [-P OUTGOING_PASSWORD] [--outgoing-port OUTGOING_PORT] - [--outgoing-ssl OUTGOING_SSL] [-F OUTGOING_FROM] - [-T OUTGOING_TO [OUTGOING_TO ...]] [-S OUTGOING_SUBJECT] - [-A OUTGOING_ATTACHMENT] [-M OUTGOING_MESSAGE] [-w] [--test] - [-s] [--debug] [-v] - [file_path [file_path ...]] + usage: parsedmarc [-h] [-o OUTPUT] [-n NAMESERVERS [NAMESERVERS ...]] + [-t TIMEOUT] [-H HOST] [-u USER] [-p PASSWORD] + [--imap-port IMAP_PORT] [--imap-no-ssl] [-r REPORTS_FOLDER] + [-a ARCHIVE_FOLDER] [-d] + [-E [ELASTICSEARCH_HOST [ELASTICSEARCH_HOST ...]]] + [--elasticsearch-index-prefix ELASTICSEARCH_INDEX_PREFIX] + [--elasticsearch-index-suffix ELASTICSEARCH_INDEX_SUFFIX] + [--hec HEC] [--hec-token HEC_TOKEN] [--hec-index HEC_INDEX] + [--hec-skip-certificate-verification] [--save-aggregate] + [--save-forensic] [-O OUTGOING_HOST] [-U OUTGOING_USER] + [-P OUTGOING_PASSWORD] [--outgoing-port OUTGOING_PORT] + [--outgoing-ssl OUTGOING_SSL] [-F OUTGOING_FROM] + [-T OUTGOING_TO [OUTGOING_TO ...]] [-S OUTGOING_SUBJECT] + [-A OUTGOING_ATTACHMENT] [-M OUTGOING_MESSAGE] [-w] [--test] + [-s] [--debug] [-v] + [file_path [file_path ...]] Parses DMARC reports @@ -83,7 +83,7 @@ CLI help IMAP password --imap-port IMAP_PORT IMAP port - --imap-no-ssl Do not use SSL when connecting to IMAP + --imap-no-ssl Do not use SSL/TLS when connecting to IMAP -r REPORTS_FOLDER, --reports-folder REPORTS_FOLDER The IMAP folder containing the reports Default: INBOX -a ARCHIVE_FOLDER, --archive-folder ARCHIVE_FOLDER @@ -134,7 +134,7 @@ CLI help -w, --watch Use an IMAP IDLE connection to process reports as they arrive in the inbox --test Do not move or delete IMAP messages - -s, --silent Only print errors + -s, --silent Only print errors and warnings --debug Print debugging information -v, --version show program's version number and exit @@ -544,6 +544,12 @@ Om the same system as Elasticsearch, pass ``--save-aggregate`` and/or privacy reasons. While aggregate DMARC reports are sent at least daily, it is normal to receive very few forensic reports. + An alternative approach is to still collect forensic/failure/ruf reports + in your DMARC inbox, but run ``parsedmarc --save-forensic`` manually on a + separate IMAP folder (using the ``-r`` option), after you have manually + moved known samples you want to save to that folder (e.g. malicious + samples non-sensitive legitimate samples). + When you first visit Kibana, it will prompt you to create an index pattern. Start by creating the index pattern ``dmarc_aggregate`` (without an ``*``), diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index 8892756..901bce0 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -125,8 +125,8 @@ def _get_base_domain(domain): try: download_psl() except Exception as error: - logger.warning("Failed to download an updated PSL - \ - {0}".format(error)) + logger.warning( + "Failed to download an updated PSL {0}".format(error)) with open(psl_path, encoding="utf-8") as psl_file: psl = publicsuffix.PublicSuffixList(psl_file) @@ -566,18 +566,18 @@ def parse_aggregate_report_xml(xml, nameservers=None, timeout=2.0): return new_report except expat.ExpatError as error: - raise InvalidAggregateReport("Invalid XML: " - "{0}".format(error.__str__())) + raise InvalidAggregateReport( + "Invalid XML: {0}".format(error.__str__())) except KeyError as error: - raise InvalidAggregateReport("Missing field: " - "{0}".format(error.__str__())) + raise InvalidAggregateReport( + "Missing field: {0}".format(error.__str__())) except AttributeError: raise InvalidAggregateReport("Report missing required section") except Exception as error: - raise InvalidAggregateReport("Unexpected error: " - "{0}".format(error.__str__())) + raise InvalidAggregateReport( + "Unexpected error: {0}".format(error.__str__())) def extract_xml(input_): @@ -618,8 +618,8 @@ def extract_xml(input_): raise InvalidAggregateReport("File objects must be opened in binary " "(rb) mode") except Exception as error: - raise InvalidAggregateReport("Invalid archive file: " - "{0}".format(error.__str__())) + raise InvalidAggregateReport( + "Invalid archive file: {0}".format(error.__str__())) return xml @@ -918,8 +918,8 @@ def parse_forensic_report(feedback_report, sample, sample_headers_only, error.__str__())) except Exception as error: - raise InvalidForensicReport("Unexpected error: " - "{0}".format(error.__str__())) + raise InvalidForensicReport( + "Unexpected error: {0}".format(error.__str__())) def parsed_forensic_reports_to_csv(reports): @@ -1262,11 +1262,14 @@ def get_dmarc_reports_from_inbox(host=None, delete_messages(msg_uids) if not server.folder_exists(archive_folder): + logger.debug("Creating IMAP folder: {0}".format(archive_folder)) server.create_folder(archive_folder) try: # Test subfolder creation if not server.folder_exists(aggregate_reports_folder): server.create_folder(aggregate_reports_folder) + logger.debug( + "Creating IMAP folder: {0}".format(archive_folder)) except imapclient.exceptions.IMAPClientError: # Only replace / with . when . doesn't work # This usually indicates a dovecot IMAP server @@ -1274,17 +1277,19 @@ def get_dmarc_reports_from_inbox(host=None, ".") forensic_reports_folder = forensic_reports_folder.replace("/", ".") + subfolders = [aggregate_reports_folder, + forensic_reports_folder, + invalid_reports_folder] - if not server.folder_exists(aggregate_reports_folder): - server.create_folder(aggregate_reports_folder) - if not server.folder_exists(forensic_reports_folder): - server.create_folder(forensic_reports_folder) - if not server.folder_exists(invalid_reports_folder): - server.create_folder(invalid_reports_folder) + for subfolder in subfolders: + if not server.folder_exists(subfolder): + logger.debug( + "Creating IMAP folder: {0}".format(subfolder)) + server.create_folder(subfolder) server.select_folder(reports_folder) messages = server.search() - logger.debug("Found {0} messages in IMAP folder " - "{1}".format(len(messages), reports_folder)) + logger.debug("Found {0} messages in IMAP folder {1}".format( + len(messages), reports_folder)) for i in range(len(messages)): number_of_messages = len(messages) message_uid = messages[i] @@ -1325,16 +1330,20 @@ def get_dmarc_reports_from_inbox(host=None, except imapclient.exceptions.IMAPClientError as error: error = error.__str__().lstrip("b'").rstrip("'").rstrip(".") error = "IMAP error: Skipping message UID {0}: {1}".format( - message_uid, error - ) + message_uid, error) logger.error("IMAP error: {0}".format(error)) except InvalidDMARCReport as error: logger.warning(error.__str__()) if not test: if delete: + logger.debug( + "Deleting message UID {0}".format(message_uid)) delete_messages([message_uid]) else: move_messages([message_uid], invalid_reports_folder) + logger.debug( + "Moving message UID {0} to {1)".format( + message_uid, invalid_reports_folder)) if not test: if delete: @@ -1344,18 +1353,17 @@ def get_dmarc_reports_from_inbox(host=None, number_of_msgs = len(processed_messages) for i in range(number_of_msgs): msg_uid = processed_messages[i] - logger.debug("Deleting message {0} of {1}: " - "UID {2}".format(i + 1, - number_of_msgs, - msg_uid)) + logger.debug( + "Deleting message {0} of {1}: UID {2}".format( + i + 1, number_of_msgs, msg_uid)) try: delete_messages([msg_uid]) except imapclient.exceptions.IMAPClientError as e: e = e.__str__().lstrip("b'").rstrip( "'").rstrip(".") - e = "IMAP error: Error deleting message UID {0}: " \ - "{1}".format(msg_uid, e) + message = "Error deleting message UID" + e = "{0} {1}: " "{2}".format(message, msg_uid, e) logger.error("IMAP error: {0}".format(e)) except (ConnectionResetError, TimeoutError) as e: logger.debug("IMAP error: {0}".format(e.__str__())) @@ -1369,24 +1377,25 @@ def get_dmarc_reports_from_inbox(host=None, delete_messages([msg_uid]) else: if len(aggregate_report_msg_uids) > 0: - logger.debug("Moving aggregate report messages " - "from {0} to " - "{1}".format(reports_folder, - aggregate_reports_folder)) + log_message = "Moving aggregate report messages from" + logger.debug( + "{0} {1} to {1}".format( + log_message, reports_folder, + aggregate_reports_folder)) number_of_msgs = len(aggregate_report_msg_uids) for i in range(number_of_msgs): msg_uid = aggregate_report_msg_uids[i] - logger.debug("Moving message {0} of {1}: " - "UID {2}".format(i+1, number_of_msgs, - msg_uid)) + logger.debug( + "Moving message {0} of {1}: UID {2}".format( + i+1, number_of_msgs, msg_uid)) try: move_messages([msg_uid], aggregate_reports_folder) except imapclient.exceptions.IMAPClientError as e: e = e.__str__().lstrip("b'").rstrip( "'").rstrip(".") - e = "Error moving message UID {0}: " \ - "{1}".format(msg_uid, e) + message = "Error moving message UID" + e = "{0} {1}: {2}".format(message, msg_uid, e) logger.error("IMAP error: {0}".format(e)) except (ConnectionResetError, TimeoutError) as error: logger.debug("IMAP error: {0}".format( @@ -1402,24 +1411,26 @@ def get_dmarc_reports_from_inbox(host=None, aggregate_reports_folder) if len(forensic_report_msg_uids) > 0: - logger.debug("Moving forensic report messages " - "from {0} to " - "{1}".format(reports_folder, - forensic_reports_folder)) + message = "Moving forensic report messages from" + logger.debug( + "{0} {1} to {2}".format(message, + reports_folder, + forensic_reports_folder)) number_of_msgs = len(forensic_report_msg_uids) for i in range(number_of_msgs): msg_uid = forensic_report_msg_uids[i] - logger.debug("Moving message {0} of {1}: " - "UID {2}".format(i + 1, number_of_msgs, - msg_uid)) + message = "Moving message" + logger.debug("{0} {1} of {2}: UID {2}".format( + message, + i + 1, number_of_msgs, msg_uid)) try: move_messages([msg_uid], forensic_reports_folder) except imapclient.exceptions.IMAPClientError as e: e = e.__str__().lstrip("b'").rstrip( "'").rstrip(".") - e = "Error moving message UID {0}: " \ - "{1}".format(msg_uid, e) + e = "Error moving message UID {0}: {1}".format( + msg_uid, e) logger.error("IMAP Error: {0}".format(e)) except (ConnectionResetError, TimeoutError) as error: logger.debug("IMAP error: {0}".format(