diff --git a/CHANGELOG.md b/CHANGELOG.md index 1176ee2..d68c0ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ------ - Save each aggregate report record as a separate Splunk event +- Fix IMAP delete action (issue # 20) +- Suppress Splunk SSL validation warnings +- Change default logging level to `WARNING` + 4.1.9 ----- diff --git a/README.rst b/README.rst index d74f820..db3ed73 100644 --- a/README.rst +++ b/README.rst @@ -76,7 +76,7 @@ CLI help IMAP password --imap-port IMAP_PORT IMAP port - --imap-no-ssl Do not use SSL when connecting to IMAP + --imap-no-ssl Do not use SSL/TLS when connecting to IMAP -r REPORTS_FOLDER, --reports-folder REPORTS_FOLDER The IMAP folder containing the reports Default: INBOX -a ARCHIVE_FOLDER, --archive-folder ARCHIVE_FOLDER @@ -127,7 +127,7 @@ CLI help -w, --watch Use an IMAP IDLE connection to process reports as they arrive in the inbox --test Do not move or delete IMAP messages - -s, --silent Only print errors + -s, --silent Only print errors and warnings --debug Print debugging information -v, --version show program's version number and exit diff --git a/docs/index.rst b/docs/index.rst index 72ae0f6..fe63a57 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -45,22 +45,22 @@ CLI help :: - usage: parsedmarc [-h] [-o OUTPUT] [-n NAMESERVERS [NAMESERVERS ...]] - [-t TIMEOUT] [-H HOST] [-u USER] [-p PASSWORD] - [--imap-port IMAP_PORT] [--imap-no-ssl] [-r REPORTS_FOLDER] - [-a ARCHIVE_FOLDER] [-d] - [-E [ELASTICSEARCH_HOST [ELASTICSEARCH_HOST ...]]] - [--elasticsearch-index-prefix ELASTICSEARCH_INDEX_PREFIX] - [--elasticsearch-index-suffix ELASTICSEARCH_INDEX_SUFFIX] - [--hec HEC] [--hec-token HEC_TOKEN] [--hec-index HEC_INDEX] - [--hec-skip-certificate-verification] [--save-aggregate] - [--save-forensic] [-O OUTGOING_HOST] [-U OUTGOING_USER] - [-P OUTGOING_PASSWORD] [--outgoing-port OUTGOING_PORT] - [--outgoing-ssl OUTGOING_SSL] [-F OUTGOING_FROM] - [-T OUTGOING_TO [OUTGOING_TO ...]] [-S OUTGOING_SUBJECT] - [-A OUTGOING_ATTACHMENT] [-M OUTGOING_MESSAGE] [-w] [--test] - [-s] [--debug] [-v] - [file_path [file_path ...]] + usage: parsedmarc [-h] [-o OUTPUT] [-n NAMESERVERS [NAMESERVERS ...]] + [-t TIMEOUT] [-H HOST] [-u USER] [-p PASSWORD] + [--imap-port IMAP_PORT] [--imap-no-ssl] [-r REPORTS_FOLDER] + [-a ARCHIVE_FOLDER] [-d] + [-E [ELASTICSEARCH_HOST [ELASTICSEARCH_HOST ...]]] + [--elasticsearch-index-prefix ELASTICSEARCH_INDEX_PREFIX] + [--elasticsearch-index-suffix ELASTICSEARCH_INDEX_SUFFIX] + [--hec HEC] [--hec-token HEC_TOKEN] [--hec-index HEC_INDEX] + [--hec-skip-certificate-verification] [--save-aggregate] + [--save-forensic] [-O OUTGOING_HOST] [-U OUTGOING_USER] + [-P OUTGOING_PASSWORD] [--outgoing-port OUTGOING_PORT] + [--outgoing-ssl OUTGOING_SSL] [-F OUTGOING_FROM] + [-T OUTGOING_TO [OUTGOING_TO ...]] [-S OUTGOING_SUBJECT] + [-A OUTGOING_ATTACHMENT] [-M OUTGOING_MESSAGE] [-w] [--test] + [-s] [--debug] [-v] + [file_path [file_path ...]] Parses DMARC reports @@ -83,7 +83,7 @@ CLI help IMAP password --imap-port IMAP_PORT IMAP port - --imap-no-ssl Do not use SSL when connecting to IMAP + --imap-no-ssl Do not use SSL/TLS when connecting to IMAP -r REPORTS_FOLDER, --reports-folder REPORTS_FOLDER The IMAP folder containing the reports Default: INBOX -a ARCHIVE_FOLDER, --archive-folder ARCHIVE_FOLDER @@ -134,7 +134,7 @@ CLI help -w, --watch Use an IMAP IDLE connection to process reports as they arrive in the inbox --test Do not move or delete IMAP messages - -s, --silent Only print errors + -s, --silent Only print errors and warnings --debug Print debugging information -v, --version show program's version number and exit @@ -544,6 +544,12 @@ Om the same system as Elasticsearch, pass ``--save-aggregate`` and/or privacy reasons. While aggregate DMARC reports are sent at least daily, it is normal to receive very few forensic reports. + An alternative approach is to still collect forensic/failure/ruf reports + in your DMARC inbox, but run ``parsedmarc --save-forensic`` manually on a + separate IMAP folder (using the ``-r`` option), after you have manually + moved known samples you want to save to that folder (e.g. malicious + samples non-sensitive legitimate samples). + When you first visit Kibana, it will prompt you to create an index pattern. Start by creating the index pattern ``dmarc_aggregate`` (without an ``*``), diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index 6f4162d..72e1ad6 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -46,8 +46,7 @@ import mailparser __version__ = "4.2.0k" -logger = logging.getLogger(__name__) -logger.setLevel(logging.ERROR) +logger = logging.getLogger("parsedmarc") feedback_report_regex = re.compile(r"^([\w\-]+): (.+)$", re.MULTILINE) xml_header_regex = re.compile(r"^<\?xml .*$", re.MULTILINE) @@ -126,8 +125,8 @@ def _get_base_domain(domain): try: download_psl() except Exception as error: - logger.warning("Failed to download an updated PSL - \ - {0}".format(error)) + logger.warning( + "Failed to download an updated PSL {0}".format(error)) with open(psl_path, encoding="utf-8") as psl_file: psl = publicsuffix.PublicSuffixList(psl_file) @@ -567,18 +566,18 @@ def parse_aggregate_report_xml(xml, nameservers=None, timeout=2.0): return new_report except expat.ExpatError as error: - raise InvalidAggregateReport("Invalid XML: " - "{0}".format(error.__str__())) + raise InvalidAggregateReport( + "Invalid XML: {0}".format(error.__str__())) except KeyError as error: - raise InvalidAggregateReport("Missing field: " - "{0}".format(error.__str__())) + raise InvalidAggregateReport( + "Missing field: {0}".format(error.__str__())) except AttributeError: raise InvalidAggregateReport("Report missing required section") except Exception as error: - raise InvalidAggregateReport("Unexpected error: " - "{0}".format(error.__str__())) + raise InvalidAggregateReport( + "Unexpected error: {0}".format(error.__str__())) def extract_xml(input_): @@ -619,8 +618,8 @@ def extract_xml(input_): raise InvalidAggregateReport("File objects must be opened in binary " "(rb) mode") except Exception as error: - raise InvalidAggregateReport("Invalid archive file: " - "{0}".format(error.__str__())) + raise InvalidAggregateReport( + "Invalid archive file: {0}".format(error.__str__())) return xml @@ -919,8 +918,8 @@ def parse_forensic_report(feedback_report, sample, sample_headers_only, error.__str__())) except Exception as error: - raise InvalidForensicReport("Unexpected error: " - "{0}".format(error.__str__())) + raise InvalidForensicReport( + "Unexpected error: {0}".format(error.__str__())) def parsed_forensic_reports_to_csv(reports): @@ -1249,10 +1248,8 @@ def get_dmarc_reports_from_inbox(host=None, if type(msg_uids) == str: msg_uids = [msg_uids] - for chunk in chunks(msg_uids, 100): - server.add_flags(chunk, [imapclient.DELETED]) - - server.expunge() + server.delete_messages(msg_uids, silent=True) + server.expunge(msg_uids) def move_messages(msg_uids, folder): if type(msg_uids) == str: @@ -1265,11 +1262,14 @@ def get_dmarc_reports_from_inbox(host=None, delete_messages(msg_uids) if not server.folder_exists(archive_folder): + logger.debug("Creating IMAP folder: {0}".format(archive_folder)) server.create_folder(archive_folder) try: # Test subfolder creation if not server.folder_exists(aggregate_reports_folder): server.create_folder(aggregate_reports_folder) + logger.debug( + "Creating IMAP folder: {0}".format(archive_folder)) except imapclient.exceptions.IMAPClientError: # Only replace / with . when . doesn't work # This usually indicates a dovecot IMAP server @@ -1277,18 +1277,19 @@ def get_dmarc_reports_from_inbox(host=None, ".") forensic_reports_folder = forensic_reports_folder.replace("/", ".") + subfolders = [aggregate_reports_folder, + forensic_reports_folder, + invalid_reports_folder] - if not server.folder_exists(aggregate_reports_folder): - server.create_folder(aggregate_reports_folder) - if not server.folder_exists(forensic_reports_folder): - server.create_folder(forensic_reports_folder) - if not server.folder_exists(invalid_reports_folder): - server.create_folder(invalid_reports_folder) + for subfolder in subfolders: + if not server.folder_exists(subfolder): + logger.debug( + "Creating IMAP folder: {0}".format(subfolder)) + server.create_folder(subfolder) server.select_folder(reports_folder) messages = server.search() - logger.debug("Found {0} messages in IMAP folder".format(len(messages), - reports_folder - )) + logger.debug("Found {0} messages in IMAP folder {1}".format( + len(messages), reports_folder)) for i in range(len(messages)): number_of_messages = len(messages) message_uid = messages[i] @@ -1329,16 +1330,20 @@ def get_dmarc_reports_from_inbox(host=None, except imapclient.exceptions.IMAPClientError as error: error = error.__str__().lstrip("b'").rstrip("'").rstrip(".") error = "IMAP error: Skipping message UID {0}: {1}".format( - message_uid, error - ) + message_uid, error) logger.error("IMAP error: {0}".format(error)) except InvalidDMARCReport as error: logger.warning(error.__str__()) if not test: if delete: + logger.debug( + "Deleting message UID {0}".format(message_uid)) delete_messages([message_uid]) else: move_messages([message_uid], invalid_reports_folder) + logger.debug( + "Moving message UID {0} to {1)".format( + message_uid, invalid_reports_folder)) if not test: if delete: @@ -1346,21 +1351,19 @@ def get_dmarc_reports_from_inbox(host=None, forensic_report_msg_uids number_of_msgs = len(processed_messages) - logger.debug("Deleting messages") for i in range(number_of_msgs): msg_uid = processed_messages[i] - logger.debug("Deleting message {0} of {1}: " - "UID {2}".format(i + 1, - number_of_msgs, - msg_uid)) + logger.debug( + "Deleting message {0} of {1}: UID {2}".format( + i + 1, number_of_msgs, msg_uid)) try: delete_messages([msg_uid]) except imapclient.exceptions.IMAPClientError as e: e = e.__str__().lstrip("b'").rstrip( "'").rstrip(".") - e = "IMAP error: Error deleting message UID {0}: " \ - "{1}".format(msg_uid, e) + message = "Error deleting message UID" + e = "{0} {1}: " "{2}".format(message, msg_uid, e) logger.error("IMAP error: {0}".format(e)) except (ConnectionResetError, TimeoutError) as e: logger.debug("IMAP error: {0}".format(e.__str__())) @@ -1374,24 +1377,25 @@ def get_dmarc_reports_from_inbox(host=None, delete_messages([msg_uid]) else: if len(aggregate_report_msg_uids) > 0: - logger.debug("Moving aggregate report messages " - "from {0} to " - "{1}".format(reports_folder, - aggregate_reports_folder)) + log_message = "Moving aggregate report messages from" + logger.debug( + "{0} {1} to {1}".format( + log_message, reports_folder, + aggregate_reports_folder)) number_of_msgs = len(aggregate_report_msg_uids) for i in range(number_of_msgs): msg_uid = aggregate_report_msg_uids[i] - logger.debug("Moving message {0} of {1}: " - "UID {2}".format(i+1, number_of_msgs, - msg_uid)) + logger.debug( + "Moving message {0} of {1}: UID {2}".format( + i+1, number_of_msgs, msg_uid)) try: move_messages([msg_uid], aggregate_reports_folder) except imapclient.exceptions.IMAPClientError as e: e = e.__str__().lstrip("b'").rstrip( "'").rstrip(".") - e = "Error moving message UID {0}: " \ - "{1}".format(msg_uid, e) + message = "Error moving message UID" + e = "{0} {1}: {2}".format(message, msg_uid, e) logger.error("IMAP error: {0}".format(e)) except (ConnectionResetError, TimeoutError) as error: logger.debug("IMAP error: {0}".format( @@ -1407,24 +1411,26 @@ def get_dmarc_reports_from_inbox(host=None, aggregate_reports_folder) if len(forensic_report_msg_uids) > 0: - logger.debug("Moving forensic report messages " - "from {0} to " - "{1}".format(reports_folder, - forensic_reports_folder)) + message = "Moving forensic report messages from" + logger.debug( + "{0} {1} to {2}".format(message, + reports_folder, + forensic_reports_folder)) number_of_msgs = len(forensic_report_msg_uids) for i in range(number_of_msgs): msg_uid = forensic_report_msg_uids[i] - logger.debug("Moving message {0} of {1}: " - "UID {2}".format(i + 1, number_of_msgs, - msg_uid)) + message = "Moving message" + logger.debug("{0} {1} of {2}: UID {2}".format( + message, + i + 1, number_of_msgs, msg_uid)) try: move_messages([msg_uid], forensic_reports_folder) except imapclient.exceptions.IMAPClientError as e: e = e.__str__().lstrip("b'").rstrip( "'").rstrip(".") - e = "Error moving message UID {0}: " \ - "{1}".format(msg_uid, e) + e = "Error moving message UID {0}: {1}".format( + msg_uid, e) logger.error("IMAP Error: {0}".format(e)) except (ConnectionResetError, TimeoutError) as error: logger.debug("IMAP error: {0}".format( @@ -1589,6 +1595,7 @@ def email_results(results, host, mail_from, mail_to, port=0, message: Override the default plain text body ssl_context: SSL context options """ + logging.debug("Emailing report to: {0}".format(",".join(mail_to))) date_string = datetime.now().strftime("%Y-%m-%d") if attachment_filename: if not attachment_filename.lower().endswith(".zip"): diff --git a/parsedmarc/cli.py b/parsedmarc/cli.py index 038cf6b..852b6eb 100644 --- a/parsedmarc/cli.py +++ b/parsedmarc/cli.py @@ -16,6 +16,8 @@ from parsedmarc import logger, IMAPError, get_dmarc_reports_from_inbox, \ parse_report_file, elastic, kafkaclient, splunk, save_output, \ watch_inbox, email_results, SMTPError, ParserError, __version__ +logger = logging.getLogger("parsedmarc") + def _main(): """Called when the module is executed""" @@ -53,8 +55,9 @@ def _main(): if args.hec: try: aggregate_reports_ = reports_["aggregate_reports"] - hec_client.save_aggregate_reports_to_splunk( - aggregate_reports_) + if len(aggregate_reports_) > 0: + hec_client.save_aggregate_reports_to_splunk( + aggregate_reports_) except splunk.SplunkError as e: logger.error("Splunk HEC error: {0}".format(e.__str__())) if args.save_forensic: @@ -79,8 +82,9 @@ def _main(): if args.hec: try: forensic_reports_ = reports_["forensic_reports"] - hec_client.save_forensic_reports_to_splunk( - forensic_reports_) + if len(forensic_reports_) > 0: + hec_client.save_forensic_reports_to_splunk( + forensic_reports_) except splunk.SplunkError as e: logger.error("Splunk HEC error: {0}".format(e.__str__())) @@ -104,7 +108,7 @@ def _main(): arg_parser.add_argument("--imap-port", default=None, help="IMAP port") arg_parser.add_argument("--imap-no-ssl", action="store_true", default=False, - help="Do not use SSL when connecting to IMAP") + help="Do not use SSL/TLS when connecting to IMAP") arg_parser.add_argument("-r", "--reports-folder", default="INBOX", help="The IMAP folder containing the reports\n" "Default: INBOX") @@ -187,7 +191,7 @@ def _main(): help="Do not move or delete IMAP messages", action="store_true", default=False) arg_parser.add_argument("-s", "--silent", action="store_true", - help="Only print errors") + help="Only print errors and warnings") arg_parser.add_argument("--debug", action="store_true", help="Print debugging information") arg_parser.add_argument("-v", "--version", action="version", @@ -198,8 +202,9 @@ def _main(): args = arg_parser.parse_args() - logging.basicConfig(level=logging.ERROR) - logger.setLevel(logging.ERROR) + logging.basicConfig(level=logging.WARNING) + logger.setLevel(logging.WARNING) + if args.debug: logging.basicConfig(level=logging.DEBUG) logger.setLevel(logging.DEBUG) diff --git a/parsedmarc/elastic.py b/parsedmarc/elastic.py index c426755..c14f012 100644 --- a/parsedmarc/elastic.py +++ b/parsedmarc/elastic.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- +import logging from collections import OrderedDict import parsedmarc @@ -7,6 +8,8 @@ from elasticsearch_dsl.search import Q from elasticsearch_dsl import connections, Object, Document, Index, Nested, \ InnerDoc, Integer, Text, Boolean, DateRange, Ip, Date +logger = logging.getLogger("parsedmarc") + class _PolicyOverride(InnerDoc): type = Text() @@ -184,6 +187,7 @@ def create_indexes(names=None, settings=None): for name in names: index = Index(name) if not index.exists(): + logger.debug("Creating Elasticsearch index: {0{".format(name)) if settings: index.put_settings(settings) index.create() @@ -201,6 +205,7 @@ def save_aggregate_report_to_elasticsearch(aggregate_report, Raises: AlreadySaved """ + logger.debug("Saving aggregate report to Elasticsearch") aggregate_report = aggregate_report.copy() metadata = aggregate_report["report_metadata"] org_name = metadata["org_name"] @@ -299,6 +304,7 @@ def save_forensic_report_to_elasticsearch(forensic_report, AlreadySaved """ + logger.debug("Saving forensic report to Elasticsearch") forensic_report = forensic_report.copy() sample_date = forensic_report["parsed_sample"]["date"] sample_date = parsedmarc.human_timestamp_to_datetime(sample_date) diff --git a/parsedmarc/splunk.py b/parsedmarc/splunk.py index 8448d15..1765d3f 100644 --- a/parsedmarc/splunk.py +++ b/parsedmarc/splunk.py @@ -1,11 +1,17 @@ +import logging from urllib.parse import urlparse import socket import json +import urllib3 import requests from parsedmarc import __version__, human_timestamp_to_timestamp +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +logger = logging.getLogger("parsedmarc") + class SplunkError(RuntimeError): """Raised when a Splunk API error occurs""" @@ -54,6 +60,7 @@ class HECClient(object): to save in Splunk """ + logger.debug("Saving aggregate reports to Splunk") if type(aggregate_reports) == dict: aggregate_reports = [aggregate_reports] @@ -115,6 +122,7 @@ class HECClient(object): to save in Splunk """ + logger.debug("Saving forensic reports to Splunk") if type(forensic_reports) == dict: forensic_reports = [forensic_reports] diff --git a/requirements.txt b/requirements.txt index 023ed17..b3a4bc4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ dnspython +urllib3 requests publicsuffix xmltodict diff --git a/setup.py b/setup.py index a4a1e69..8e6412f 100644 --- a/setup.py +++ b/setup.py @@ -92,8 +92,8 @@ setup( # requirements files see: # https://packaging.python.org/en/latest/requirements.html install_requires=['dnspython', 'publicsuffix', 'xmltodict', 'geoip2', - 'dnspython', 'imapclient', 'mail-parser', 'dateparser', - 'elasticsearch>=6.3.0,<7.0.0', + 'urllib3', 'requests', 'imapclient', 'mail-parser', + 'dateparser', 'elasticsearch>=6.3.0,<7.0.0', 'elasticsearch-dsl>=6.2.1,<7.0.0' ], diff --git a/splunk/dmarc_forensic_dashboard.xml b/splunk/dmarc_forensic_dashboard.xml index a4d611b..f501b76 100644 --- a/splunk/dmarc_forensic_dashboard.xml +++ b/splunk/dmarc_forensic_dashboard.xml @@ -38,7 +38,7 @@ Forensic samples - index="email" sourcetype="dmarc:forensic" parsed_sample.headers.From=$header_from$ parsed_sample.headers.To=$header_to$ parsed_sample.headers.Subject=$header_subject$ source.ip_address=$source_ip_address$ source.reverse_dns=$source_reverse_dns$ source.country=$source_country$ | fillnull value="none" | stats count by _time,parsed_sample.headers.From,parsed_sample.headers.To,parsed_sample.headers.Reply-To,parsed_sample.headers.Subject | sort -_time + index="email" sourcetype="dmarc:forensic" parsed_sample.headers.From=$header_from$ parsed_sample.headers.To=$header_to$ parsed_sample.headers.Subject=$header_subject$ source.ip_address=$source_ip_address$ source.reverse_dns=$source_reverse_dns$ source.country=$source_country$ | fillnull value="none" | stats count by arrival_date_utc,parsed_sample.headers.From,parsed_sample.headers.To,parsed_sample.headers.Reply-To,parsed_sample.headers.Subject | sort -arrival_date_utc $time_range.earliest$ $time_range.latest$ @@ -95,4 +95,4 @@
- + \ No newline at end of file