From 88e3a5e0d689672ae07236c8c79d3c15e21fa728 Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Wed, 10 Oct 2018 09:40:35 -0400 Subject: [PATCH] 4.2.0 Release --- CHANGELOG.md | 2 +- README.rst | 4 +- docs/index.rst | 42 ++++++++++------- parsedmarc/__init__.py | 103 +++++++++++++++++++++++------------------ 4 files changed, 84 insertions(+), 67 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a716a31..d68c0acc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ------ - Save each aggregate report record as a separate Splunk event -- Fix IAMP delete action +- Fix IMAP delete action (issue # 20) - Suppress Splunk SSL validation warnings - Change default logging level to `WARNING` diff --git a/README.rst b/README.rst index d74f8206..db3ed73d 100644 --- a/README.rst +++ b/README.rst @@ -76,7 +76,7 @@ CLI help IMAP password --imap-port IMAP_PORT IMAP port - --imap-no-ssl Do not use SSL when connecting to IMAP + --imap-no-ssl Do not use SSL/TLS when connecting to IMAP -r REPORTS_FOLDER, --reports-folder REPORTS_FOLDER The IMAP folder containing the reports Default: INBOX -a ARCHIVE_FOLDER, --archive-folder ARCHIVE_FOLDER @@ -127,7 +127,7 @@ CLI help -w, --watch Use an IMAP IDLE connection to process reports as they arrive in the inbox --test Do not move or delete IMAP messages - -s, --silent Only print errors + -s, --silent Only print errors and warnings --debug Print debugging information -v, --version show program's version number and exit diff --git a/docs/index.rst b/docs/index.rst index 72ae0f65..fe63a575 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -45,22 +45,22 @@ CLI help :: - usage: parsedmarc [-h] [-o OUTPUT] [-n NAMESERVERS [NAMESERVERS ...]] - [-t TIMEOUT] [-H HOST] [-u USER] [-p PASSWORD] - [--imap-port IMAP_PORT] [--imap-no-ssl] [-r REPORTS_FOLDER] - [-a ARCHIVE_FOLDER] [-d] - [-E [ELASTICSEARCH_HOST [ELASTICSEARCH_HOST ...]]] - [--elasticsearch-index-prefix ELASTICSEARCH_INDEX_PREFIX] - [--elasticsearch-index-suffix ELASTICSEARCH_INDEX_SUFFIX] - [--hec HEC] [--hec-token HEC_TOKEN] [--hec-index HEC_INDEX] - [--hec-skip-certificate-verification] [--save-aggregate] - [--save-forensic] [-O OUTGOING_HOST] [-U OUTGOING_USER] - [-P OUTGOING_PASSWORD] [--outgoing-port OUTGOING_PORT] - [--outgoing-ssl OUTGOING_SSL] [-F OUTGOING_FROM] - [-T OUTGOING_TO [OUTGOING_TO ...]] [-S OUTGOING_SUBJECT] - [-A OUTGOING_ATTACHMENT] [-M OUTGOING_MESSAGE] [-w] [--test] - [-s] [--debug] [-v] - [file_path [file_path ...]] + usage: parsedmarc [-h] [-o OUTPUT] [-n NAMESERVERS [NAMESERVERS ...]] + [-t TIMEOUT] [-H HOST] [-u USER] [-p PASSWORD] + [--imap-port IMAP_PORT] [--imap-no-ssl] [-r REPORTS_FOLDER] + [-a ARCHIVE_FOLDER] [-d] + [-E [ELASTICSEARCH_HOST [ELASTICSEARCH_HOST ...]]] + [--elasticsearch-index-prefix ELASTICSEARCH_INDEX_PREFIX] + [--elasticsearch-index-suffix ELASTICSEARCH_INDEX_SUFFIX] + [--hec HEC] [--hec-token HEC_TOKEN] [--hec-index HEC_INDEX] + [--hec-skip-certificate-verification] [--save-aggregate] + [--save-forensic] [-O OUTGOING_HOST] [-U OUTGOING_USER] + [-P OUTGOING_PASSWORD] [--outgoing-port OUTGOING_PORT] + [--outgoing-ssl OUTGOING_SSL] [-F OUTGOING_FROM] + [-T OUTGOING_TO [OUTGOING_TO ...]] [-S OUTGOING_SUBJECT] + [-A OUTGOING_ATTACHMENT] [-M OUTGOING_MESSAGE] [-w] [--test] + [-s] [--debug] [-v] + [file_path [file_path ...]] Parses DMARC reports @@ -83,7 +83,7 @@ CLI help IMAP password --imap-port IMAP_PORT IMAP port - --imap-no-ssl Do not use SSL when connecting to IMAP + --imap-no-ssl Do not use SSL/TLS when connecting to IMAP -r REPORTS_FOLDER, --reports-folder REPORTS_FOLDER The IMAP folder containing the reports Default: INBOX -a ARCHIVE_FOLDER, --archive-folder ARCHIVE_FOLDER @@ -134,7 +134,7 @@ CLI help -w, --watch Use an IMAP IDLE connection to process reports as they arrive in the inbox --test Do not move or delete IMAP messages - -s, --silent Only print errors + -s, --silent Only print errors and warnings --debug Print debugging information -v, --version show program's version number and exit @@ -544,6 +544,12 @@ Om the same system as Elasticsearch, pass ``--save-aggregate`` and/or privacy reasons. While aggregate DMARC reports are sent at least daily, it is normal to receive very few forensic reports. + An alternative approach is to still collect forensic/failure/ruf reports + in your DMARC inbox, but run ``parsedmarc --save-forensic`` manually on a + separate IMAP folder (using the ``-r`` option), after you have manually + moved known samples you want to save to that folder (e.g. malicious + samples non-sensitive legitimate samples). + When you first visit Kibana, it will prompt you to create an index pattern. Start by creating the index pattern ``dmarc_aggregate`` (without an ``*``), diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index 88927568..901bce02 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -125,8 +125,8 @@ def _get_base_domain(domain): try: download_psl() except Exception as error: - logger.warning("Failed to download an updated PSL - \ - {0}".format(error)) + logger.warning( + "Failed to download an updated PSL {0}".format(error)) with open(psl_path, encoding="utf-8") as psl_file: psl = publicsuffix.PublicSuffixList(psl_file) @@ -566,18 +566,18 @@ def parse_aggregate_report_xml(xml, nameservers=None, timeout=2.0): return new_report except expat.ExpatError as error: - raise InvalidAggregateReport("Invalid XML: " - "{0}".format(error.__str__())) + raise InvalidAggregateReport( + "Invalid XML: {0}".format(error.__str__())) except KeyError as error: - raise InvalidAggregateReport("Missing field: " - "{0}".format(error.__str__())) + raise InvalidAggregateReport( + "Missing field: {0}".format(error.__str__())) except AttributeError: raise InvalidAggregateReport("Report missing required section") except Exception as error: - raise InvalidAggregateReport("Unexpected error: " - "{0}".format(error.__str__())) + raise InvalidAggregateReport( + "Unexpected error: {0}".format(error.__str__())) def extract_xml(input_): @@ -618,8 +618,8 @@ def extract_xml(input_): raise InvalidAggregateReport("File objects must be opened in binary " "(rb) mode") except Exception as error: - raise InvalidAggregateReport("Invalid archive file: " - "{0}".format(error.__str__())) + raise InvalidAggregateReport( + "Invalid archive file: {0}".format(error.__str__())) return xml @@ -918,8 +918,8 @@ def parse_forensic_report(feedback_report, sample, sample_headers_only, error.__str__())) except Exception as error: - raise InvalidForensicReport("Unexpected error: " - "{0}".format(error.__str__())) + raise InvalidForensicReport( + "Unexpected error: {0}".format(error.__str__())) def parsed_forensic_reports_to_csv(reports): @@ -1262,11 +1262,14 @@ def get_dmarc_reports_from_inbox(host=None, delete_messages(msg_uids) if not server.folder_exists(archive_folder): + logger.debug("Creating IMAP folder: {0}".format(archive_folder)) server.create_folder(archive_folder) try: # Test subfolder creation if not server.folder_exists(aggregate_reports_folder): server.create_folder(aggregate_reports_folder) + logger.debug( + "Creating IMAP folder: {0}".format(archive_folder)) except imapclient.exceptions.IMAPClientError: # Only replace / with . when . doesn't work # This usually indicates a dovecot IMAP server @@ -1274,17 +1277,19 @@ def get_dmarc_reports_from_inbox(host=None, ".") forensic_reports_folder = forensic_reports_folder.replace("/", ".") + subfolders = [aggregate_reports_folder, + forensic_reports_folder, + invalid_reports_folder] - if not server.folder_exists(aggregate_reports_folder): - server.create_folder(aggregate_reports_folder) - if not server.folder_exists(forensic_reports_folder): - server.create_folder(forensic_reports_folder) - if not server.folder_exists(invalid_reports_folder): - server.create_folder(invalid_reports_folder) + for subfolder in subfolders: + if not server.folder_exists(subfolder): + logger.debug( + "Creating IMAP folder: {0}".format(subfolder)) + server.create_folder(subfolder) server.select_folder(reports_folder) messages = server.search() - logger.debug("Found {0} messages in IMAP folder " - "{1}".format(len(messages), reports_folder)) + logger.debug("Found {0} messages in IMAP folder {1}".format( + len(messages), reports_folder)) for i in range(len(messages)): number_of_messages = len(messages) message_uid = messages[i] @@ -1325,16 +1330,20 @@ def get_dmarc_reports_from_inbox(host=None, except imapclient.exceptions.IMAPClientError as error: error = error.__str__().lstrip("b'").rstrip("'").rstrip(".") error = "IMAP error: Skipping message UID {0}: {1}".format( - message_uid, error - ) + message_uid, error) logger.error("IMAP error: {0}".format(error)) except InvalidDMARCReport as error: logger.warning(error.__str__()) if not test: if delete: + logger.debug( + "Deleting message UID {0}".format(message_uid)) delete_messages([message_uid]) else: move_messages([message_uid], invalid_reports_folder) + logger.debug( + "Moving message UID {0} to {1)".format( + message_uid, invalid_reports_folder)) if not test: if delete: @@ -1344,18 +1353,17 @@ def get_dmarc_reports_from_inbox(host=None, number_of_msgs = len(processed_messages) for i in range(number_of_msgs): msg_uid = processed_messages[i] - logger.debug("Deleting message {0} of {1}: " - "UID {2}".format(i + 1, - number_of_msgs, - msg_uid)) + logger.debug( + "Deleting message {0} of {1}: UID {2}".format( + i + 1, number_of_msgs, msg_uid)) try: delete_messages([msg_uid]) except imapclient.exceptions.IMAPClientError as e: e = e.__str__().lstrip("b'").rstrip( "'").rstrip(".") - e = "IMAP error: Error deleting message UID {0}: " \ - "{1}".format(msg_uid, e) + message = "Error deleting message UID" + e = "{0} {1}: " "{2}".format(message, msg_uid, e) logger.error("IMAP error: {0}".format(e)) except (ConnectionResetError, TimeoutError) as e: logger.debug("IMAP error: {0}".format(e.__str__())) @@ -1369,24 +1377,25 @@ def get_dmarc_reports_from_inbox(host=None, delete_messages([msg_uid]) else: if len(aggregate_report_msg_uids) > 0: - logger.debug("Moving aggregate report messages " - "from {0} to " - "{1}".format(reports_folder, - aggregate_reports_folder)) + log_message = "Moving aggregate report messages from" + logger.debug( + "{0} {1} to {1}".format( + log_message, reports_folder, + aggregate_reports_folder)) number_of_msgs = len(aggregate_report_msg_uids) for i in range(number_of_msgs): msg_uid = aggregate_report_msg_uids[i] - logger.debug("Moving message {0} of {1}: " - "UID {2}".format(i+1, number_of_msgs, - msg_uid)) + logger.debug( + "Moving message {0} of {1}: UID {2}".format( + i+1, number_of_msgs, msg_uid)) try: move_messages([msg_uid], aggregate_reports_folder) except imapclient.exceptions.IMAPClientError as e: e = e.__str__().lstrip("b'").rstrip( "'").rstrip(".") - e = "Error moving message UID {0}: " \ - "{1}".format(msg_uid, e) + message = "Error moving message UID" + e = "{0} {1}: {2}".format(message, msg_uid, e) logger.error("IMAP error: {0}".format(e)) except (ConnectionResetError, TimeoutError) as error: logger.debug("IMAP error: {0}".format( @@ -1402,24 +1411,26 @@ def get_dmarc_reports_from_inbox(host=None, aggregate_reports_folder) if len(forensic_report_msg_uids) > 0: - logger.debug("Moving forensic report messages " - "from {0} to " - "{1}".format(reports_folder, - forensic_reports_folder)) + message = "Moving forensic report messages from" + logger.debug( + "{0} {1} to {2}".format(message, + reports_folder, + forensic_reports_folder)) number_of_msgs = len(forensic_report_msg_uids) for i in range(number_of_msgs): msg_uid = forensic_report_msg_uids[i] - logger.debug("Moving message {0} of {1}: " - "UID {2}".format(i + 1, number_of_msgs, - msg_uid)) + message = "Moving message" + logger.debug("{0} {1} of {2}: UID {2}".format( + message, + i + 1, number_of_msgs, msg_uid)) try: move_messages([msg_uid], forensic_reports_folder) except imapclient.exceptions.IMAPClientError as e: e = e.__str__().lstrip("b'").rstrip( "'").rstrip(".") - e = "Error moving message UID {0}: " \ - "{1}".format(msg_uid, e) + e = "Error moving message UID {0}: {1}".format( + msg_uid, e) logger.error("IMAP Error: {0}".format(e)) except (ConnectionResetError, TimeoutError) as error: logger.debug("IMAP error: {0}".format(