amorfo77/parsedmarc
1
0 Fork 0
mirror of https://github.com/domainaware/parsedmarc.git synced 2026-10-08 05:17:13 +00:00
Code Issues Projects Releases Wiki Activity

4.2.0 Release

Browse Source
This commit is contained in:
Sean Whalen committed 2018-10-10 09:40:35 -04:00
1 parent 20afbba7e2
commit 88e3a5e0d6
4 files changed
+84 -67
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

CHANGELOG.md
+1 -1
View File
@@ -2,7 +2,7 @@
------
- Save each aggregate report record as a separate Splunk event
- Fix IAMP delete action
- Fix IMAP delete action (issue # 20)
- Suppress Splunk SSL validation warnings
- Change default logging level to `WARNING`
README.rst
+2 -2
View File
@@ -76,7 +76,7 @@ CLI help
IMAP password
--imap-port IMAP_PORT
IMAP port
--imap-no-ssl Do not use SSL when connecting to IMAP
--imap-no-ssl Do not use SSL/TLS when connecting to IMAP
-r REPORTS_FOLDER, --reports-folder REPORTS_FOLDER
The IMAP folder containing the reports Default: INBOX
-a ARCHIVE_FOLDER, --archive-folder ARCHIVE_FOLDER
@@ -127,7 +127,7 @@ CLI help
-w, --watch Use an IMAP IDLE connection to process reports as they
arrive in the inbox
--test Do not move or delete IMAP messages
-s, --silent Only print errors
-s, --silent Only print errors and warnings
--debug Print debugging information
-v, --version show program's version number and exit
docs/index.rst
+24 -18
View File
@@ -45,22 +45,22 @@ CLI help
::
usage: parsedmarc [-h] [-o OUTPUT] [-n NAMESERVERS [NAMESERVERS ...]]
[-t TIMEOUT] [-H HOST] [-u USER] [-p PASSWORD]
[--imap-port IMAP_PORT] [--imap-no-ssl] [-r REPORTS_FOLDER]
[-a ARCHIVE_FOLDER] [-d]
[-E [ELASTICSEARCH_HOST [ELASTICSEARCH_HOST ...]]]
[--elasticsearch-index-prefix ELASTICSEARCH_INDEX_PREFIX]
[--elasticsearch-index-suffix ELASTICSEARCH_INDEX_SUFFIX]
[--hec HEC] [--hec-token HEC_TOKEN] [--hec-index HEC_INDEX]
[--hec-skip-certificate-verification] [--save-aggregate]
[--save-forensic] [-O OUTGOING_HOST] [-U OUTGOING_USER]
[-P OUTGOING_PASSWORD] [--outgoing-port OUTGOING_PORT]
[--outgoing-ssl OUTGOING_SSL] [-F OUTGOING_FROM]
[-T OUTGOING_TO [OUTGOING_TO ...]] [-S OUTGOING_SUBJECT]
[-A OUTGOING_ATTACHMENT] [-M OUTGOING_MESSAGE] [-w] [--test]
[-s] [--debug] [-v]
[file_path [file_path ...]]
usage: parsedmarc [-h] [-o OUTPUT] [-n NAMESERVERS [NAMESERVERS ...]]
[-t TIMEOUT] [-H HOST] [-u USER] [-p PASSWORD]
[--imap-port IMAP_PORT] [--imap-no-ssl] [-r REPORTS_FOLDER]
[-a ARCHIVE_FOLDER] [-d]
[-E [ELASTICSEARCH_HOST [ELASTICSEARCH_HOST ...]]]
[--elasticsearch-index-prefix ELASTICSEARCH_INDEX_PREFIX]
[--elasticsearch-index-suffix ELASTICSEARCH_INDEX_SUFFIX]
[--hec HEC] [--hec-token HEC_TOKEN] [--hec-index HEC_INDEX]
[--hec-skip-certificate-verification] [--save-aggregate]
[--save-forensic] [-O OUTGOING_HOST] [-U OUTGOING_USER]
[-P OUTGOING_PASSWORD] [--outgoing-port OUTGOING_PORT]
[--outgoing-ssl OUTGOING_SSL] [-F OUTGOING_FROM]
[-T OUTGOING_TO [OUTGOING_TO ...]] [-S OUTGOING_SUBJECT]
[-A OUTGOING_ATTACHMENT] [-M OUTGOING_MESSAGE] [-w] [--test]
[-s] [--debug] [-v]
[file_path [file_path ...]]
Parses DMARC reports
@@ -83,7 +83,7 @@ CLI help
IMAP password
--imap-port IMAP_PORT
IMAP port
--imap-no-ssl Do not use SSL when connecting to IMAP
--imap-no-ssl Do not use SSL/TLS when connecting to IMAP
-r REPORTS_FOLDER, --reports-folder REPORTS_FOLDER
The IMAP folder containing the reports Default: INBOX
-a ARCHIVE_FOLDER, --archive-folder ARCHIVE_FOLDER
@@ -134,7 +134,7 @@ CLI help
-w, --watch Use an IMAP IDLE connection to process reports as they
arrive in the inbox
--test Do not move or delete IMAP messages
-s, --silent Only print errors
-s, --silent Only print errors and warnings
--debug Print debugging information
-v, --version show program's version number and exit
@@ -544,6 +544,12 @@ Om the same system as Elasticsearch, pass ``--save-aggregate`` and/or
privacy reasons. While aggregate DMARC reports are sent at least daily,
it is normal to receive very few forensic reports.
An alternative approach is to still collect forensic/failure/ruf reports
in your DMARC inbox, but run ``parsedmarc --save-forensic`` manually on a
separate IMAP folder (using the ``-r`` option), after you have manually
moved known samples you want to save to that folder (e.g. malicious
samples non-sensitive legitimate samples).
When you first visit Kibana, it will prompt you to create an index pattern.
Start by creating the index pattern ``dmarc_aggregate`` (without an ``*``),
parsedmarc/__init__.py
+57 -46
View File
@@ -125,8 +125,8 @@ def _get_base_domain(domain):
try:
download_psl()
except Exception as error:
logger.warning("Failed to download an updated PSL - \
{0}".format(error))
logger.warning(
"Failed to download an updated PSL {0}".format(error))
with open(psl_path, encoding="utf-8") as psl_file:
psl = publicsuffix.PublicSuffixList(psl_file)
@@ -566,18 +566,18 @@ def parse_aggregate_report_xml(xml, nameservers=None, timeout=2.0):
return new_report
except expat.ExpatError as error:
raise InvalidAggregateReport("Invalid XML: "
"{0}".format(error.__str__()))
raise InvalidAggregateReport(
"Invalid XML: {0}".format(error.__str__()))
except KeyError as error:
raise InvalidAggregateReport("Missing field: "
"{0}".format(error.__str__()))
raise InvalidAggregateReport(
"Missing field: {0}".format(error.__str__()))
except AttributeError:
raise InvalidAggregateReport("Report missing required section")
except Exception as error:
raise InvalidAggregateReport("Unexpected error: "
"{0}".format(error.__str__()))
raise InvalidAggregateReport(
"Unexpected error: {0}".format(error.__str__()))
def extract_xml(input_):
@@ -618,8 +618,8 @@ def extract_xml(input_):
raise InvalidAggregateReport("File objects must be opened in binary "
"(rb) mode")
except Exception as error:
raise InvalidAggregateReport("Invalid archive file: "
"{0}".format(error.__str__()))
raise InvalidAggregateReport(
"Invalid archive file: {0}".format(error.__str__()))
return xml
@@ -918,8 +918,8 @@ def parse_forensic_report(feedback_report, sample, sample_headers_only,
error.__str__()))
except Exception as error:
raise InvalidForensicReport("Unexpected error: "
"{0}".format(error.__str__()))
raise InvalidForensicReport(
"Unexpected error: {0}".format(error.__str__()))
def parsed_forensic_reports_to_csv(reports):
@@ -1262,11 +1262,14 @@ def get_dmarc_reports_from_inbox(host=None,
delete_messages(msg_uids)
if not server.folder_exists(archive_folder):
logger.debug("Creating IMAP folder: {0}".format(archive_folder))
server.create_folder(archive_folder)
try:
# Test subfolder creation
if not server.folder_exists(aggregate_reports_folder):
server.create_folder(aggregate_reports_folder)
logger.debug(
"Creating IMAP folder: {0}".format(archive_folder))
except imapclient.exceptions.IMAPClientError:
# Only replace / with . when . doesn't work
# This usually indicates a dovecot IMAP server
@@ -1274,17 +1277,19 @@ def get_dmarc_reports_from_inbox(host=None,
".")
forensic_reports_folder = forensic_reports_folder.replace("/",
".")
subfolders = [aggregate_reports_folder,
forensic_reports_folder,
invalid_reports_folder]
if not server.folder_exists(aggregate_reports_folder):
server.create_folder(aggregate_reports_folder)
if not server.folder_exists(forensic_reports_folder):
server.create_folder(forensic_reports_folder)
if not server.folder_exists(invalid_reports_folder):
server.create_folder(invalid_reports_folder)
for subfolder in subfolders:
if not server.folder_exists(subfolder):
logger.debug(
"Creating IMAP folder: {0}".format(subfolder))
server.create_folder(subfolder)
server.select_folder(reports_folder)
messages = server.search()
logger.debug("Found {0} messages in IMAP folder "
"{1}".format(len(messages), reports_folder))
logger.debug("Found {0} messages in IMAP folder {1}".format(
len(messages), reports_folder))
for i in range(len(messages)):
number_of_messages = len(messages)
message_uid = messages[i]
@@ -1325,16 +1330,20 @@ def get_dmarc_reports_from_inbox(host=None,
except imapclient.exceptions.IMAPClientError as error:
error = error.__str__().lstrip("b'").rstrip("'").rstrip(".")
error = "IMAP error: Skipping message UID {0}: {1}".format(
message_uid, error
)
message_uid, error)
logger.error("IMAP error: {0}".format(error))
except InvalidDMARCReport as error:
logger.warning(error.__str__())
if not test:
if delete:
logger.debug(
"Deleting message UID {0}".format(message_uid))
delete_messages([message_uid])
else:
move_messages([message_uid], invalid_reports_folder)
logger.debug(
"Moving message UID {0} to {1)".format(
message_uid, invalid_reports_folder))
if not test:
if delete:
@@ -1344,18 +1353,17 @@ def get_dmarc_reports_from_inbox(host=None,
number_of_msgs = len(processed_messages)
for i in range(number_of_msgs):
msg_uid = processed_messages[i]
logger.debug("Deleting message {0} of {1}: "
"UID {2}".format(i + 1,
number_of_msgs,
msg_uid))
logger.debug(
"Deleting message {0} of {1}: UID {2}".format(
i + 1, number_of_msgs, msg_uid))
try:
delete_messages([msg_uid])
except imapclient.exceptions.IMAPClientError as e:
e = e.__str__().lstrip("b'").rstrip(
"'").rstrip(".")
e = "IMAP error: Error deleting message UID {0}: " \
"{1}".format(msg_uid, e)
message = "Error deleting message UID"
e = "{0} {1}: " "{2}".format(message, msg_uid, e)
logger.error("IMAP error: {0}".format(e))
except (ConnectionResetError, TimeoutError) as e:
logger.debug("IMAP error: {0}".format(e.__str__()))
@@ -1369,24 +1377,25 @@ def get_dmarc_reports_from_inbox(host=None,
delete_messages([msg_uid])
else:
if len(aggregate_report_msg_uids) > 0:
logger.debug("Moving aggregate report messages "
"from {0} to "
"{1}".format(reports_folder,
aggregate_reports_folder))
log_message = "Moving aggregate report messages from"
logger.debug(
"{0} {1} to {1}".format(
log_message, reports_folder,
aggregate_reports_folder))
number_of_msgs = len(aggregate_report_msg_uids)
for i in range(number_of_msgs):
msg_uid = aggregate_report_msg_uids[i]
logger.debug("Moving message {0} of {1}: "
"UID {2}".format(i+1, number_of_msgs,
msg_uid))
logger.debug(
"Moving message {0} of {1}: UID {2}".format(
i+1, number_of_msgs, msg_uid))
try:
move_messages([msg_uid],
aggregate_reports_folder)
except imapclient.exceptions.IMAPClientError as e:
e = e.__str__().lstrip("b'").rstrip(
"'").rstrip(".")
e = "Error moving message UID {0}: " \
"{1}".format(msg_uid, e)
message = "Error moving message UID"
e = "{0} {1}: {2}".format(message, msg_uid, e)
logger.error("IMAP error: {0}".format(e))
except (ConnectionResetError, TimeoutError) as error:
logger.debug("IMAP error: {0}".format(
@@ -1402,24 +1411,26 @@ def get_dmarc_reports_from_inbox(host=None,
aggregate_reports_folder)
if len(forensic_report_msg_uids) > 0:
logger.debug("Moving forensic report messages "
"from {0} to "
"{1}".format(reports_folder,
forensic_reports_folder))
message = "Moving forensic report messages from"
logger.debug(
"{0} {1} to {2}".format(message,
reports_folder,
forensic_reports_folder))
number_of_msgs = len(forensic_report_msg_uids)
for i in range(number_of_msgs):
msg_uid = forensic_report_msg_uids[i]
logger.debug("Moving message {0} of {1}: "
"UID {2}".format(i + 1, number_of_msgs,
msg_uid))
message = "Moving message"
logger.debug("{0} {1} of {2}: UID {2}".format(
message,
i + 1, number_of_msgs, msg_uid))
try:
move_messages([msg_uid],
forensic_reports_folder)
except imapclient.exceptions.IMAPClientError as e:
e = e.__str__().lstrip("b'").rstrip(
"'").rstrip(".")
e = "Error moving message UID {0}: " \
"{1}".format(msg_uid, e)
e = "Error moving message UID {0}: {1}".format(
msg_uid, e)
logger.error("IMAP Error: {0}".format(e))
except (ConnectionResetError, TimeoutError) as error:
logger.debug("IMAP error: {0}".format(