From 253d421e295bbc43ba2e1819a1c5436d66d86728 Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Tue, 25 Sep 2018 13:40:55 -0400 Subject: [PATCH] Splunk and SMTP improvements SMTP issue #12 fixed (based on PR #13 ) --- CHANGELOG.md | 5 +- README.rst | 10 +++- docs/index.rst | 121 ++++++++++++++++++++++------------------- parsedmarc/__init__.py | 15 +++-- parsedmarc/cli.py | 9 ++- 5 files changed, 92 insertions(+), 68 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e094c27..58b21e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,9 @@ GeoIP DB to avoid being blocked by security proxies - Add alignment booleans to JSON output - Fix `.msg` parsing CLI exception when `msgconvert` is not found in the system path -- Fix `SMTP AUTH extension not supported by server` error on some SMTP servers -(#12) +- Add `--outgoing-port` and `--outgoing-ssl` options +- Fall back to plain text SMTP if `--outgoing-ssl` is not used and `STARTTLS` +is not supported by the server - Always use `\n` as the newline when generating CSVs 3.9.7 diff --git a/README.rst b/README.rst index 1910f7c..49411c5 100644 --- a/README.rst +++ b/README.rst @@ -38,13 +38,14 @@ CLI help :: - usage: cli.py [-h] [-o OUTPUT] [-n NAMESERVERS [NAMESERVERS ...]] [-t TIMEOUT] + usage: cli.py [-h] [-o OUTPUT] [-n NAMESERVERS [NAMESERVERS ...]] [-t TIMEOUT] [-H HOST] [-u USER] [-p PASSWORD] [-r REPORTS_FOLDER] [-a ARCHIVE_FOLDER] [-d] [-E [ELASTICSEARCH_HOST [ELASTICSEARCH_HOST ...]]] [--hec HEC] [--hec-key HEC_KEY] [--hec-index HEC_INDEX] [--save-aggregate] [--save-forensic] [-O OUTGOING_HOST] [-U OUTGOING_USER] - [-P OUTGOING_PASSWORD] [-F OUTGOING_FROM] + [-P OUTGOING_PASSWORD] [--outgoing-port OUTGOING_PORT] + [--outgoing-SSL OUTGOING_SSL] [-F OUTGOING_FROM] [-T OUTGOING_TO [OUTGOING_TO ...]] [-S OUTGOING_SUBJECT] [-A OUTGOING_ATTACHMENT] [-M OUTGOING_MESSAGE] [-w] [--test] [-s] [--debug] [-v] @@ -92,6 +93,11 @@ CLI help Email the results using this user -P OUTGOING_PASSWORD, --outgoing-password OUTGOING_PASSWORD Email the results using this password + --outgoing-port OUTGOING_PORT + Email the results using this port + --outgoing-SSL OUTGOING_SSL + Use SSL/TLS instead of STARTTLS (more secure, and + required by some providers, like Gmail) -F OUTGOING_FROM, --outgoing-from OUTGOING_FROM Email the results using this from address -T OUTGOING_TO [OUTGOING_TO ...], --outgoing-to OUTGOING_TO [OUTGOING_TO ...] diff --git a/docs/index.rst b/docs/index.rst index 7ce562c..177e03f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -51,70 +51,77 @@ CLI help [-E [ELASTICSEARCH_HOST [ELASTICSEARCH_HOST ...]]] [--hec HEC] [--hec-key HEC_KEY] [--hec-index HEC_INDEX] [--save-aggregate] [--save-forensic] [-O OUTGOING_HOST] [-U OUTGOING_USER] - [-P OUTGOING_PASSWORD] [-F OUTGOING_FROM] + [-P OUTGOING_PASSWORD] [--outgoing-port OUTGOING_PORT] + [--outgoing-SSL OUTGOING_SSL] [-F OUTGOING_FROM] [-T OUTGOING_TO [OUTGOING_TO ...]] [-S OUTGOING_SUBJECT] [-A OUTGOING_ATTACHMENT] [-M OUTGOING_MESSAGE] [-w] [--test] [-s] [--debug] [-v] [file_path [file_path ...]] - Parses DMARC reports + Parses DMARC reports - positional arguments: - file_path one or more paths to aggregate or forensic report - files or emails + positional arguments: + file_path one or more paths to aggregate or forensic report + files or emails + + optional arguments: + -h, --help show this help message and exit + -o OUTPUT, --output OUTPUT + Write output files to the given directory + -n NAMESERVERS [NAMESERVERS ...], --nameservers NAMESERVERS [NAMESERVERS ...] + nameservers to query (Default is Cloudflare's) + -t TIMEOUT, --timeout TIMEOUT + number of seconds to wait for an answer from DNS + (default 2.0) + -H HOST, --host HOST IMAP hostname or IP address + -u USER, --user USER IMAP user + -p PASSWORD, --password PASSWORD + IMAP password + -r REPORTS_FOLDER, --reports-folder REPORTS_FOLDER + The IMAP folder containing the reports Default: INBOX + -a ARCHIVE_FOLDER, --archive-folder ARCHIVE_FOLDER + Specifies the IMAP folder to move messages to after + processing them Default: Archive + -d, --delete Delete the reports after processing them + -E [ELASTICSEARCH_HOST [ELASTICSEARCH_HOST ...]], --elasticsearch-host [ELASTICSEARCH_HOST [ELASTICSEARCH_HOST ...]] + A list of one or more Elasticsearch hostnames or URLs + to use (e.g. localhost:9200) + --hec HEC URL to a Splunk HTTP Event Collector (HEC) + --hec-key HEC_KEY The authorization key for a Splunk HTTP event + collector (HEC) + --hec-index HEC_INDEX + The index to use when sending events to the Splunk + HTTP Events + --save-aggregate Save aggregate reports to search indexes + --save-forensic Save forensic reports to search indexes + -O OUTGOING_HOST, --outgoing-host OUTGOING_HOST + Email the results using this host + -U OUTGOING_USER, --outgoing-user OUTGOING_USER + Email the results using this user + -P OUTGOING_PASSWORD, --outgoing-password OUTGOING_PASSWORD + Email the results using this password + --outgoing-port OUTGOING_PORT + Email the results using this port + --outgoing-SSL OUTGOING_SSL + Use SSL/TLS instead of STARTTLS (more secure, and + required by some providers, like Gmail) + -F OUTGOING_FROM, --outgoing-from OUTGOING_FROM + Email the results using this from address + -T OUTGOING_TO [OUTGOING_TO ...], --outgoing-to OUTGOING_TO [OUTGOING_TO ...] + Email the results to these addresses + -S OUTGOING_SUBJECT, --outgoing-subject OUTGOING_SUBJECT + Email the results using this subject + -A OUTGOING_ATTACHMENT, --outgoing-attachment OUTGOING_ATTACHMENT + Email the results using this filename + -M OUTGOING_MESSAGE, --outgoing-message OUTGOING_MESSAGE + Email the results using this message + -w, --watch Use an IMAP IDLE connection to process reports as they + arrive in the inbox + --test Do not move or delete IMAP messages + -s, --silent Only print errors + --debug Print debugging information + -v, --version show program's version number and exit - optional arguments: - -h, --help show this help message and exit - -o OUTPUT, --output OUTPUT - Write output files to the given directory - -n NAMESERVERS [NAMESERVERS ...], --nameservers NAMESERVERS [NAMESERVERS ...] - nameservers to query (Default is Cloudflare's) - -t TIMEOUT, --timeout TIMEOUT - number of seconds to wait for an answer from DNS - (default 2.0) - -H HOST, --host HOST IMAP hostname or IP address - -u USER, --user USER IMAP user - -p PASSWORD, --password PASSWORD - IMAP password - -r REPORTS_FOLDER, --reports-folder REPORTS_FOLDER - The IMAP folder containing the reports Default: INBOX - -a ARCHIVE_FOLDER, --archive-folder ARCHIVE_FOLDER - Specifies the IMAP folder to move messages to after - processing them Default: Archive - -d, --delete Delete the reports after processing them - -E [ELASTICSEARCH_HOST [ELASTICSEARCH_HOST ...]], --elasticsearch-host [ELASTICSEARCH_HOST [ELASTICSEARCH_HOST ...]] - A list of one or more Elasticsearch hostnames or URLs - to use (e.g. localhost:9200) - --hec HEC URL to a Splunk HTTP Event Collector (HEC) - --hec-key HEC_KEY The authorization key for a Splunk HTTP event - collector (HEC) - --hec-index HEC_INDEX - The index to use when sending events to the Splunk - HTTP Events - --save-aggregate Save aggregate reports to search indexes - --save-forensic Save forensic reports to search indexes - -O OUTGOING_HOST, --outgoing-host OUTGOING_HOST - Email the results using this host - -U OUTGOING_USER, --outgoing-user OUTGOING_USER - Email the results using this user - -P OUTGOING_PASSWORD, --outgoing-password OUTGOING_PASSWORD - Email the results using this password - -F OUTGOING_FROM, --outgoing-from OUTGOING_FROM - Email the results using this from address - -T OUTGOING_TO [OUTGOING_TO ...], --outgoing-to OUTGOING_TO [OUTGOING_TO ...] - Email the results to these addresses - -S OUTGOING_SUBJECT, --outgoing-subject OUTGOING_SUBJECT - Email the results using this subject - -A OUTGOING_ATTACHMENT, --outgoing-attachment OUTGOING_ATTACHMENT - Email the results using this filename - -M OUTGOING_MESSAGE, --outgoing-message OUTGOING_MESSAGE - Email the results using this message - -w, --watch Use an IMAP IDLE connection to process reports as they - arrive in the inbox - --test Do not move or delete IMAP messages - -s, --silent Only print errors - --debug Print debugging information - -v, --version show program's version number and exit SPF and DMARC record validation =============================== diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index 4523dcb..4bbcd29 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -237,6 +237,7 @@ def human_timestamp_to_timestamp(human_timestamp): """ return human_timestamp_to_datetime(human_timestamp).timestamp() + def _get_ip_address_country(ip_address): """ Uses the MaxMind Geolite2 Country database to return the ISO code for the @@ -1399,7 +1400,7 @@ def get_report_zip(results): return storage.getvalue() -def email_results(results, host, mail_from, mail_to, port=587, starttls=True, +def email_results(results, host, mail_from, mail_to, port=0, use_ssl=False, user=None, password=None, subject=None, attachment_filename=None, message=None, ssl_context=None): """ @@ -1411,7 +1412,6 @@ def email_results(results, host, mail_from, mail_to, port=587, starttls=True, mail_from: The value of the message from header mail_to : A list of addresses to mail to port (int): Port to use - starttls (bool): use STARTTLS use_ssl (bool): Require a SSL connection from the start user: An optional username password: An optional password @@ -1451,14 +1451,17 @@ def email_results(results, host, mail_from, mail_to, port=587, starttls=True, if use_ssl: server = smtplib.SMTP_SSL(host, port=port, context=ssl_context) server.connect(host, port) - server.helo() + server.ehlo_or_helo_if_needed() else: server = smtplib.SMTP(host, port=port) server.connect(host, port) - server.ehlo() - if starttls: + server.ehlo_or_helo_if_needed() + if server.has_extn("starttls"): server.starttls(context=ssl_context) - server.helo() + server.ehlo() + else: + logger.warning("SMTP server does not support STARTTLS. " + "Proceeding in plain text!") if user and password: server.login(user, password) server.sendmail(mail_from, mail_to, msg.as_string()) diff --git a/parsedmarc/cli.py b/parsedmarc/cli.py index f754bec..cb4cbf4 100644 --- a/parsedmarc/cli.py +++ b/parsedmarc/cli.py @@ -114,6 +114,12 @@ def _main(): help="Email the results using this user") arg_parser.add_argument("-P", "--outgoing-password", help="Email the results using this password") + arg_parser.add_argument("--outgoing-port", + help="Email the results using this port") + arg_parser.add_argument("--outgoing-SSL", + help="Use SSL/TLS instead of STARTTLS (more " + "secure, and required by some providers, " + "like Gmail)") arg_parser.add_argument("-F", "--outgoing-from", help="Email the results using this from address") arg_parser.add_argument("-T", "--outgoing-to", nargs="+", @@ -234,7 +240,8 @@ def _main(): try: email_results(results, args.outgoing_host, args.outgoing_from, - args.outgoing_to, user=args.outgoing_user, + args.outgoing_to, use_ssl=args.outgoing_ssl, + user=args.outgoing_user, password=args.outgoing_password, subject=args.outgoing_subject) except SMTPError as error: