amorfo77/parsedmarc
1
0
Fork 0
mirror of https://github.com/domainaware/parsedmarc.git synced 2026-04-04 04:38:53 +00:00
Code Issues Projects Releases Wiki Activity

6.3.0

Browse Source
This commit is contained in:
Sean Whalen
2019-03-29 12:44:35 -04:00
parent 2f44d8fe3d
commit a7814d1bf7
8 changed files with 119 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
CHANGELOG.md
View File

@@ -1,3 +1,14 @@
6.3.0
-----
- Fix IMAP IDLE response processing for some mail servers (#67)
- Exit with a critical error when options are missing (#68)
- Add IMAP responses to debug logging
- Add `smtp` option `skip_certificate_verification`
- Add `kafka` option `skip_certificate_verification`
- Suppress `mailparser` logging output
- Suppress `msgconvert` warnings
6.2.2
-----

2
README.rst
View File

@@ -171,12 +171,14 @@ The full set of configuration options are:
- ``user`` - str: The Kafka user
- ``passsword`` - str: The Kafka password
- ``ssl`` - bool: Use an encrypted SSL/TLS connection (Default: True)
- ``skip_certificate_verification`` - bool: Skip certificate verification (not recommended)
- ``aggregate_topic`` - str: The Kafka topic for aggregate reports
- ``forensic_topic`` - str: The Kafka topic for forensic reports
- ``smtp``
- ``host`` - str: The SMTP hostname
- ``port`` - int: The SMTP port (Default: 25)
- ``ssl`` - bool: Require SSL/TLS instead of using STARTTLS
- ``skip_certificate_verification`` - bool: Skip certificate verification (not recommended)
- ``user`` - str: the SMTP username
- ``password`` - str: the SMTP password
- ``from`` - str: The From header to use in the email

2
docs/index.rst
View File

@@ -175,12 +175,14 @@ The full set of configuration options are:
- ``user`` - str: The Kafka user
- ``passsword`` - str: The Kafka password
- ``ssl`` - bool: Use an encrypted SSL/TLS connection (Default: True)
- ``skip_certificate_verification`` - bool: Skip certificate verification (not recommended)
- ``aggregate_topic`` - str: The Kafka topic for aggregate reports
- ``forensic_topic`` - str: The Kafka topic for forensic reports
- ``smtp``
- ``host`` - str: The SMTP hostname
- ``port`` - int: The SMTP port (Default: 25)
- ``ssl`` - bool: Require SSL/TLS instead of using STARTTLS
- ``skip_certificate_verification`` - bool: Skip certificate verification (not recommended)
- ``user`` - str: the SMTP username
- ``password`` - str: the SMTP password
- ``from`` - str: The From header to use in the email

2
parsedmarc/__init__.py
View File

@@ -38,7 +38,7 @@ from parsedmarc.utils import is_outlook_msg, convert_outlook_msg
from parsedmarc.utils import timestamp_to_human, human_timestamp_to_datetime
from parsedmarc.utils import parse_email
__version__ = "6.2.2"
__version__ = "6.3.0"
logging.basicConfig(
format='%(levelname)8s:%(filename)s:%(lineno)d:'

102
parsedmarc/cli.py
View File

@@ -39,12 +39,12 @@ def cli_parse(file_path, sa, nameservers, dns_timeout, parallel=False):
strip_attachment_payloads=sa,
parallel=parallel)
except ParserError as error:
return (error, file_path)
return error, file_path
finally:
global counter
with counter.get_lock():
counter.value += 1
return (file_results, file_path)
return file_results, file_path
def init(ctr):
@@ -62,10 +62,17 @@ def _main():
print(output_str)
if opts.kafka_hosts:
try:
ssl_context = None
if opts.kafka_skip_certificate_verification:
logger.debug("Skipping IMAP certificate verification")
ssl_context = create_default_context()
ssl_context.check_hostname = False
ssl_context.verify_mode = CERT_NONE
kafka_client = kafkaclient.KafkaClient(
opts.kafka_hosts,
username=opts.kafka_username,
password=opts.kafka_password
password=opts.kafka_password,
ssl_context=ssl_context
)
except Exception as error_:
logger.error("Kafka Error: {0}".format(error_.__str__()))
@@ -198,9 +205,11 @@ def _main():
kafka_aggregate_topic=None,
kafka_forensic_topic=None,
kafka_ssl=False,
kafka_skip_certificate_verification=False,
smtp_host=None,
smtp_port=25,
smtp_ssl=False,
smtp_skip_certificate_verification=False,
smtp_user=None,
smtp_password=None,
smtp_from=None,
@@ -250,6 +259,10 @@ def _main():
imap_config = config["imap"]
if "host" in imap_config:
opts.imap_host = imap_config["host"]
else:
logger.error("host setting missing from the "
"imap config section")
exit(-1)
if "port" in imap_config:
opts.imap_port = imap_config["port"]
if "ssl" in imap_config:
@@ -260,8 +273,17 @@ def _main():
opts.imap_skip_certificate_verification = imap_verify
if "user" in imap_config:
opts.imap_user = imap_config["user"]
else:
logger.critical("user setting missing from the "
"imap config section")
exit(-1)
if "password" in imap_config:
opts.imap_password = imap_config["password"]
else:
logger.critical("password setting missing from the "
"imap config section")
exit(-1)
if "reports_folder" in imap_config:
opts.imap_reports_folder = imap_config["reports_folder"]
if "archive_folder" in imap_config:
@@ -277,6 +299,10 @@ def _main():
if "hosts" in elasticsearch_config:
opts.elasticsearch_hosts = _str_to_list(elasticsearch_config[
"hosts"])
else:
logger.critical("hosts setting missing from the "
"elasticsearch config section")
exit(-1)
if "index_suffix" in elasticsearch_config:
opts.elasticsearch_index_suffix = elasticsearch_config[
"index_suffix"]
@@ -293,10 +319,22 @@ def _main():
hec_config = config["splunk_hec"]
if "url" in hec_config:
opts.hec = hec_config["url"]
else:
logger.critical("url setting missing from the "
"splunk_hec config section")
exit(-1)
if "token" in hec_config:
opts.hec_token = hec_config["token"]
else:
logger.critical("token setting missing from the "
"splunk_hec config section")
exit(-1)
if "index" in hec_config:
opts.hec_index = hec_config["index"]
else:
logger.critical("index setting missing from the "
"splunk_hec config section")
exit(-1)
if "skip_certificate_verification" in hec_config:
opts.hec_skip_certificate_verification = hec_config[
"skip_certificate_verification"]
@@ -304,32 +342,77 @@ def _main():
kafka_config = config["kafka"]
if "hosts" in kafka_config:
opts.kafka_hosts = _str_to_list(kafka_config["hosts"])
else:
logger.critical("hosts setting missing from the "
"kafka config section")
exit(-1)
if "user" in kafka_config:
opts.kafka_username = kafka_config["user"]
else:
logger.critical("user setting missing from the "
"kafka config section")
exit(-1)
if "password" in kafka_config:
opts.kafka_password = kafka_config["password"]
else:
logger.critical("password setting missing from the "
"kafka config section")
exit(-1)
if "ssl" in kafka_config:
opts.kafka_ssl = kafka_config["ssl"].getboolean()
if "skip_certificate_verification" in kafka_config:
kafka_verify = kafka_config.getboolean(
"skip_certificate_verification")
opts.kafka_skip_certificate_verification = kafka_verify
if "aggregate_topic" in kafka_config:
opts.kafka_aggregate = kafka_config["aggregate_topic"]
else:
logger.critical("aggregate_topic setting missing from the "
"kafka config section")
exit(-1)
if "forensic_topic" in kafka_config:
opts.kafka_username = kafka_config["forensic_topic"]
else:
logger.critical("forensic_topic setting missing from the "
"splunk_hec config section")
if "smtp" in config.sections():
smtp_config = config["smtp"]
if "host" in smtp_config:
opts.smtp_host = smtp_config["host"]
else:
logger.critical("host setting missing from the "
"smtp config section")
exit(-1)
if "port" in smtp_config:
opts.smtp_port = smtp_config["port"]
if "ssl" in smtp_config:
opts.smtp_ssl = smtp_config.getboolean("ssl")
if "skip_certificate_verification" in smtp_config:
smtp_verify = smtp_config.getboolean(
"skip_certificate_verification")
opts.smtp_skip_certificate_verification = smtp_verify
if "user" in smtp_config:
opts.smtp_user = smtp_config["user"]
else:
logger.critical("user setting missing from the "
"smtp config section")
exit(-1)
if "password" in smtp_config:
opts.smtp_password = smtp_config["password"]
else:
logger.critical("password setting missing from the "
"smtp config section")
exit(-1)
if "from" in smtp_config:
opts.smtp_from = smtp_config["from"]
else:
logger.critical("from setting missing from the "
"smtp config section")
if "to" in smtp_config:
opts.smtp_to = _str_to_list(smtp_config["to"])
else:
logger.critical("to setting missing from the "
"smtp config section")
if "subject" in smtp_config:
opts.smtp_subject = smtp_config["subject"]
if "attachment" in smtp_config:
@@ -472,16 +555,19 @@ def _main():
process_reports(results)
if opts.smtp_host:
if opts.smtp_from is None or opts.smtp_to is None:
logger.error("Missing mail from and/or mail to")
exit(1)
try:
ssl_context = None
if opts.smtp_skip_certificate_verification:
logger.debug("Skipping SMTP certificate verification")
ssl_context = create_default_context()
ssl_context.check_hostname = False
ssl_context.verify_mode = CERT_NONE
email_results(results, opts.smtp_host, opts.smtp_from,
opts.smtp_to, ssl=opts.smtp_ssl,
user=opts.smtp_user,
password=opts.smtp_password,
subject=opts.smtp_subject)
subject=opts.smtp_subject,
ssl_context=ssl_context)
except SMTPError as error:
logger.error("SMTP Error: {0}".format(error.__str__()))
exit(1)

5
parsedmarc/kafkaclient.py
View File

@@ -20,7 +20,7 @@ class KafkaError(RuntimeError):
class KafkaClient(object):
def __init__(self, kafka_hosts, ssl=False, username=None,
password=None):
password=None, ssl_context=None):
"""
Initializes the Kafka client
Args:
@@ -29,6 +29,7 @@ class KafkaClient(object):
ssl (bool): Use a SSL/TLS connection
username (str): An optional username
password (str): An optional password
ssl_context: SSL context options
Notes:
``use_ssl=True`` is implied when a username or password are
@@ -44,7 +45,7 @@ class KafkaClient(object):
client_id="parsedmarc-{0}".format(__version__))
if ssl or username or password:
config["security_protocol"] = "SSL"
config["ssl_context"] = create_default_context()
config["ssl_context"] = ssl_context or create_default_context()
if username or password:
config["sasl_plain_username"] = username or ""
config["sasl_plain_password"] = password or ""

7
parsedmarc/utils.py
View File

@@ -31,8 +31,10 @@ USER_AGENT = "Mozilla/5.0 ((0 {1})) parsedmarc".format(
platform.release(),
)
null_file = open(os.devnull, "w")
logger = logging.getLogger("parsedmarc")
mailparser_logger = logging.getLogger("mailparser")
mailparser_logger.setLevel(logging.CRITICAL)
tempdir = tempfile.mkdtemp()
@@ -441,7 +443,8 @@ def convert_outlook_msg(msg_bytes):
with open("sample.msg", "wb") as msg_file:
msg_file.write(msg_bytes)
try:
subprocess.check_call(["msgconvert", "sample.msg"])
subprocess.check_call(["msgconvert", "sample.msg"],
stdout=null_file, stderr=null_file)
eml_path = "sample.eml"
with open(eml_path, "rb") as eml_file:
rfc822 = eml_file.read()

2
setup.py
View File

@@ -14,7 +14,7 @@ from setuptools import setup
from codecs import open
from os import path
__version__ = "6.2.2"
__version__ = "6.3.0"
description = "A Python package and CLI for parsing aggregate and " \
"forensic DMARC reports"