diff --git a/CHANGELOG.md b/CHANGELOG.md index f5131d7..c218d41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,14 @@ +6.3.0 +----- + +- Fix IMAP IDLE response processing for some mail servers (#67) +- Exit with a critical error when options are missing (#68) +- Add IMAP responses to debug logging +- Add `smtp` option `skip_certificate_verification` +- Add `kafka` option `skip_certificate_verification` +- Suppress `mailparser` logging output +- Suppress `msgconvert` warnings + 6.2.2 ----- diff --git a/README.rst b/README.rst index d29240a..083bf3a 100644 --- a/README.rst +++ b/README.rst @@ -171,12 +171,14 @@ The full set of configuration options are: - ``user`` - str: The Kafka user - ``passsword`` - str: The Kafka password - ``ssl`` - bool: Use an encrypted SSL/TLS connection (Default: True) + - ``skip_certificate_verification`` - bool: Skip certificate verification (not recommended) - ``aggregate_topic`` - str: The Kafka topic for aggregate reports - ``forensic_topic`` - str: The Kafka topic for forensic reports - ``smtp`` - ``host`` - str: The SMTP hostname - ``port`` - int: The SMTP port (Default: 25) - ``ssl`` - bool: Require SSL/TLS instead of using STARTTLS + - ``skip_certificate_verification`` - bool: Skip certificate verification (not recommended) - ``user`` - str: the SMTP username - ``password`` - str: the SMTP password - ``from`` - str: The From header to use in the email diff --git a/docs/index.rst b/docs/index.rst index a4785ce..b7ab9ce 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -175,12 +175,14 @@ The full set of configuration options are: - ``user`` - str: The Kafka user - ``passsword`` - str: The Kafka password - ``ssl`` - bool: Use an encrypted SSL/TLS connection (Default: True) + - ``skip_certificate_verification`` - bool: Skip certificate verification (not recommended) - ``aggregate_topic`` - str: The Kafka topic for aggregate reports - ``forensic_topic`` - str: The Kafka topic for forensic reports - ``smtp`` - ``host`` - str: The SMTP hostname - ``port`` - int: The SMTP port (Default: 25) - ``ssl`` - bool: Require SSL/TLS instead of using STARTTLS + - ``skip_certificate_verification`` - bool: Skip certificate verification (not recommended) - ``user`` - str: the SMTP username - ``password`` - str: the SMTP password - ``from`` - str: The From header to use in the email diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index 941ccad..b6cfd46 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -38,7 +38,7 @@ from parsedmarc.utils import is_outlook_msg, convert_outlook_msg from parsedmarc.utils import timestamp_to_human, human_timestamp_to_datetime from parsedmarc.utils import parse_email -__version__ = "6.2.2" +__version__ = "6.3.0" logging.basicConfig( format='%(levelname)8s:%(filename)s:%(lineno)d:' diff --git a/parsedmarc/cli.py b/parsedmarc/cli.py index 93f6720..eb3fe3c 100644 --- a/parsedmarc/cli.py +++ b/parsedmarc/cli.py @@ -39,12 +39,12 @@ def cli_parse(file_path, sa, nameservers, dns_timeout, parallel=False): strip_attachment_payloads=sa, parallel=parallel) except ParserError as error: - return (error, file_path) + return error, file_path finally: global counter with counter.get_lock(): counter.value += 1 - return (file_results, file_path) + return file_results, file_path def init(ctr): @@ -62,10 +62,17 @@ def _main(): print(output_str) if opts.kafka_hosts: try: + ssl_context = None + if opts.kafka_skip_certificate_verification: + logger.debug("Skipping IMAP certificate verification") + ssl_context = create_default_context() + ssl_context.check_hostname = False + ssl_context.verify_mode = CERT_NONE kafka_client = kafkaclient.KafkaClient( opts.kafka_hosts, username=opts.kafka_username, - password=opts.kafka_password + password=opts.kafka_password, + ssl_context=ssl_context ) except Exception as error_: logger.error("Kafka Error: {0}".format(error_.__str__())) @@ -198,9 +205,11 @@ def _main(): kafka_aggregate_topic=None, kafka_forensic_topic=None, kafka_ssl=False, + kafka_skip_certificate_verification=False, smtp_host=None, smtp_port=25, smtp_ssl=False, + smtp_skip_certificate_verification=False, smtp_user=None, smtp_password=None, smtp_from=None, @@ -250,6 +259,10 @@ def _main(): imap_config = config["imap"] if "host" in imap_config: opts.imap_host = imap_config["host"] + else: + logger.error("host setting missing from the " + "imap config section") + exit(-1) if "port" in imap_config: opts.imap_port = imap_config["port"] if "ssl" in imap_config: @@ -260,8 +273,17 @@ def _main(): opts.imap_skip_certificate_verification = imap_verify if "user" in imap_config: opts.imap_user = imap_config["user"] + else: + logger.critical("user setting missing from the " + "imap config section") + exit(-1) if "password" in imap_config: opts.imap_password = imap_config["password"] + else: + logger.critical("password setting missing from the " + "imap config section") + exit(-1) + if "reports_folder" in imap_config: opts.imap_reports_folder = imap_config["reports_folder"] if "archive_folder" in imap_config: @@ -277,6 +299,10 @@ def _main(): if "hosts" in elasticsearch_config: opts.elasticsearch_hosts = _str_to_list(elasticsearch_config[ "hosts"]) + else: + logger.critical("hosts setting missing from the " + "elasticsearch config section") + exit(-1) if "index_suffix" in elasticsearch_config: opts.elasticsearch_index_suffix = elasticsearch_config[ "index_suffix"] @@ -293,10 +319,22 @@ def _main(): hec_config = config["splunk_hec"] if "url" in hec_config: opts.hec = hec_config["url"] + else: + logger.critical("url setting missing from the " + "splunk_hec config section") + exit(-1) if "token" in hec_config: opts.hec_token = hec_config["token"] + else: + logger.critical("token setting missing from the " + "splunk_hec config section") + exit(-1) if "index" in hec_config: opts.hec_index = hec_config["index"] + else: + logger.critical("index setting missing from the " + "splunk_hec config section") + exit(-1) if "skip_certificate_verification" in hec_config: opts.hec_skip_certificate_verification = hec_config[ "skip_certificate_verification"] @@ -304,32 +342,77 @@ def _main(): kafka_config = config["kafka"] if "hosts" in kafka_config: opts.kafka_hosts = _str_to_list(kafka_config["hosts"]) + else: + logger.critical("hosts setting missing from the " + "kafka config section") + exit(-1) if "user" in kafka_config: opts.kafka_username = kafka_config["user"] + else: + logger.critical("user setting missing from the " + "kafka config section") + exit(-1) if "password" in kafka_config: opts.kafka_password = kafka_config["password"] + else: + logger.critical("password setting missing from the " + "kafka config section") + exit(-1) if "ssl" in kafka_config: opts.kafka_ssl = kafka_config["ssl"].getboolean() + if "skip_certificate_verification" in kafka_config: + kafka_verify = kafka_config.getboolean( + "skip_certificate_verification") + opts.kafka_skip_certificate_verification = kafka_verify if "aggregate_topic" in kafka_config: opts.kafka_aggregate = kafka_config["aggregate_topic"] + else: + logger.critical("aggregate_topic setting missing from the " + "kafka config section") + exit(-1) if "forensic_topic" in kafka_config: opts.kafka_username = kafka_config["forensic_topic"] + else: + logger.critical("forensic_topic setting missing from the " + "splunk_hec config section") if "smtp" in config.sections(): smtp_config = config["smtp"] if "host" in smtp_config: opts.smtp_host = smtp_config["host"] + else: + logger.critical("host setting missing from the " + "smtp config section") + exit(-1) if "port" in smtp_config: opts.smtp_port = smtp_config["port"] if "ssl" in smtp_config: opts.smtp_ssl = smtp_config.getboolean("ssl") + if "skip_certificate_verification" in smtp_config: + smtp_verify = smtp_config.getboolean( + "skip_certificate_verification") + opts.smtp_skip_certificate_verification = smtp_verify if "user" in smtp_config: opts.smtp_user = smtp_config["user"] + else: + logger.critical("user setting missing from the " + "smtp config section") + exit(-1) if "password" in smtp_config: opts.smtp_password = smtp_config["password"] + else: + logger.critical("password setting missing from the " + "smtp config section") + exit(-1) if "from" in smtp_config: opts.smtp_from = smtp_config["from"] + else: + logger.critical("from setting missing from the " + "smtp config section") if "to" in smtp_config: opts.smtp_to = _str_to_list(smtp_config["to"]) + else: + logger.critical("to setting missing from the " + "smtp config section") if "subject" in smtp_config: opts.smtp_subject = smtp_config["subject"] if "attachment" in smtp_config: @@ -472,16 +555,19 @@ def _main(): process_reports(results) if opts.smtp_host: - if opts.smtp_from is None or opts.smtp_to is None: - logger.error("Missing mail from and/or mail to") - exit(1) - try: + ssl_context = None + if opts.smtp_skip_certificate_verification: + logger.debug("Skipping SMTP certificate verification") + ssl_context = create_default_context() + ssl_context.check_hostname = False + ssl_context.verify_mode = CERT_NONE email_results(results, opts.smtp_host, opts.smtp_from, opts.smtp_to, ssl=opts.smtp_ssl, user=opts.smtp_user, password=opts.smtp_password, - subject=opts.smtp_subject) + subject=opts.smtp_subject, + ssl_context=ssl_context) except SMTPError as error: logger.error("SMTP Error: {0}".format(error.__str__())) exit(1) diff --git a/parsedmarc/kafkaclient.py b/parsedmarc/kafkaclient.py index b76157b..6204631 100644 --- a/parsedmarc/kafkaclient.py +++ b/parsedmarc/kafkaclient.py @@ -20,7 +20,7 @@ class KafkaError(RuntimeError): class KafkaClient(object): def __init__(self, kafka_hosts, ssl=False, username=None, - password=None): + password=None, ssl_context=None): """ Initializes the Kafka client Args: @@ -29,6 +29,7 @@ class KafkaClient(object): ssl (bool): Use a SSL/TLS connection username (str): An optional username password (str): An optional password + ssl_context: SSL context options Notes: ``use_ssl=True`` is implied when a username or password are @@ -44,7 +45,7 @@ class KafkaClient(object): client_id="parsedmarc-{0}".format(__version__)) if ssl or username or password: config["security_protocol"] = "SSL" - config["ssl_context"] = create_default_context() + config["ssl_context"] = ssl_context or create_default_context() if username or password: config["sasl_plain_username"] = username or "" config["sasl_plain_password"] = password or "" diff --git a/parsedmarc/utils.py b/parsedmarc/utils.py index c4b1a46..a4dfb54 100644 --- a/parsedmarc/utils.py +++ b/parsedmarc/utils.py @@ -31,8 +31,10 @@ USER_AGENT = "Mozilla/5.0 ((0 {1})) parsedmarc".format( platform.release(), ) - +null_file = open(os.devnull, "w") logger = logging.getLogger("parsedmarc") +mailparser_logger = logging.getLogger("mailparser") +mailparser_logger.setLevel(logging.CRITICAL) tempdir = tempfile.mkdtemp() @@ -441,7 +443,8 @@ def convert_outlook_msg(msg_bytes): with open("sample.msg", "wb") as msg_file: msg_file.write(msg_bytes) try: - subprocess.check_call(["msgconvert", "sample.msg"]) + subprocess.check_call(["msgconvert", "sample.msg"], + stdout=null_file, stderr=null_file) eml_path = "sample.eml" with open(eml_path, "rb") as eml_file: rfc822 = eml_file.read() diff --git a/setup.py b/setup.py index 04ef204..f3ad16c 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ from setuptools import setup from codecs import open from os import path -__version__ = "6.2.2" +__version__ = "6.3.0" description = "A Python package and CLI for parsing aggregate and " \ "forensic DMARC reports"