From 51fd81a918b40d110363d83fc98f96a1b288e4a5 Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Mon, 19 Feb 2024 19:23:01 -0500 Subject: [PATCH] Update docs --- _modules/index.html | 12 +- _modules/parsedmarc.html | 387 ++++++++++++++++++++++++++----- _modules/parsedmarc/elastic.html | 207 ++++++++++++++++- _modules/parsedmarc/splunk.html | 55 ++++- _modules/parsedmarc/utils.html | 14 +- _sources/elasticsearch.md.txt | 2 +- _sources/output.md.txt | 43 ++++ _sources/usage.md.txt | 12 +- _static/documentation_options.js | 2 +- api.html | 110 +++++++-- contributing.html | 12 +- davmail.html | 12 +- dmarc.html | 12 +- elasticsearch.html | 14 +- genindex.html | 36 ++- index.html | 12 +- installation.html | 12 +- kibana.html | 12 +- mailing-lists.html | 12 +- objects.inv | Bin 973 -> 1016 bytes output.html | 56 ++++- py-modindex.html | 12 +- search.html | 12 +- searchindex.js | 2 +- splunk.html | 12 +- usage.html | 27 ++- 26 files changed, 912 insertions(+), 187 deletions(-) diff --git a/_modules/index.html b/_modules/index.html index 1b8f2a5..065e334 100644 --- a/_modules/index.html +++ b/_modules/index.html @@ -1,11 +1,13 @@ - + - Overview: module code — parsedmarc 8.6.4 documentation - - + Overview: module code — parsedmarc 8.7.0 documentation + + + + @@ -33,7 +35,7 @@ parsedmarc
- 8.6.4 + 8.7.0
diff --git a/_modules/parsedmarc.html b/_modules/parsedmarc.html index 2f35a88..2f3e6ca 100644 --- a/_modules/parsedmarc.html +++ b/_modules/parsedmarc.html @@ -1,11 +1,13 @@ - + - parsedmarc — parsedmarc 8.6.4 documentation - - + parsedmarc — parsedmarc 8.7.0 documentation + + + + @@ -33,7 +35,7 @@ parsedmarc
- 8.6.4 + 8.7.0
@@ -118,7 +120,7 @@ from parsedmarc.utils import parse_email from parsedmarc.utils import timestamp_to_human, human_timestamp_to_datetime -__version__ = "8.6.4" +__version__ = "8.7.0" logger.debug("parsedmarc v{0}".format(__version__)) @@ -130,6 +132,7 @@ MAGIC_ZIP = b"\x50\x4B\x03\x04" MAGIC_GZIP = b"\x1F\x8B" MAGIC_XML = b"\x3c\x3f\x78\x6d\x6c\x20" +MAGIC_JSON = b"\7b" IP_ADDRESS_CACHE = ExpiringDict(max_len=10000, max_age_seconds=1800) @@ -142,6 +145,10 @@ """Raised when an invalid DMARC report is encountered"""
+
[docs]class InvalidSMTPTLSReport(ParserError): + """Raised when an invalid SMTP TLS report is encountered"""
+ +
[docs]class InvalidAggregateReport(InvalidDMARCReport): """Raised when an invalid DMARC aggregate report is encountered"""
@@ -288,6 +295,179 @@ return new_record +def _parse_smtp_tls_failure_details(failure_details): + try: + new_failure_details = OrderedDict( + result_type=failure_details["result-type"], + failed_session_count=failure_details["failed-session-count"], + sending_mta_ip=failure_details["sending-mta-ip"], + receiving_ip=failure_details["receiving-ip"] + ) + + if "receiving-mx-hostname" in failure_details: + new_failure_details["receiving_mx_hostname"] = failure_details[ + "receiving-mx-hostname"] + if "receiving-mx-helo" in failure_details: + new_failure_details["receiving_mx_helo"] = failure_details[ + "receiving-mx-helo"] + if "additional-info-uri" in failure_details: + new_failure_details["additional_info_uri"] = failure_details[ + "additional-info-uri"] + if "failure-reason-code" in failure_details: + new_failure_details["failure_reason_code"] = failure_details[ + "failure-reason-code"] + + return new_failure_details + + except KeyError as e: + raise InvalidSMTPTLSReport(f"Missing required failure details field:" + f" {e}") + except Exception as e: + raise InvalidSMTPTLSReport(str(e)) + + +def _parse_smtp_tls_report_policy(policy): + policy_types = ["tlsa", "sts", "no-policy-found"] + try: + policy_domain = policy["policy"]["policy-domain"] + policy_type = policy["policy"]["policy-type"] + failure_details = [] + if policy_type not in policy_types: + raise InvalidSMTPTLSReport(f"Invalid policy type " + f"{policy_type}") + new_policy = OrderedDict(policy_domain=policy_domain, + policy_type=policy_type) + if "policy-string" in policy["policy"]: + if isinstance(policy["policy"]["policy-string"], list): + if len(policy["policy"]["policy-string"]) > 0: + new_policy["policy_strings"] = policy["policy"][ + "policy-string"] + + if "mx-host-pattern" in policy["policy"]: + if isinstance(policy["policy"]["mx-host-pattern"], list): + if len(policy["policy"]["mx-host-pattern"]) > 0: + new_policy["mx_host_patterns"] = policy["policy"][ + "mx-host-pattern"] + new_policy["successful_session_count"] = policy["summary"][ + "total-successful-session-count"] + new_policy["failed_session_count"] = policy["summary"][ + "total-failure-session-count"] + if "failure-details" in policy: + for details in policy["failure-details"]: + failure_details.append(_parse_smtp_tls_failure_details( + details)) + new_policy["failure_details"] = failure_details + + return new_policy + + except KeyError as e: + raise InvalidSMTPTLSReport(f"Missing required policy field: {e}") + except Exception as e: + raise InvalidSMTPTLSReport(str(e)) + + +
[docs]def parse_smtp_tls_report_json(report): + """Parses and validates an SMTP TLS report""" + required_fields = ["organization-name", "date-range", + "contact-info", "report-id", + "policies"] + + try: + policies = [] + report = json.loads(report) + for required_field in required_fields: + if required_field not in report: + raise Exception(f"Missing required field: {required_field}]") + if not isinstance(report["policies"], list): + policies_type = type(report["policies"]) + raise InvalidSMTPTLSReport(f"policies must be a list, " + f"not {policies_type}") + for policy in report["policies"]: + policies.append(_parse_smtp_tls_report_policy(policy)) + + new_report = OrderedDict( + organization_name=report["organization-name"], + begin_date=report["date-range"]["start-datetime"], + end_date=report["date-range"]["end-datetime"], + report_id=report["report-id"], + policies=policies + ) + + return new_report + + except KeyError as e: + InvalidSMTPTLSReport(f"Missing required field: {e}") + except Exception as e: + raise InvalidSMTPTLSReport(str(e))
+ + +
[docs]def parsed_smtp_tls_reports_to_csv_rows(reports): + """Converts one oor more parsed SMTP TLS reports into a list of single + layer OrderedDict objects suitable for use in a CSV""" + if type(reports) is OrderedDict: + reports = [reports] + + rows = [] + for report in reports: + common_fields = OrderedDict( + organization_name=report["organization_name"], + begin_date=report["begin_date"], + end_date=report["end_date"], + report_id=report["report_id"] + ) + record = common_fields.copy() + for policy in report["policies"]: + if "policy_strings" in policy: + record["policy_strings"] = "|".join(policy["policy_strings"]) + if "mx_host_patterns" in policy: + record["mx_host_patterns"] = "|".join( + policy["mx_host_patterns"]) + successful_record = record.copy() + successful_record["successful_session_count"] = policy[ + "successful_session_count"] + rows.append(successful_record) + if "failure_details" in policy: + for failure_details in policy["failure_details"]: + failure_record = record.copy() + for key in failure_details.keys(): + failure_record[key] = failure_details[key] + rows.append(failure_record) + + return rows
+ + +
[docs]def parsed_smtp_tls_reports_to_csv(reports): + """ + Converts one or more parsed SMTP TLS reports to flat CSV format, including + headers + + Args: + reports: A parsed aggregate report or list of parsed aggregate reports + + Returns: + str: Parsed aggregate report data in flat CSV format, including headers + """ + + fields = ["organization_name", "begin_date", "end_date", "report_id", + "successful_session_count", "failed_session_count", + "policy_domain", "policy_type", "policy_strings", + "mx_host_patterns", "sending_mta_ip", "receiving_ip", + "receiving_mx_hostname", "receiving_mx_helo", + "additional_info_uri", "failure_reason_code"] + + csv_file_object = StringIO(newline="\n") + writer = DictWriter(csv_file_object, fields) + writer.writeheader() + + rows = parsed_smtp_tls_reports_to_csv_rows(reports) + + for row in rows: + writer.writerow(row) + csv_file_object.flush() + + return csv_file_object.getvalue()
+ +
[docs]def parse_aggregate_report_xml(xml, ip_db_path=None, offline=False, nameservers=None, timeout=2.0, parallel=False, keep_alive=None): @@ -308,6 +488,8 @@ """ errors = [] # Parse XML and recover from errors + if isinstance(xml, bytes): + xml = xml.decode(errors='ignore') try: xmltodict.parse(xml)["feedback"] except Exception as e: @@ -452,21 +634,27 @@ "Unexpected error: {0}".format(error.__str__()))
-
[docs]def extract_xml(input_): +
[docs]def extract_report(input_): """ - Extracts xml from a zip or gzip file at the given path, file-like object, + Extracts text from a zip or gzip file at the given path, file-like object, or bytes. Args: input_: A path to a file, a file like object, or bytes Returns: - str: The extracted XML + str: The extracted text """ try: + file_object = BytesIO() if type(input_) is str: - file_object = open(input_, "rb") + try: + file_object = BytesIO(b64decode(input_)) + except binascii.Error: + pass + if file_object is None: + file_object = open(input_, "rb") elif type(input_) is bytes: file_object = BytesIO(input_) else: @@ -476,30 +664,31 @@ file_object.seek(0) if header.startswith(MAGIC_ZIP): _zip = zipfile.ZipFile(file_object) - xml = _zip.open(_zip.namelist()[0]).read().decode(errors='ignore') + report = _zip.open(_zip.namelist()[0]).read().decode( + errors='ignore') elif header.startswith(MAGIC_GZIP): - xml = zlib.decompress(file_object.getvalue(), - zlib.MAX_WBITS | 16).decode(errors='ignore') - elif header.startswith(MAGIC_XML): - xml = file_object.read().decode(errors='ignore') + report = zlib.decompress( + file_object.getvalue(), + zlib.MAX_WBITS | 16).decode(errors='ignore') + elif header.startswith(MAGIC_XML) or header.startswith(MAGIC_JSON): + report = file_object.read().decode(errors='ignore') else: file_object.close() - raise InvalidAggregateReport("Not a valid zip, gzip, or xml file") + raise ParserError("Not a valid zip, gzip, json, or xml file") file_object.close() except FileNotFoundError: - raise InvalidAggregateReport("File was not found") + raise ParserError("File was not found") except UnicodeDecodeError: file_object.close() - raise InvalidAggregateReport("File objects must be opened in binary " - "(rb) mode") + raise ParserError("File objects must be opened in binary (rb) mode") except Exception as error: file_object.close() - raise InvalidAggregateReport( + raise ParserError( "Invalid archive file: {0}".format(error.__str__())) - return xml
+ return report
[docs]def parse_aggregate_report_file(_input, offline=False, ip_db_path=None, @@ -523,7 +712,11 @@ Returns: OrderedDict: The parsed DMARC aggregate report """ - xml = extract_xml(_input) + + try: + xml = extract_report(_input) + except Exception as e: + raise InvalidAggregateReport(e) return parse_aggregate_report_xml(xml, ip_db_path=ip_db_path, @@ -591,7 +784,7 @@ row["dmarc_aligned"] = record["alignment"]["dmarc"] row["disposition"] = record["policy_evaluated"]["disposition"] policy_override_reasons = list(map( - lambda r_: r_["type"], + lambda r_: r_["type"] or "none", record["policy_evaluated"] ["policy_override_reasons"])) policy_override_comments = list(map( @@ -903,12 +1096,14 @@ msg = email.message_from_string(input_) except Exception as e: - raise InvalidDMARCReport(e.__str__()) + raise ParserError(e.__str__()) subject = None feedback_report = None + smtp_tls_report = None sample = None if "From" in msg_headers: - logger.info("Parsing mail from {0}".format(msg_headers["From"])) + logger.info("Parsing mail from {0} on {1}".format(msg_headers["From"], + date)) if "Subject" in msg_headers: subject = msg_headers["Subject"] for part in msg.walk(): @@ -934,34 +1129,57 @@ sample = payload elif content_type == "message/rfc822": sample = payload + elif content_type == "application/tlsrpt+json": + if "{" not in payload: + payload = str(b64decode(payload)) + smtp_tls_report = parse_smtp_tls_report_json(payload) + return OrderedDict([("report_type", "smtp_tls"), + ("report", smtp_tls_report)]) + elif content_type == "application/tlsrpt+gzip": + payload = extract_report(payload) + smtp_tls_report = parse_smtp_tls_report_json(payload) + return OrderedDict([("report_type", "smtp_tls"), + ("report", smtp_tls_report)]) + elif content_type == "text/plain": if "A message claiming to be from you has failed" in payload: - parts = payload.split("detected.") - field_matches = text_report_regex.findall(parts[0]) - fields = dict() - for match in field_matches: - field_name = match[0].lower().replace(" ", "-") - fields[field_name] = match[1].strip() - feedback_report = "Arrival-Date: {}\n" \ - "Source-IP: {}" \ - "".format(fields["received-date"], - fields["sender-ip-address"]) + try: + parts = payload.split("detected.", 1) + field_matches = text_report_regex.findall(parts[0]) + fields = dict() + for match in field_matches: + field_name = match[0].lower().replace(" ", "-") + fields[field_name] = match[1].strip() + + feedback_report = "Arrival-Date: {}\n" \ + "Source-IP: {}" \ + "".format(fields["received-date"], + fields["sender-ip-address"]) + except Exception as e: + error = 'Unable to parse message with ' \ + 'subject "{0}": {1}'.format(subject, e) + raise InvalidDMARCReport(error) + sample = parts[1].lstrip() - sample = sample.replace("=\r\n", "") logger.debug(sample) else: try: payload = b64decode(payload) if payload.startswith(MAGIC_ZIP) or \ - payload.startswith(MAGIC_GZIP) or \ - payload.startswith(MAGIC_XML): + payload.startswith(MAGIC_GZIP): + payload = extract_report(payload) ns = nameservers - aggregate_report = parse_aggregate_report_file( + if payload.startswith("{"): + smtp_tls_report = parse_smtp_tls_report_json(payload) + result = OrderedDict([("report_type", "smtp_tls"), + ("report", smtp_tls_report)]) + return result + aggregate_report = parse_aggregate_report_xml( payload, ip_db_path=ip_db_path, offline=offline, nameservers=ns, - dns_timeout=dns_timeout, + timeout=dns_timeout, parallel=parallel, keep_alive=keep_alive) result = OrderedDict([("report_type", "aggregate"), @@ -975,12 +1193,12 @@ error = 'Message with subject "{0}" ' \ 'is not a valid ' \ 'aggregate DMARC report: {1}'.format(subject, e) - raise InvalidAggregateReport(error) + raise ParserError(error) except Exception as e: error = 'Unable to parse message with ' \ 'subject "{0}": {1}'.format(subject, e) - raise InvalidDMARCReport(error) + raise ParserError(error) if feedback_report and sample: try: @@ -1007,7 +1225,7 @@ if result is None: error = 'Message with subject "{0}" is ' \ - 'not a valid DMARC report'.format(subject) + 'not a valid report'.format(subject) raise InvalidDMARCReport(error)
@@ -1054,18 +1272,22 @@ ("report", report)]) except InvalidAggregateReport: try: - sa = strip_attachment_payloads - results = parse_report_email(content, - ip_db_path=ip_db_path, - offline=offline, - nameservers=nameservers, - dns_timeout=dns_timeout, - strip_attachment_payloads=sa, - parallel=parallel, - keep_alive=keep_alive) - except InvalidDMARCReport: - raise InvalidDMARCReport("Not a valid aggregate or forensic " - "report") + report = parse_smtp_tls_report_json(content) + results = OrderedDict([("report_type", "smtp_tls"), + ("report", report)]) + except InvalidSMTPTLSReport: + try: + sa = strip_attachment_payloads + results = parse_report_email(content, + ip_db_path=ip_db_path, + offline=offline, + nameservers=nameservers, + dns_timeout=dns_timeout, + strip_attachment_payloads=sa, + parallel=parallel, + keep_alive=keep_alive) + except InvalidDMARCReport: + raise ParserError("Not a valid report") return results
@@ -1094,6 +1316,7 @@ """ aggregate_reports = [] forensic_reports = [] + smtp_tls_reports = [] try: mbox = mailbox.mbox(input_) message_keys = mbox.keys() @@ -1119,12 +1342,15 @@ aggregate_reports.append(parsed_email["report"]) elif parsed_email["report_type"] == "forensic": forensic_reports.append(parsed_email["report"]) + elif parsed_email["report_type"] == "smtp_tls": + smtp_tls_reports.append(parsed_email["report"]) except InvalidDMARCReport as error: logger.warning(error.__str__()) except mailbox.NoSuchMailboxError: raise InvalidDMARCReport("Mailbox {0} does not exist".format(input_)) return OrderedDict([("aggregate_reports", aggregate_reports), - ("forensic_reports", forensic_reports)]) + ("forensic_reports", forensic_reports), + ("smtp_tls_reports", smtp_tls_reports)])
[docs]def get_dmarc_reports_from_mailbox(connection: MailboxConnection, @@ -1172,20 +1398,25 @@ aggregate_reports = [] forensic_reports = [] + smtp_tls_reports = [] aggregate_report_msg_uids = [] forensic_report_msg_uids = [] + smtp_tls_msg_uids = [] aggregate_reports_folder = "{0}/Aggregate".format(archive_folder) forensic_reports_folder = "{0}/Forensic".format(archive_folder) + smtp_tls_reports_folder = "{0}/SMTP-TLS".format(archive_folder) invalid_reports_folder = "{0}/Invalid".format(archive_folder) if results: aggregate_reports = results["aggregate_reports"].copy() forensic_reports = results["forensic_reports"].copy() + smtp_tls_reports = results["smtp_tls_reports"].copy() if not test and create_folders: connection.create_folder(archive_folder) connection.create_folder(aggregate_reports_folder) connection.create_folder(forensic_reports_folder) + connection.create_folder(smtp_tls_reports_folder) connection.create_folder(invalid_reports_folder) messages = connection.fetch_messages(reports_folder, batch_size=batch_size) @@ -1221,7 +1452,10 @@ elif parsed_email["report_type"] == "forensic": forensic_reports.append(parsed_email["report"]) forensic_report_msg_uids.append(msg_uid) - except InvalidDMARCReport as error: + elif parsed_email["report_type"] == "smtp_tls": + smtp_tls_reports.append(parsed_email["report"]) + smtp_tls_msg_uids.append(msg_uid) + except ParserError as error: logger.warning(error.__str__()) if not test: if delete: @@ -1237,7 +1471,8 @@ if not test: if delete: processed_messages = aggregate_report_msg_uids + \ - forensic_report_msg_uids + forensic_report_msg_uids + \ + smtp_tls_msg_uids number_of_processed_msgs = len(processed_messages) for i in range(number_of_processed_msgs): @@ -1292,8 +1527,29 @@ e = "Error moving message UID {0}: {1}".format( msg_uid, e) logger.error("Mailbox error: {0}".format(e)) + if len(smtp_tls_msg_uids) > 0: + message = "Moving SMTP TLS report messages from" + logger.debug( + "{0} {1} to {2}".format(message, + reports_folder, + smtp_tls_reports_folder)) + number_of_smtp_tls_uids = len(smtp_tls_msg_uids) + for i in range(number_of_smtp_tls_uids): + msg_uid = smtp_tls_msg_uids[i] + message = "Moving message" + logger.debug("{0} {1} of {2}: UID {3}".format( + message, + i + 1, smtp_tls_msg_uids, msg_uid)) + try: + connection.move_message(msg_uid, + smtp_tls_reports_folder) + except Exception as e: + e = "Error moving message UID {0}: {1}".format( + msg_uid, e) + logger.error("Mailbox error: {0}".format(e)) results = OrderedDict([("aggregate_reports", aggregate_reports), - ("forensic_reports", forensic_reports)]) + ("forensic_reports", forensic_reports), + ("smtp_tls_reports", smtp_tls_reports)]) total_messages = len(connection.fetch_messages(reports_folder)) @@ -1405,8 +1661,10 @@
[docs]def save_output(results, output_directory="output", aggregate_json_filename="aggregate.json", forensic_json_filename="forensic.json", + smtp_tls_json_filename="smtp_tls.json", aggregate_csv_filename="aggregate.csv", - forensic_csv_filename="forensic.csv"): + forensic_csv_filename="forensic.csv", + smtp_tls_csv_filename="smtp_tls.csv"): """ Save report data in the given directory @@ -1415,12 +1673,15 @@ output_directory (str): The path to the directory to save in aggregate_json_filename (str): Filename for the aggregate JSON file forensic_json_filename (str): Filename for the forensic JSON file + smtp_tls_json_filename (str): Filename for the SMTP TLS JSON file aggregate_csv_filename (str): Filename for the aggregate CSV file forensic_csv_filename (str): Filename for the forensic CSV file + smtp_tls_csv_filename (str): Filename for the SMTP TLS CSV file """ aggregate_reports = results["aggregate_reports"] forensic_reports = results["forensic_reports"] + smtp_tls_reports = results["smtp_tls_reports"] if os.path.exists(output_directory): if not os.path.isdir(output_directory): @@ -1440,6 +1701,12 @@ append_csv(os.path.join(output_directory, forensic_csv_filename), parsed_forensic_reports_to_csv(forensic_reports)) + append_json(os.path.join(output_directory, smtp_tls_json_filename), + smtp_tls_reports) + + append_csv(os.path.join(output_directory, smtp_tls_csv_filename), + parsed_smtp_tls_reports_to_csv(smtp_tls_reports)) + samples_directory = os.path.join(output_directory, "samples") if not os.path.exists(samples_directory): os.makedirs(samples_directory) diff --git a/_modules/parsedmarc/elastic.html b/_modules/parsedmarc/elastic.html index f4ce099..be7ad03 100644 --- a/_modules/parsedmarc/elastic.html +++ b/_modules/parsedmarc/elastic.html @@ -1,11 +1,13 @@ - + - parsedmarc.elastic — parsedmarc 8.6.4 documentation - - + parsedmarc.elastic — parsedmarc 8.7.0 documentation + + + + @@ -33,7 +35,7 @@ parsedmarc
- 8.6.4 + 8.7.0
@@ -249,12 +251,78 @@ sample = Object(_ForensicSampleDoc) +class _SMTPTLSFailureDetailsDoc(InnerDoc): + result_type = Text() + sending_mta_ip = Ip() + receiving_mx_helo = Text() + receiving_ip = Ip() + failed_session_count = Integer() + additional_information_uri = Text() + failure_reason_code = Text() + + +class _SMTPTLSPolicyDoc(InnerDoc): + policy_domain = Text() + policy_type = Text() + policy_strings = Text() + mx_host_patterns = Text() + successful_session_count = Integer() + failed_session_count = Integer() + failure_details = Nested(_SMTPTLSFailureDetailsDoc) + + def add_failure_details(self, result_type, ip_address, + receiving_ip, + receiving_mx_helo, + failed_session_count, + receiving_mx_hostname=None, + additional_information_uri=None, + failure_reason_code=None): + self.failure_details.append( + result_type=result_type, + ip_address=ip_address, + receiving_mx_hostname=receiving_mx_hostname, + receiving_mx_helo=receiving_mx_helo, + receiving_ip=receiving_ip, + failed_session_count=failed_session_count, + additional_information=additional_information_uri, + failure_reason_code=failure_reason_code + ) + + +class _SMTPTLSFailureReportDoc(Document): + + class Index: + name = "smtp_tls" + + organization_name = Text() + date_range = Date() + date_begin = Date() + date_end = Date() + contact_info = Text() + report_id = Text() + policies = Nested(_SMTPTLSPolicyDoc) + + def add_policy(self, policy_type, policy_domain, + successful_session_count, + failed_session_count, + policy_string=None, + mx_host_patterns=None, + failure_details=None): + self.policies.append(policy_type=policy_type, + policy_domain=policy_domain, + successful_session_count=successful_session_count, + failed_session_count=failed_session_count, + policy_string=policy_string, + mx_host_patterns=mx_host_patterns, + failure_details=failure_details) + +
[docs]class AlreadySaved(ValueError): """Raised when a report to be saved matches an existing report"""
[docs]def set_hosts(hosts, use_ssl=False, ssl_cert_path=None, - username=None, password=None, timeout=60.0): + username=None, password=None, apiKey=None, timeout=60.0): """ Sets the Elasticsearch hosts to use @@ -264,6 +332,7 @@ ssl_cert_path (str): Path to the certificate chain username (str): The username to use for authentication password (str): The password to use for authentication + apiKey (str): The Base64 encoded API key to use for authentication timeout (float): Timeout in seconds """ if not isinstance(hosts, list): @@ -281,6 +350,8 @@ conn_params['verify_certs'] = False if username: conn_params['http_auth'] = (username+":"+password) + if apiKey: + conn_params['api_key'] = apiKey connections.create_connection(**conn_params)
@@ -635,6 +706,130 @@ except KeyError as e: raise InvalidForensicReport( "Forensic report missing required field: {0}".format(e.__str__()))
+ + +
[docs]def save_smtp_tls_report_to_elasticsearch(report, + index_suffix=None, + monthly_indexes=False, + number_of_shards=1, + number_of_replicas=0): + """ + Saves a parsed SMTP TLS report to elasticSearch + + Args: + report (OrderedDict): A parsed SMTP TLS report + index_suffix (str): The suffix of the name of the index to save to + monthly_indexes (bool): Use monthly indexes instead of daily indexes + number_of_shards (int): The number of shards to use in the index + number_of_replicas (int): The number of replicas to use in the index + + Raises: + AlreadySaved + """ + logger.info("Saving aggregate report to Elasticsearch") + org_name = report["org_name"] + report_id = report["report_id"] + begin_date = human_timestamp_to_datetime(report["begin_date"], + to_utc=True) + end_date = human_timestamp_to_datetime(report["end_date"], + to_utc=True) + begin_date_human = begin_date.strftime("%Y-%m-%d %H:%M:%SZ") + end_date_human = end_date.strftime("%Y-%m-%d %H:%M:%SZ") + if monthly_indexes: + index_date = begin_date.strftime("%Y-%m") + else: + index_date = begin_date.strftime("%Y-%m-%d") + report["begin_date"] = begin_date + report["end_date"] = end_date + + org_name_query = Q(dict(match_phrase=dict(org_name=org_name))) + report_id_query = Q(dict(match_phrase=dict(report_id=report_id))) + begin_date_query = Q(dict(match=dict(date_begin=begin_date))) + end_date_query = Q(dict(match=dict(date_end=end_date))) + + if index_suffix is not None: + search = Search(index="smtp_tls_{0}*".format(index_suffix)) + else: + search = Search(index="smtp_tls") + query = org_name_query & report_id_query + query = query & begin_date_query & end_date_query + search.query = query + + try: + existing = search.execute() + except Exception as error_: + raise ElasticsearchError("Elasticsearch's search for existing report \ + error: {}".format(error_.__str__())) + + if len(existing) > 0: + raise AlreadySaved(f"An SMTP TLS report ID {report_id} from " + f" {org_name} with a date range of " + f"{begin_date_human} UTC to " + f"{end_date_human} UTC already " + "exists in Elasticsearch") + + index = "smtp_tls" + if index_suffix: + index = "{0}_{1}".format(index, index_suffix) + index = "{0}-{1}".format(index, index_date) + index_settings = dict(number_of_shards=number_of_shards, + number_of_replicas=number_of_replicas) + + smtp_tls_doc = _SMTPTLSFailureReportDoc( + organization_name=report["organization_name"], + date_range=[report["date_begin"], report["date_end"]], + date_begin=report["date_begin"], + date_end=report["date_end"], + contact_info=report["contact_info"], + report_id=report["report_id"] + ) + + for policy in report['policies']: + policy_strings = None + mx_host_patterns = None + if "policy_strings" in policy: + policy_strings = policy["policy_strings"] + if "mx_host_patterns" in policy: + mx_host_patterns = policy["mx_host_patterns"] + policy_doc = _SMTPTLSPolicyDoc( + policy_domain=policy["policy_domain"], + policy_type=policy["policy_type"], + policy_string=policy_strings, + mx_host_patterns=mx_host_patterns + ) + if "failure_details" in policy: + failure_details = policy["failure_details"] + receiving_mx_hostname = None + additional_information_uri = None + failure_reason_code = None + if "receiving_mx_hostname" in failure_details: + receiving_mx_hostname = failure_details[ + "receiving_mx_hostname"] + if "additional_information_uri" in failure_details: + additional_information_uri = failure_details[ + "additional_information_uri"] + if "failure_reason_code" in failure_details: + failure_reason_code = failure_details["failure_reason_code"] + policy_doc.add_failure_details( + result_type=failure_details["result_type"], + ip_address=failure_details["ip_address"], + receiving_ip=failure_details["receiving_ip"], + receiving_mx_helo=failure_details["receiving_mx_helo"], + failed_session_count=failure_details["failed_session_count"], + receiving_mx_hostname=receiving_mx_hostname, + additional_information_uri=additional_information_uri, + failure_reason_code=failure_reason_code + ) + smtp_tls_doc.policies.append(policy_doc) + + create_indexes([index], index_settings) + smtp_tls_doc.meta.index = index + + try: + smtp_tls_doc.save() + except Exception as e: + raise ElasticsearchError( + "Elasticsearch error: {0}".format(e.__str__()))
diff --git a/_modules/parsedmarc/splunk.html b/_modules/parsedmarc/splunk.html index da8a6b3..5956e43 100644 --- a/_modules/parsedmarc/splunk.html +++ b/_modules/parsedmarc/splunk.html @@ -1,11 +1,13 @@ - + - parsedmarc.splunk — parsedmarc 8.6.4 documentation - - + parsedmarc.splunk — parsedmarc 8.7.0 documentation + + + + @@ -33,7 +35,7 @@ parsedmarc
- 8.6.4 + 8.7.0
@@ -92,7 +94,7 @@ from parsedmarc import __version__ from parsedmarc.log import logger -from parsedmarc.utils import human_timestamp_to_timestamp +from parsedmarc.utils import human_timestamp_to_unix_timestamp urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) @@ -189,7 +191,7 @@ "spf"] data["sourcetype"] = "dmarc:aggregate" - timestamp = human_timestamp_to_timestamp( + timestamp = human_timestamp_to_unix_timestamp( new_report["begin_date"]) data["time"] = timestamp data["event"] = new_report.copy() @@ -225,12 +227,49 @@ for report in forensic_reports: data = self._common_data.copy() data["sourcetype"] = "dmarc:forensic" - timestamp = human_timestamp_to_timestamp( + timestamp = human_timestamp_to_unix_timestamp( report["arrival_date_utc"]) data["time"] = timestamp data["event"] = report.copy() json_str += "{0}\n".format(json.dumps(data)) + if not self.session.verify: + logger.debug("Skipping certificate verification for Splunk HEC") + try: + response = self.session.post(self.url, data=json_str, + timeout=self.timeout) + response = response.json() + except Exception as e: + raise SplunkError(e.__str__()) + if response["code"] != 0: + raise SplunkError(response["text"])
+ +
[docs] def save_smtp_tls_reports_to_splunk(self, reports): + """ + Saves aggregate DMARC reports to Splunk + + Args: + reports: A list of SMTP TLS report dictionaries + to save in Splunk + + """ + logger.debug("Saving SMTP TLS reports to Splunk") + if isinstance(reports, dict): + reports = [reports] + + if len(reports) < 1: + return + + data = self._common_data.copy() + json_str = "" + for report in reports: + data["sourcetype"] = "smtp:tls" + timestamp = human_timestamp_to_unix_timestamp( + report["begin_date"]) + data["time"] = timestamp + data["event"] = report.copy() + json_str += "{0}\n".format(json.dumps(data)) + if not self.session.verify: logger.debug("Skipping certificate verification for Splunk HEC") try: diff --git a/_modules/parsedmarc/utils.html b/_modules/parsedmarc/utils.html index 7221ede..cc21b27 100644 --- a/_modules/parsedmarc/utils.html +++ b/_modules/parsedmarc/utils.html @@ -1,11 +1,13 @@ - + - parsedmarc.utils — parsedmarc 8.6.4 documentation - - + parsedmarc.utils — parsedmarc 8.7.0 documentation + + + + @@ -33,7 +35,7 @@ parsedmarc
- 8.6.4 + 8.7.0
@@ -303,7 +305,7 @@ return dt.astimezone(timezone.utc) if to_utc else dt
-
[docs]def human_timestamp_to_timestamp(human_timestamp): +
[docs]def human_timestamp_to_unix_timestamp(human_timestamp): """ Converts a human-readable timestamp into a UNIX timestamp diff --git a/_sources/elasticsearch.md.txt b/_sources/elasticsearch.md.txt index 52dd271..96f5bdf 100644 --- a/_sources/elasticsearch.md.txt +++ b/_sources/elasticsearch.md.txt @@ -227,7 +227,7 @@ Kibana index patterns with versions that match the upgraded indexes: Starting in version 5.0.0, `parsedmarc` stores data in a separate index for each day to make it easy to comply with records -retention regulations such as GDPR. For fore information, +retention regulations such as GDPR. For more information, check out the Elastic guide to [managing time-based indexes efficiently](https://www.elastic.co/blog/managing-time-based-indices-efficiently). [elasticsearch]: https://www.elastic.co/guide/en/elasticsearch/reference/current/rpm.html diff --git a/_sources/output.md.txt b/_sources/output.md.txt index f676891..61b8273 100644 --- a/_sources/output.md.txt +++ b/_sources/output.md.txt @@ -187,3 +187,46 @@ Thanks to GitHub user [xennn](https://github.com/xennn) for the anonymized feedback_type,user_agent,version,original_envelope_id,original_mail_from,original_rcpt_to,arrival_date,arrival_date_utc,subject,message_id,authentication_results,dkim_domain,source_ip_address,source_country,source_reverse_dns,source_base_domain,delivery_result,auth_failure,reported_domain,authentication_mechanisms,sample_headers_only auth-failure,Lua/1.0,1.0,,sharepoint@domain.de,peter.pan@domain.de,"Mon, 01 Oct 2018 11:20:27 +0200",2018-10-01 09:20:27,Subject,<38.E7.30937.BD6E1BB5@ mailrelay.de>,"dmarc=fail (p=none, dis=none) header.from=domain.de",,10.10.10.10,,,,policy,dmarc,domain.de,,False ``` + +### JSON SMTP TLS report + +```json +[ + { + "organization_name": "Example Inc.", + "begin_date": "2024-01-09T00:00:00Z", + "end_date": "2024-01-09T23:59:59Z", + "report_id": "2024-01-09T00:00:00Z_example.com", + "policies": [ + { + "policy_domain": "example.com", + "policy_type": "sts", + "policy_strings": [ + "version: STSv1", + "mode: testing", + "mx: example.com", + "max_age: 86400" + ], + "successful_session_count": 0, + "failed_session_count": 3, + "failure_details": [ + { + "result_type": "validation-failure", + "failed_session_count": 2, + "sending_mta_ip": "209.85.222.201", + "receiving_ip": "173.212.201.41", + "receiving_mx_hostname": "example.com" + }, + { + "result_type": "validation-failure", + "failed_session_count": 1, + "sending_mta_ip": "209.85.208.176", + "receiving_ip": "173.212.201.41", + "receiving_mx_hostname": "example.com" + } + ] + } + ] + } +] +``` \ No newline at end of file diff --git a/_sources/usage.md.txt b/_sources/usage.md.txt index cd5d8b3..cc0e035 100644 --- a/_sources/usage.md.txt +++ b/_sources/usage.md.txt @@ -137,9 +137,9 @@ The full set of configuration options are: - `archive_folder` - str: The mailbox folder (or label for Gmail) to sort processed emails into (Default: `Archive`) - `watch` - bool: Use the IMAP `IDLE` command to process - - messages as they arrive or poll MS Graph for new messages + messages as they arrive or poll MS Graph for new messages - `delete` - bool: Delete messages after processing them, - - instead of archiving them + instead of archiving them - `test` - bool: Do not move or delete messages - `batch_size` - int: Number of messages to read and process before saving. Default `10`. Use `0` for no limit. @@ -225,9 +225,12 @@ The full set of configuration options are: Special characters in the username or password must be [URL encoded]. ::: - + - `user` - str: Basic auth username + - `password` - str: Basic auth password + - `apiKey` - str: API key - `ssl` - bool: Use an encrypted SSL/TLS connection (Default: `True`) + - `timeout` - float: Timeout in seconds (Default: 60) - `cert_path` - str: Path to a trusted certificates - `index_suffix` - str: A suffix to apply to the index names - `monthly_indexes` - bool: Use monthly indexes instead of daily indexes @@ -292,6 +295,8 @@ The full set of configuration options are: (Default: `https://www.googleapis.com/auth/gmail.modify`) - `oauth2_port` - int: The TCP port for the local server to listen on for the OAuth2 response (Default: `8080`) + - `paginate_messages` - bool: When `True`, fetch all applicable Gmail messages. + When `False`, only fetch up to 100 new messages per run (Default: `True`) - `log_analytics` - `client_id` - str: The app registration's client ID - `client_secret` - str: The app registration's client secret @@ -300,6 +305,7 @@ The full set of configuration options are: - `dcr_immutable_id` - str: The immutable ID of the Data Collection Rule (DCR) - `dcr_aggregate_stream` - str: The stream name for aggregate reports in the DCR - `dcr_forensic_stream` - str: The stream name for the forensic reports in the DCR + - `dcr_smtp_tls_stream` - str: The stream name for the SMTP TLS reports in the DCR :::{note} Information regarding the setup of the Data Collection Rule can be found [here](https://learn.microsoft.com/en-us/azure/azure-monitor/logs/tutorial-logs-ingestion-portal). diff --git a/_static/documentation_options.js b/_static/documentation_options.js index 7a4ffb0..fc208ff 100644 --- a/_static/documentation_options.js +++ b/_static/documentation_options.js @@ -1,6 +1,6 @@ var DOCUMENTATION_OPTIONS = { URL_ROOT: document.getElementById("documentation_options").getAttribute('data-url_root'), - VERSION: '8.6.4', + VERSION: '8.7.0', LANGUAGE: 'en', COLLAPSE_INDEX: false, BUILDER: 'html', diff --git a/api.html b/api.html index ed1ed76..c33b396 100644 --- a/api.html +++ b/api.html @@ -1,12 +1,14 @@ - + - API reference — parsedmarc 8.6.4 documentation - - + API reference — parsedmarc 8.7.0 documentation + + + + @@ -35,7 +37,7 @@ parsedmarc
- 8.6.4 + 8.7.0
@@ -61,9 +63,10 @@
  • InvalidAggregateReport
  • InvalidDMARCReport
  • InvalidForensicReport
    • +
  • InvalidSMTPTLSReport
  • ParserError
  • email_results()
    • -
  • extract_xml()
    • +
  • extract_report()
  • get_dmarc_reports_from_mailbox()
  • get_dmarc_reports_from_mbox()
  • get_report_zip()
    • @@ -72,10 +75,13 @@
  • parse_forensic_report()
  • parse_report_email()
  • parse_report_file()
    • +
  • parse_smtp_tls_report_json()
  • parsed_aggregate_reports_to_csv()
  • parsed_aggregate_reports_to_csv_rows()
  • parsed_forensic_reports_to_csv()
  • parsed_forensic_reports_to_csv_rows()
    • +
  • parsed_smtp_tls_reports_to_csv()
    • +
  • parsed_smtp_tls_reports_to_csv_rows()
  • save_output()
  • watch_inbox()
    • @@ -87,6 +93,7 @@
  • migrate_indexes()
  • save_aggregate_report_to_elasticsearch()
  • save_forensic_report_to_elasticsearch()
    • +
  • save_smtp_tls_report_to_elasticsearch()
  • set_hosts()
    • @@ -94,6 +101,7 @@
  • HECClient
  • SplunkError
    • @@ -110,7 +118,7 @@
  • get_ip_address_info()
  • get_reverse_dns()
  • human_timestamp_to_datetime()
    • -
  • human_timestamp_to_timestamp()
    • +
  • human_timestamp_to_unix_timestamp()
  • is_mbox()
  • is_outlook_msg()
  • parse_email()
    • @@ -171,6 +179,12 @@

    Raised when an invalid DMARC forensic report is encountered

    +
    +
    +exception parsedmarc.InvalidSMTPTLSReport[source]
    +

    Raised when an invalid SMTP TLS report is encountered

    +
    +
    exception parsedmarc.ParserError[source]
    @@ -204,16 +218,16 @@
    -
    -parsedmarc.extract_xml(input_)[source]
    -

    Extracts xml from a zip or gzip file at the given path, file-like object, +

    +parsedmarc.extract_report(input_)[source]
    +

    Extracts text from a zip or gzip file at the given path, file-like object, or bytes.

    Parameters

    input – A path to a file, a file like object, or bytes

    Returns
    -

    The extracted XML

    +

    The extracted text

    Return type

    str

    @@ -442,6 +456,12 @@ forensic report results

    +
    +
    +parsedmarc.parse_smtp_tls_report_json(report)[source]
    +

    Parses and validates an SMTP TLS report

    +
    +
    parsedmarc.parsed_aggregate_reports_to_csv(reports)[source]
    @@ -515,9 +535,34 @@ format

    +
    +
    +parsedmarc.parsed_smtp_tls_reports_to_csv(reports)[source]
    +

    Converts one or more parsed SMTP TLS reports to flat CSV format, including +headers

    +
    +
    Parameters
    +

    reports – A parsed aggregate report or list of parsed aggregate reports

    +
    +
    Returns
    +

    Parsed aggregate report data in flat CSV format, including headers

    +
    +
    Return type
    +

    str

    +
    +
    +
    + +
    +
    +parsedmarc.parsed_smtp_tls_reports_to_csv_rows(reports)[source]
    +

    Converts one oor more parsed SMTP TLS reports into a list of single +layer OrderedDict objects suitable for use in a CSV

    +
    +
    -parsedmarc.save_output(results, output_directory='output', aggregate_json_filename='aggregate.json', forensic_json_filename='forensic.json', aggregate_csv_filename='aggregate.csv', forensic_csv_filename='forensic.csv')[source]
    +parsedmarc.save_output(results, output_directory='output', aggregate_json_filename='aggregate.json', forensic_json_filename='forensic.json', smtp_tls_json_filename='smtp_tls.json', aggregate_csv_filename='aggregate.csv', forensic_csv_filename='forensic.csv', smtp_tls_csv_filename='smtp_tls.csv')[source]

    Save report data in the given directory

    Parameters
    @@ -526,8 +571,10 @@ format

  • output_directory (str) – The path to the directory to save in

  • aggregate_json_filename (str) – Filename for the aggregate JSON file

  • forensic_json_filename (str) – Filename for the forensic JSON file

    • +

  • smtp_tls_json_filename (str) – Filename for the SMTP TLS JSON file

  • aggregate_csv_filename (str) – Filename for the aggregate CSV file

  • forensic_csv_filename (str) – Filename for the forensic CSV file

    • +

  • smtp_tls_csv_filename (str) – Filename for the SMTP TLS CSV file

    • @@ -649,9 +696,29 @@ index

    +
    +
    +parsedmarc.elastic.save_smtp_tls_report_to_elasticsearch(report, index_suffix=None, monthly_indexes=False, number_of_shards=1, number_of_replicas=0)[source]
    +

    Saves a parsed SMTP TLS report to elasticSearch

    +
    +
    Parameters
    +
      +

    • report (OrderedDict) – A parsed SMTP TLS report

      • +

    • index_suffix (str) – The suffix of the name of the index to save to

      • +

    • monthly_indexes (bool) – Use monthly indexes instead of daily indexes

      • +

    • number_of_shards (int) – The number of shards to use in the index

      • +

    • number_of_replicas (int) – The number of replicas to use in the index

      • +
    +
    +
    Raises
    +

    AlreadySaved –

    +
    +
    +
    +
    -parsedmarc.elastic.set_hosts(hosts, use_ssl=False, ssl_cert_path=None, username=None, password=None, timeout=60.0)[source]
    +parsedmarc.elastic.set_hosts(hosts, use_ssl=False, ssl_cert_path=None, username=None, password=None, apiKey=None, timeout=60.0)[source]

    Sets the Elasticsearch hosts to use

    Parameters
    @@ -661,6 +728,7 @@ index

  • ssl_cert_path (str) – Path to the certificate chain

  • username (str) – The username to use for authentication

  • password (str) – The password to use for authentication

    • +

  • apiKey (str) – The Base64 encoded API key to use for authentication

  • timeout (float) – Timeout in seconds

    • @@ -711,6 +779,18 @@ to save in Splunk

    +
    +
    +save_smtp_tls_reports_to_splunk(reports)[source]
    +

    Saves aggregate DMARC reports to Splunk

    +
    +
    Parameters
    +

    reports – A list of SMTP TLS report dictionaries +to save in Splunk

    +
    +
    +
    +
    @@ -897,8 +977,8 @@ with the given IPv4 or IPv6 address

    -
    -parsedmarc.utils.human_timestamp_to_timestamp(human_timestamp)[source]
    +
    +parsedmarc.utils.human_timestamp_to_unix_timestamp(human_timestamp)[source]

    Converts a human-readable timestamp into a UNIX timestamp

    Parameters
    diff --git a/contributing.html b/contributing.html index 99933cc..9d1040c 100644 --- a/contributing.html +++ b/contributing.html @@ -1,12 +1,14 @@ - + - Contributing to parsedmarc — parsedmarc 8.6.4 documentation - - + Contributing to parsedmarc — parsedmarc 8.7.0 documentation + + + + @@ -36,7 +38,7 @@ parsedmarc
    - 8.6.4 + 8.7.0
    diff --git a/davmail.html b/davmail.html index 8a66226..b53b64c 100644 --- a/davmail.html +++ b/davmail.html @@ -1,12 +1,14 @@ - + - Accessing an inbox using OWA/EWS — parsedmarc 8.6.4 documentation - - + Accessing an inbox using OWA/EWS — parsedmarc 8.7.0 documentation + + + + @@ -36,7 +38,7 @@ parsedmarc
    - 8.6.4 + 8.7.0
    diff --git a/dmarc.html b/dmarc.html index 1dc9762..d306c6c 100644 --- a/dmarc.html +++ b/dmarc.html @@ -1,12 +1,14 @@ - + - Understanding DMARC — parsedmarc 8.6.4 documentation - - + Understanding DMARC — parsedmarc 8.7.0 documentation + + + + @@ -36,7 +38,7 @@ parsedmarc
    - 8.6.4 + 8.7.0
    diff --git a/elasticsearch.html b/elasticsearch.html index 7a70b15..1ab050a 100644 --- a/elasticsearch.html +++ b/elasticsearch.html @@ -1,12 +1,14 @@ - + - Elasticsearch and Kibana — parsedmarc 8.6.4 documentation - - + Elasticsearch and Kibana — parsedmarc 8.7.0 documentation + + + + @@ -36,7 +38,7 @@ parsedmarc
    - 8.6.4 + 8.7.0
    @@ -266,7 +268,7 @@ Saved Objects page

    Records retention

    Starting in version 5.0.0, parsedmarc stores data in a separate index for each day to make it easy to comply with records -retention regulations such as GDPR. For fore information, +retention regulations such as GDPR. For more information, check out the Elastic guide to managing time-based indexes efficiently.

    diff --git a/genindex.html b/genindex.html index 606950d..d92e608 100644 --- a/genindex.html +++ b/genindex.html @@ -1,11 +1,13 @@ - + - Index — parsedmarc 8.6.4 documentation - - + Index — parsedmarc 8.7.0 documentation + + + + @@ -33,7 +35,7 @@ parsedmarc
    - 8.6.4 + 8.7.0
    @@ -142,7 +144,7 @@ @@ -180,7 +182,7 @@ @@ -191,10 +193,12 @@
  • InvalidAggregateReport
  • InvalidDMARCReport +
    • +
  • InvalidForensicReport
    • - +
    • save_output() (in module parsedmarc) +
      • +
    • save_smtp_tls_report_to_elasticsearch() (in module parsedmarc.elastic) +
      • +
    • save_smtp_tls_reports_to_splunk() (parsedmarc.splunk.HECClient method)
    • set_hosts() (in module parsedmarc.elastic)
      • diff --git a/index.html b/index.html index 46806f9..b974fe2 100644 --- a/index.html +++ b/index.html @@ -1,12 +1,14 @@ - + - parsedmarc documentation - Open source DMARC report analyzer and visualizer — parsedmarc 8.6.4 documentation - - + parsedmarc documentation - Open source DMARC report analyzer and visualizer — parsedmarc 8.7.0 documentation + + + + @@ -35,7 +37,7 @@ parsedmarc
      - 8.6.4 + 8.7.0
      diff --git a/installation.html b/installation.html index 503e5b7..d79a33a 100644 --- a/installation.html +++ b/installation.html @@ -1,12 +1,14 @@ - + - Installation — parsedmarc 8.6.4 documentation - - + Installation — parsedmarc 8.7.0 documentation + + + + @@ -36,7 +38,7 @@ parsedmarc
      - 8.6.4 + 8.7.0
      diff --git a/kibana.html b/kibana.html index 36a89dc..1899dca 100644 --- a/kibana.html +++ b/kibana.html @@ -1,12 +1,14 @@ - + - Using the Kibana dashboards — parsedmarc 8.6.4 documentation - - + Using the Kibana dashboards — parsedmarc 8.7.0 documentation + + + + @@ -36,7 +38,7 @@ parsedmarc
      - 8.6.4 + 8.7.0
      diff --git a/mailing-lists.html b/mailing-lists.html index 9951db7..f39ebb5 100644 --- a/mailing-lists.html +++ b/mailing-lists.html @@ -1,12 +1,14 @@ - + - What about mailing lists? — parsedmarc 8.6.4 documentation - - + What about mailing lists? — parsedmarc 8.7.0 documentation + + + + @@ -34,7 +36,7 @@ parsedmarc
      - 8.6.4 + 8.7.0
      diff --git a/objects.inv b/objects.inv index 6f8f818bc13af9853bd3bc165cf9efb1cc9939c8..43f29cd0da84d5c06b4c5b75419a70c3800f043b 100644 GIT binary patch delta 887 zcmV--1Bm?12lxk&Km#`}Fp)!2e_Pe4>C{Jyz{O_8-~mpP{`EV+Nl2Qi#_3Cf55Dhy zaK}MIv;kHVP4i}RQ7hI8$Uo<#;p58I;zN*(KIWsW*)1Jk%M}rvP0CV3Ni5t!qqKDl zu%^&hu4MjYPjVW4bu+oU9EAL#G)TkgAn^R=es=$Ro`#*dp`L23bYBPve_||83Tp=$ zPlN`-HgmE9mZ)w#4gUsESPe0FOi{Uof=dQ#Fb=(JCG82Wd8vn^4SHOVZB`Z9?@?h@ z0kKy@NS9s8yeFrrZBtlb-is4#Q7L142dyStpl;KO{+PAZgxb#Ho}dy9^P>)CiltU{ z;nXkG`b?5EhAz_2yy*$@f5Vq4wd7cjXECg6kwkEmfJxX+rQh%&z+`WaGyAVWPyzva z5ijGagPPU*uoPN7nKY`S_z$z^6_Nqw97TATIj@kUnBxyJ)!L@*o06sxt$a*rYXLF- zIla6T9Hdu7@z67GYT08>w+8QVWDwX&v1Ha$yLvY1sl7=jo%_F}e>UD)E=4?x^rp34Jn|hk}JS729r=I^;p!V?3sxH&wwfh#ex|4_B9FV8j6DzOrZxd z8S9Q!Now3{vcw;&xh#`OcvBF@u#205s#aRPIfsJFrAo$YzzuL=kR}ORwKb82f z>c(X_!`gI$b9q|Hf7g3S5*s%umx>)`X!?bQnbMjA?q3UflN|E%gMGl@Bi|GLL@cc^ z!QO~1e&oio3#Mp3+Rx>a*>$dA33${}$gt?KToz~&mQ1ufmIvHrywYNF?<~p~S^2O> z_)gTRq!eqy;n2xoc)lWr9ot}i0Ta;vkkU$d8jJNk{vKkka?J&krJ}*Vi1|t_5ZNakF(a`(s+#* N#LVt2R^$KC3DN6?O0ctv1dx zz?MPlgjVI3BguL6&E4$bdKB`9HlVCvdEkYAG;?FL843YOtrJ6b(Zb>)5 zGtEus@!tXp>oE>b85(y$gyOIT>(T2@v4P-5RAxNdV#I{*D}$J@N5p9YX;foK*ZsD? z7biGUYwHFF*ia$Sw(aC_%+?vo9N9KEw8e`H~MlVP{V0CS_|ufnyyvYFky4+}B++2&EbNZ&Yr{zH=E zPUw2wxwad+g|?NQddfMkBVzhSfJ^FqXlL*Q!7vsurjF`sl9 z{<20%=Y+I5nGAsGjeb^AQ=V^YI`=I%VoFCe(`p0Qe^@V4>nGCKe`;XSL_7zkIgk~# z@a=0B(w~(cM!3cZR5sS17>YJPEM4P|GeT9_B%&phbDU0=Fx@F#m`SIMIN0v}t{Xb8I6oxRmGcl0_tJ&ZsZv^9u2W5i zb$#%t;B*txp5S{?Z;~-=2am%bqvKt8y?>m2nm;a!`013wL!<%x8}H8s6Q#IEgwLJg z^un6pP8K5mr|qKXz3sBkwAcu7|vJ;Poa!MFage>kf3W)h8K8cJO38XP$^ zY{#U$Z$Xt-cZNaqyX-GgXkG69f(hPigzfNi3}4X>#wlgQpO`z!_kJBE#Z!nU1w=`o zzmREHLmxM8jjdOj8g2`}MYuYV!e8JMf|G^F^H@{#e1+|~UoFGx_W}pmnMAg`$$lXc z+{FnPQ`+B0J`$5r5>=Pj7N{CUwn(Vnu(8(*jw>^#!e<=5YF&w$*I9d!?@c diff --git a/output.html b/output.html index 8cb25b7..95893ba 100644 --- a/output.html +++ b/output.html @@ -1,12 +1,14 @@ - + - Sample outputs — parsedmarc 8.6.4 documentation - - + Sample outputs — parsedmarc 8.7.0 documentation + + + + @@ -36,7 +38,7 @@ parsedmarc
      - 8.6.4 + 8.7.0
    @@ -284,6 +287,49 @@ auth-failure,Lua/1.0,1.0,,sharepoint@domain.de,peter.pan@domain.de,"Mon, 01
    +
    +

    JSON SMTP TLS report

    +
    [
+  {
+    "organization_name": "Example Inc.",
+    "begin_date": "2024-01-09T00:00:00Z",
+    "end_date": "2024-01-09T23:59:59Z",
+    "report_id": "2024-01-09T00:00:00Z_example.com",
+    "policies": [
+      {
+        "policy_domain": "example.com",
+        "policy_type": "sts",
+        "policy_strings": [
+          "version: STSv1",
+          "mode: testing",
+          "mx: example.com",
+          "max_age: 86400"
+        ],
+        "successful_session_count": 0,
+        "failed_session_count": 3,
+        "failure_details": [
+          {
+            "result_type": "validation-failure",
+            "failed_session_count": 2,
+            "sending_mta_ip": "209.85.222.201",
+            "receiving_ip": "173.212.201.41",
+            "receiving_mx_hostname": "example.com"
+          },
+          {
+            "result_type": "validation-failure",
+            "failed_session_count": 1,
+            "sending_mta_ip": "209.85.208.176",
+            "receiving_ip": "173.212.201.41",
+            "receiving_mx_hostname": "example.com"
+          }
+        ]
+      }
+    ]
+  }
+]
+
    +
    +
    diff --git a/py-modindex.html b/py-modindex.html index 6e958cd..782e46a 100644 --- a/py-modindex.html +++ b/py-modindex.html @@ -1,11 +1,13 @@ - + - Python Module Index — parsedmarc 8.6.4 documentation - - + Python Module Index — parsedmarc 8.7.0 documentation + + + + @@ -36,7 +38,7 @@ parsedmarc
    - 8.6.4 + 8.7.0
    diff --git a/search.html b/search.html index b1f2c09..627cf26 100644 --- a/search.html +++ b/search.html @@ -1,11 +1,13 @@ - + - Search — parsedmarc 8.6.4 documentation - - + Search — parsedmarc 8.7.0 documentation + + + + @@ -36,7 +38,7 @@ parsedmarc
    - 8.6.4 + 8.7.0
    diff --git a/usage.html b/usage.html index 0211a3f..c3f29dd 100644 --- a/usage.html +++ b/usage.html @@ -1,12 +1,14 @@ - + - Using parsedmarc — parsedmarc 8.6.4 documentation - - + Using parsedmarc — parsedmarc 8.7.0 documentation + + + + @@ -36,7 +38,7 @@ parsedmarc
    - 8.6.4 + 8.7.0
    @@ -227,10 +229,10 @@ Gmail) where the incoming reports can be found (Default: INBOX)

  • archive_folder - str: The mailbox folder (or label for Gmail) to sort processed emails into (Default: Archive)

    • -

  • watch - bool: Use the IMAP IDLE command to process

    • -

  • messages as they arrive or poll MS Graph for new messages

    • -

  • delete - bool: Delete messages after processing them,

    • -

  • instead of archiving them

    • +

  • watch - bool: Use the IMAP IDLE command to process +messages as they arrive or poll MS Graph for new messages

    • +

  • delete - bool: Delete messages after processing them, +instead of archiving them

  • test - bool: Do not move or delete messages

  • batch_size - int: Number of messages to read and process before saving. Default 10. Use 0 for no limit.

    • @@ -325,8 +327,12 @@ or URLs (e.g. 127.0 URL encoded.

    +

  • user - str: Basic auth username

    • +

  • password - str: Basic auth password

    • +

  • apiKey - str: API key

  • ssl - bool: Use an encrypted SSL/TLS connection (Default: True)

    • +

  • timeout - float: Timeout in seconds (Default: 60)

  • cert_path - str: Path to a trusted certificates

  • index_suffix - str: A suffix to apply to the index names

  • monthly_indexes - bool: Use monthly indexes instead of daily indexes

    • @@ -410,6 +416,8 @@ acquiring credentials (Default: https://www.googleapis.com/auth/gmail.modify)

  • oauth2_port - int: The TCP port for the local server to listen on for the OAuth2 response (Default: 8080)

    • +

  • paginate_messages - bool: When True, fetch all applicable Gmail messages. +When False, only fetch up to 100 new messages per run (Default: True)

  • log_analytics

    @@ -421,6 +429,7 @@ listen on for the OAuth2 response (Default: dcr_immutable_id - str: The immutable ID of the Data Collection Rule (DCR)

  • dcr_aggregate_stream - str: The stream name for aggregate reports in the DCR

  • dcr_forensic_stream - str: The stream name for the forensic reports in the DCR

    • +

  • dcr_smtp_tls_stream - str: The stream name for the SMTP TLS reports in the DCR

    • Note