diff --git a/_modules/index.html b/_modules/index.html index 5ac9f1b6..e0394e01 100644 --- a/_modules/index.html +++ b/_modules/index.html @@ -5,14 +5,14 @@ - Overview: module code — parsedmarc 10.2.4 documentation + Overview: module code — parsedmarc 10.3.0 documentation - + diff --git a/_modules/parsedmarc.html b/_modules/parsedmarc.html index 3cd22f6a..4d27bef2 100644 --- a/_modules/parsedmarc.html +++ b/_modules/parsedmarc.html @@ -5,14 +5,14 @@ - parsedmarc — parsedmarc 10.2.4 documentation + parsedmarc — parsedmarc 10.3.0 documentation - + @@ -115,6 +115,7 @@ import xmltodict from expiringdict import ExpiringDict from mailsuite.smtp import send_email +from tqdm import tqdm from parsedmarc.constants import ( DEFAULT_DNS_MAX_RETRIES, @@ -147,7 +148,7 @@ timestamp_to_human, ) -logger.debug("parsedmarc v{0}".format(__version__)) +logger.debug(f"parsedmarc v{__version__}") feedback_report_regex = re.compile(r"^([\w\-]+): (.+)$", re.MULTILINE) xml_header_regex = re.compile(r"^<\?xml .*?>", re.MULTILINE) @@ -254,7 +255,7 @@ if not frames: return "" last = frames[-1] - return " (raised at {0}:{1})".format(last.filename, last.lineno) + return f" (raised at {last.filename}:{last.lineno})" def _text(value: Any) -> str | None: @@ -930,7 +931,7 @@ try: xmltodict.parse(xml)["feedback"] except Exception as e: - errors.append("Invalid XML: {0}".format(e.__str__())) + errors.append(f"Invalid XML: {e.__str__()}") try: tree = etree.parse( BytesIO(xml.encode("utf-8")), @@ -978,13 +979,10 @@ if new_org_name is not None: org_name = new_org_name if not org_name: - logger.debug( - "Could not parse org_name from XML.\r\n{0}".format(report.__str__()) - ) + logger.debug(f"Could not parse org_name from XML.\r\n{report.__str__()}") raise KeyError( - "Organization name is missing. \ - This field is a requirement for \ - saving the report" + "Organization name is missing. This field is a requirement " + "for saving the report" ) new_report_metadata["org_name"] = org_name new_report_metadata["org_email"] = report_metadata["email"] @@ -1083,14 +1081,14 @@ if policy_published["np"] is not None: np_ = policy_published["np"] if np_ not in ("none", "quarantine", "reject"): - logger.warning("Invalid np value: {0}".format(np_)) + logger.warning(f"Invalid np value: {np_}") new_policy_published["np"] = np_ testing = None if "testing" in policy_published: if policy_published["testing"] is not None: testing = policy_published["testing"] if testing not in ("n", "y"): - logger.warning("Invalid testing value: {0}".format(testing)) + logger.warning(f"Invalid testing value: {testing}") new_policy_published["testing"] = testing discovery_method = None if "discovery_method" in policy_published: @@ -1098,7 +1096,7 @@ discovery_method = policy_published["discovery_method"] if discovery_method not in ("psl", "treewalk"): logger.warning( - "Invalid discovery_method value: {0}".format(discovery_method) + f"Invalid discovery_method value: {discovery_method}" ) new_policy_published["discovery_method"] = discovery_method new_report["policy_published"] = new_policy_published @@ -1108,7 +1106,7 @@ if keep_alive is not None and i > 0 and i % 20 == 0: logger.debug("Sending keepalive cmd") keep_alive() - logger.debug("Processed {0}/{1}".format(i, len(report["record"]))) + logger.debug("Processed {}/{}".format(i, len(report["record"]))) try: report_record = _parse_report_record( report["record"][i], @@ -1130,7 +1128,7 @@ normalize=normalize_timespan, ) except Exception as e: - logger.warning("Could not parse record: {0}".format(e)) + logger.warning(f"Could not parse record: {e}") else: report_record = _parse_report_record( @@ -1158,20 +1156,16 @@ return cast(AggregateReport, new_report) except expat.ExpatError as error: - raise InvalidAggregateReport( - "Invalid XML: {0}".format(error.__str__()) - ) from error + raise InvalidAggregateReport(f"Invalid XML: {error.__str__()}") from error except KeyError as error: - raise InvalidAggregateReport( - "Missing field: {0}".format(error.__str__()) - ) from error + raise InvalidAggregateReport(f"Missing field: {error.__str__()}") from error except AttributeError as error: raise InvalidAggregateReport("Report missing required section") from error except Exception as error: raise InvalidAggregateReport( - "Unexpected error: {0}{1}".format(error.__str__(), _exc_origin(error)) + f"Unexpected error: {error.__str__()}{_exc_origin(error)}" ) from error @@ -1252,7 +1246,7 @@ except Exception as error: raise ParserError( - "Invalid archive file: {0}{1}".format(error.__str__(), _exc_origin(error)) + f"Invalid archive file: {error.__str__()}{_exc_origin(error)}" ) from error finally: if file_object: @@ -1731,13 +1725,11 @@ return cast(FailureReport, parsed_report) except KeyError as error: - raise InvalidFailureReport( - "Missing value: {0}".format(error.__str__()) - ) from error + raise InvalidFailureReport(f"Missing value: {error.__str__()}") from error except Exception as error: raise InvalidFailureReport( - "Unexpected error: {0}{1}".format(error.__str__(), _exc_origin(error)) + f"Unexpected error: {error.__str__()}{_exc_origin(error)}" ) from error @@ -1922,7 +1914,7 @@ sample = None is_feedback_report: bool = False if "From" in msg_headers: - logger.info("Parsing mail from {0} on {1}".format(msg_headers["From"], date)) + logger.info("Parsing mail from {} on {}".format(msg_headers["From"], date)) if "Subject" in msg_headers: subject = msg_headers["Subject"] for part in msg.walk(): @@ -1972,9 +1964,7 @@ fields["received-date"], fields["sender-ip-address"] ) except Exception as e: - error = 'Unable to parse message with subject "{0}": {1}{2}'.format( - subject, e, _exc_origin(e) - ) + error = f'Unable to parse message with subject "{subject}": {e}{_exc_origin(e)}' raise InvalidDMARCReport(error) from e sample = parts[1].lstrip() @@ -2016,15 +2006,12 @@ except InvalidDMARCReport as e: error = ( - 'Message with subject "{0}" is not a valid ' - "DMARC report: {1}".format(subject, e) + f'Message with subject "{subject}" is not a valid DMARC report: {e}' ) raise ParserError(error) from e except Exception as e: - error = 'Unable to parse message with subject "{0}": {1}{2}'.format( - subject, e, _exc_origin(e) - ) + error = f'Unable to parse message with subject "{subject}": {e}{_exc_origin(e)}' raise ParserError(error) from e if feedback_report and sample: @@ -2045,9 +2032,9 @@ ) except InvalidFailureReport as e: error = ( - 'Message with subject "{0}" ' + f'Message with subject "{subject}" ' "is not a valid " - "failure DMARC report: {1}".format(subject, e) + f"failure DMARC report: {e}" ) raise InvalidFailureReport(error) from e @@ -2055,7 +2042,7 @@ return result if result is None: - error = 'Message with subject "{0}" is not a valid report'.format(subject) + error = f'Message with subject "{subject}" is not a valid report' raise InvalidDMARCReport(error) return result @@ -2099,11 +2086,11 @@ sniff = sniff.lstrip() if sniff.startswith("<"): - return "Invalid aggregate report: {0}".format(aggregate_error) + return f"Invalid aggregate report: {aggregate_error}" if sniff.startswith("{"): - return "Invalid SMTP TLS report: {0}".format(smtp_tls_error) + return f"Invalid SMTP TLS report: {smtp_tls_error}" if _looks_like_email(sniff): - return "Invalid report email: {0}".format(email_error) + return f"Invalid report email: {email_error}" return ( "Not a recognized report format (not a DMARC aggregate XML report, " "an SMTP TLS JSON report, or a DMARC report email)" @@ -2153,7 +2140,7 @@ file_object: BinaryIO if isinstance(input_, (str, os.PathLike)): file_path = os.fspath(input_) - logger.debug("Parsing {0}".format(file_path)) + logger.debug(f"Parsing {file_path}") file_object = open(file_path, "rb") elif isinstance(input_, (bytes, bytearray, memoryview)): file_object = BytesIO(bytes(input_)) @@ -2265,10 +2252,10 @@ mbox = mailbox.mbox(input_) message_keys = mbox.keys() total_messages = len(message_keys) - logger.debug("Found {0} messages in {1}".format(total_messages, input_)) - for i in range(len(message_keys)): + logger.debug(f"Found {total_messages} messages in {input_}") + for i in tqdm(range(total_messages), disable=None): message_key = message_keys[i] - logger.info("Processing message {0} of {1}".format(i + 1, total_messages)) + logger.info(f"Processing message {i + 1} of {total_messages}") msg_content = mbox.get_string(message_key) try: sa = strip_attachment_payloads @@ -2304,7 +2291,7 @@ except InvalidDMARCReport as error: logger.warning(error.__str__()) except mailbox.NoSuchMailboxError: - raise InvalidDMARCReport("Mailbox {0} does not exist".format(input_)) + raise InvalidDMARCReport(f"Mailbox {input_} does not exist") return { "aggregate_reports": aggregate_reports, "failure_reports": failure_reports, @@ -2331,8 +2318,8 @@ (warn, don't crash). Uses the folder-management API added in mailsuite 2.1.0 (``folder_exists`` / ``rename_folder`` / ``merge_folders``). """ - old_folder = "{0}/Forensic".format(archive_folder) - new_folder = "{0}/Failure".format(archive_folder) + old_folder = f"{archive_folder}/Forensic" + new_folder = f"{archive_folder}/Failure" try: if not connection.folder_exists(old_folder): return @@ -2341,23 +2328,13 @@ # created Failure folder): move the legacy folder's messages into # the new one and drop the now-empty legacy folder. connection.merge_folders(old_folder, new_folder) - logger.info( - "Merged legacy archive folder {0} into {1}".format( - old_folder, new_folder - ) - ) + logger.info(f"Merged legacy archive folder {old_folder} into {new_folder}") else: connection.rename_folder(old_folder, new_folder) - logger.info( - "Renamed legacy archive folder {0} to {1}".format( - old_folder, new_folder - ) - ) + logger.info(f"Renamed legacy archive folder {old_folder} to {new_folder}") except Exception as error: logger.warning( - "Could not migrate legacy archive folder {0} to {1}: {2}".format( - old_folder, new_folder, error - ) + f"Could not migrate legacy archive folder {old_folder} to {new_folder}: {error}" ) @@ -2432,10 +2409,10 @@ aggregate_report_msg_uids = [] failure_report_msg_uids = [] smtp_tls_msg_uids = [] - aggregate_reports_folder = "{0}/Aggregate".format(archive_folder) - failure_reports_folder = "{0}/Failure".format(archive_folder) - smtp_tls_reports_folder = "{0}/SMTP-TLS".format(archive_folder) - invalid_reports_folder = "{0}/Invalid".format(archive_folder) + aggregate_reports_folder = f"{archive_folder}/Aggregate" + failure_reports_folder = f"{archive_folder}/Failure" + smtp_tls_reports_folder = f"{archive_folder}/SMTP-TLS" + invalid_reports_folder = f"{archive_folder}/Invalid" if results: aggregate_reports = results["aggregate_reports"].copy() @@ -2464,17 +2441,17 @@ _since = int(s[1]) * 60 * 24 * 7 else: logger.warning( - "Incorrect format for 'since' option. \ - Provided value:{0}, Expected values:(5m|3h|2d|1w). \ - Ignoring option, fetching messages for last 24hrs" - "SMTP does not support a time or timezone in since." - "See https://www.rfc-editor.org/rfc/rfc3501#page-52".format(since) + f"Incorrect format for 'since' option. Provided value: {since}, " + "expected values: (5m|3h|2d|1w). Ignoring option, fetching " + "messages for last 24hrs. SMTP does not support a time or " + "timezone in since. See " + "https://www.rfc-editor.org/rfc/rfc3501#page-52" ) if isinstance(connection, IMAPConnection): logger.debug( - "Only days and weeks values in 'since' option are \ - considered for IMAP connections. Examples: 2d or 1w" + "Only days and weeks values in 'since' option are considered " + "for IMAP connections. Examples: 2d or 1w" ) since = (datetime.now(timezone.utc) - timedelta(minutes=_since)).strftime( "%d-%b-%Y" @@ -2495,22 +2472,18 @@ reports_folder, batch_size=batch_size, since=since ) total_messages = len(messages) - logger.debug("Found {0} messages in {1}".format(len(messages), reports_folder)) + logger.debug(f"Found {len(messages)} messages in {reports_folder}") if batch_size and not since: message_limit = min(total_messages, batch_size) else: message_limit = total_messages - logger.debug("Processing {0} messages".format(message_limit)) + logger.debug(f"Processing {message_limit} messages") for i in range(message_limit): msg_uid = messages[i] - logger.debug( - "Processing message {0} of {1}: UID {2}".format( - i + 1, message_limit, msg_uid - ) - ) + logger.debug(f"Processing message {i + 1} of {message_limit}: UID {msg_uid}") message_id: int | str if isinstance(connection, IMAPConnection): message_id = int(msg_uid) @@ -2562,16 +2535,14 @@ logger.warning(error.__str__()) if not test: if delete: - logger.debug("Deleting message UID {0}".format(msg_uid)) + logger.debug(f"Deleting message UID {msg_uid}") if isinstance(connection, IMAPConnection): connection.delete_message(int(message_id)) else: connection.delete_message(str(message_id)) else: logger.debug( - "Moving message UID {0} to {1}".format( - msg_uid, invalid_reports_folder - ) + f"Moving message UID {msg_uid} to {invalid_reports_folder}" ) if isinstance(connection, IMAPConnection): connection.move_message(int(message_id), invalid_reports_folder) @@ -2588,81 +2559,63 @@ for i in range(number_of_processed_msgs): msg_uid = processed_messages[i] logger.debug( - "Deleting message {0} of {1}: UID {2}".format( - i + 1, number_of_processed_msgs, msg_uid - ) + f"Deleting message {i + 1} of {number_of_processed_msgs}: UID {msg_uid}" ) try: connection.delete_message(msg_uid) except Exception as e: message = "Error deleting message UID" - e = "{0} {1}: {2}".format(message, msg_uid, e) - logger.error("Mailbox error: {0}".format(e)) + e = f"{message} {msg_uid}: {e}" + logger.error(f"Mailbox error: {e}") else: if len(aggregate_report_msg_uids) > 0: log_message = "Moving aggregate report messages from" logger.debug( - "{0} {1} to {2}".format( - log_message, reports_folder, aggregate_reports_folder - ) + f"{log_message} {reports_folder} to {aggregate_reports_folder}" ) number_of_agg_report_msgs = len(aggregate_report_msg_uids) for i in range(number_of_agg_report_msgs): msg_uid = aggregate_report_msg_uids[i] logger.debug( - "Moving message {0} of {1}: UID {2}".format( - i + 1, number_of_agg_report_msgs, msg_uid - ) + f"Moving message {i + 1} of {number_of_agg_report_msgs}: UID {msg_uid}" ) try: connection.move_message(msg_uid, aggregate_reports_folder) except Exception as e: message = "Error moving message UID" - e = "{0} {1}: {2}".format(message, msg_uid, e) - logger.error("Mailbox error: {0}".format(e)) + e = f"{message} {msg_uid}: {e}" + logger.error(f"Mailbox error: {e}") if len(failure_report_msg_uids) > 0: message = "Moving failure report messages from" - logger.debug( - "{0} {1} to {2}".format( - message, reports_folder, failure_reports_folder - ) - ) + logger.debug(f"{message} {reports_folder} to {failure_reports_folder}") number_of_failure_msgs = len(failure_report_msg_uids) for i in range(number_of_failure_msgs): msg_uid = failure_report_msg_uids[i] message = "Moving message" logger.debug( - "{0} {1} of {2}: UID {3}".format( - message, i + 1, number_of_failure_msgs, msg_uid - ) + f"{message} {i + 1} of {number_of_failure_msgs}: UID {msg_uid}" ) try: connection.move_message(msg_uid, failure_reports_folder) except Exception as e: - e = "Error moving message UID {0}: {1}".format(msg_uid, e) - logger.error("Mailbox error: {0}".format(e)) + e = f"Error moving message UID {msg_uid}: {e}" + logger.error(f"Mailbox error: {e}") if len(smtp_tls_msg_uids) > 0: message = "Moving SMTP TLS report messages from" - logger.debug( - "{0} {1} to {2}".format( - message, reports_folder, smtp_tls_reports_folder - ) - ) + logger.debug(f"{message} {reports_folder} to {smtp_tls_reports_folder}") number_of_smtp_tls_uids = len(smtp_tls_msg_uids) for i in range(number_of_smtp_tls_uids): msg_uid = smtp_tls_msg_uids[i] message = "Moving message" logger.debug( - "{0} {1} of {2}: UID {3}".format( - message, i + 1, number_of_smtp_tls_uids, msg_uid - ) + f"{message} {i + 1} of {number_of_smtp_tls_uids}: UID {msg_uid}" ) try: connection.move_message(msg_uid, smtp_tls_reports_folder) except Exception as e: - e = "Error moving message UID {0}: {1}".format(msg_uid, e) - logger.error("Mailbox error: {0}".format(e)) + e = f"Error moving message UID {msg_uid}: {e}" + logger.error(f"Mailbox error: {e}") results = { "aggregate_reports": aggregate_reports, "failure_reports": failure_reports, @@ -2888,7 +2841,7 @@ if os.path.exists(output_directory): if not os.path.isdir(output_directory): - raise ValueError("{0} is not a directory".format(output_directory)) + raise ValueError(f"{output_directory} is not a directory") else: os.makedirs(output_directory) @@ -2935,11 +2888,11 @@ while filename in sample_filenames: message_count += 1 - filename = "{0} ({1})".format(subject, message_count) + filename = f"{subject} ({message_count})" sample_filenames.append(filename) - filename = "{0}.eml".format(filename) + filename = f"{filename}.eml" path = os.path.join(samples_directory, filename) with open(path, "w", newline="\n", encoding="utf-8") as sample_file: sample_file.write(sample) @@ -3013,12 +2966,12 @@ attachment_filename += ".zip" filename = attachment_filename else: - filename = "DMARC-{0}.zip".format(date_string) + filename = f"DMARC-{date_string}.zip" if subject is None: - subject = "DMARC results for {0}".format(date_string) + subject = f"DMARC results for {date_string}" if message is None: - message = "DMARC results for {0}".format(date_string) + message = f"DMARC results for {date_string}" zip_bytes = get_report_zip(results) attachments = [(filename, zip_bytes)] diff --git a/_modules/parsedmarc/elastic.html b/_modules/parsedmarc/elastic.html index 5a5163c1..64c75855 100644 --- a/_modules/parsedmarc/elastic.html +++ b/_modules/parsedmarc/elastic.html @@ -5,14 +5,14 @@ - parsedmarc.elastic — parsedmarc 10.2.4 documentation + parsedmarc.elastic — parsedmarc 10.3.0 documentation - + @@ -126,6 +126,181 @@ # settings (e.g. ``refresh_interval``) are accepted and pass through. _SERVERLESS_REJECTED_SETTINGS = frozenset({"number_of_shards", "number_of_replicas"}) +# Guard query for the dkim_results_combined/spf_results_combined backfill +# (see ``migrate_indexes``). Matches only documents that have at least one +# DKIM or SPF auth result and are missing the corresponding combined field. +# Empty arrays are invisible to ``exists``, so documents with zero +# DKIM/SPF results are correctly skipped (verified against real data; +# this also makes the query idempotent — a backfilled document no longer +# matches). Each result is matched on an OR of its ``domain``/``result`` +# subfields as defense in depth: the parsers we audited never store a +# result without both, but an empty string indexes no text tokens and is +# invisible to ``exists``, and the storage shape of every historical +# parsedmarc version can't be audited — matching either subfield costs +# nothing and cannot skip a document that has something to backfill. +_COMBINED_BACKFILL_QUERY: dict[str, Any] = { + "bool": { + "minimum_should_match": 1, + "should": [ + { + "bool": { + "must": [ + { + "bool": { + "minimum_should_match": 1, + "should": [ + {"exists": {"field": "dkim_results.domain"}}, + {"exists": {"field": "dkim_results.result"}}, + ], + } + } + ], + "must_not": [{"exists": {"field": "dkim_results_combined"}}], + } + }, + { + "bool": { + "must": [ + { + "bool": { + "minimum_should_match": 1, + "should": [ + {"exists": {"field": "spf_results.domain"}}, + {"exists": {"field": "spf_results.result"}}, + ], + } + } + ], + "must_not": [{"exists": {"field": "spf_results_combined"}}], + } + }, + ], + } +} + +# Painless script that (re)derives dkim_results_combined/spf_results_combined +# from dkim_results/spf_results, matching the format written by +# save_aggregate_report_to_elasticsearch(): "{selector} / {domain} / {result}" +# per DKIM result and "{scope} / {domain} / {result}" per SPF result. +_COMBINED_BACKFILL_SCRIPT = ( + "List dk = new ArrayList(); " + "def dr = ctx._source.dkim_results; " + "if (dr != null) { " + "if (!(dr instanceof List)) { dr = [dr]; } " + "for (e in dr) { " + "if (e == null) { continue; } " + 'def sel = e.selector != null ? e.selector : "none"; ' + 'def dom = e.domain != null ? e.domain : "none"; ' + 'def res = e.result != null ? e.result : "none"; ' + 'dk.add(sel + " / " + dom + " / " + res); ' + "} } " + "ctx._source.dkim_results_combined = dk; " + "List sp = new ArrayList(); " + "def sr = ctx._source.spf_results; " + "if (sr != null) { " + "if (!(sr instanceof List)) { sr = [sr]; } " + "for (e in sr) { " + "if (e == null) { continue; } " + 'def sc = e.scope != null ? e.scope : "mfrom"; ' + 'def dom = e.domain != null ? e.domain : "none"; ' + 'def res = e.result != null ? e.result : (e.results != null ? e.results : "none"); ' + 'sp.add(sc + " / " + dom + " / " + res); ' + "} } " + "ctx._source.spf_results_combined = sp;" +) + +# Guard query for the policies_combined/failure_details_combined backfill +# (see ``migrate_indexes``). Matches only SMTP TLS documents that have at +# least one policy or failure detail and are missing the corresponding +# combined field. Empty arrays are invisible to ``exists``, so documents +# with zero policies/failure details are correctly skipped (this also +# makes the query idempotent — a backfilled document no longer matches). +# Each result is matched on an OR of its relevant subfields as defense in +# depth: the parsers we audited never store a policy/failure detail +# without these fields, but an empty string indexes no text tokens and is +# invisible to ``exists``, and the storage shape of every historical +# parsedmarc version can't be audited — matching either subfield costs +# nothing and cannot skip a document that has something to backfill. +_SMTP_TLS_COMBINED_BACKFILL_QUERY: dict[str, Any] = { + "bool": { + "minimum_should_match": 1, + "should": [ + { + "bool": { + "must": [ + { + "bool": { + "minimum_should_match": 1, + "should": [ + {"exists": {"field": "policies.policy_domain"}}, + {"exists": {"field": "policies.policy_type"}}, + ], + } + } + ], + "must_not": [{"exists": {"field": "policies_combined"}}], + } + }, + { + "bool": { + "must": [ + { + "bool": { + "minimum_should_match": 1, + "should": [ + { + "exists": { + "field": "policies.failure_details.result_type" + } + }, + { + "exists": { + "field": "policies.failure_details.sending_mta_ip" + } + }, + ], + } + } + ], + "must_not": [{"exists": {"field": "failure_details_combined"}}], + } + }, + ], + } +} + +# Painless script that (re)derives policies_combined/failure_details_combined +# from policies/policies.failure_details, matching the format written by +# save_smtp_tls_report_to_elasticsearch(): "{policy_domain} / {policy_type}" +# per policy and "{policy_domain} / {policy_type} / {result_type} / +# {sending_mta_ip} / {receiving_ip} / {receiving_mx_hostname}" per failure +# detail. +_SMTP_TLS_COMBINED_BACKFILL_SCRIPT = ( + "List pols = new ArrayList(); " + "List dets = new ArrayList(); " + "def ps = ctx._source.policies; " + "if (ps != null) { " + "if (!(ps instanceof List)) { ps = [ps]; } " + "for (p in ps) { " + "if (p == null) { continue; } " + 'def dom = p.policy_domain != null ? p.policy_domain : "none"; ' + 'def typ = p.policy_type != null ? p.policy_type : "none"; ' + 'pols.add(dom + " / " + typ); ' + "def fds = p.failure_details; " + "if (fds != null) { " + "if (!(fds instanceof List)) { fds = [fds]; } " + "for (f in fds) { " + "if (f == null) { continue; } " + 'def rt = f.result_type != null ? f.result_type : "none"; ' + 'def smi = f.sending_mta_ip != null ? f.sending_mta_ip : "none"; ' + 'def ri = f.receiving_ip != null ? f.receiving_ip : "none"; ' + 'def rmh = f.receiving_mx_hostname != null ? f.receiving_mx_hostname : "none"; ' + 'dets.add(dom + " / " + typ + " / " + rt + " / " + smi + " / " + ri + " / " + rmh); ' + "} } } } " + "ctx._source.policies_combined = pols; " + "ctx._source.failure_details_combined = dets;" +) + class _PolicyOverride(InnerDoc): # The elasticsearch.dsl 8.x type stubs use dataclass_transform and only @@ -229,6 +404,11 @@ header_from = Text() envelope_from = Text() envelope_to = Text() + # Nested(...) on the two auth-result fields below is only the DSL's + # in-memory document shape; it is never installed as a mapping. + # create_indexes() deliberately skips Index.document() registration so + # these fields stay dynamic-mapped as plain `object` in the cluster + # (see the comment there and issue #169). dkim_results = Nested(_DKIMResult) spf_results = Nested(_SPFResult) # One "{selector} / {domain} / {result}" (DKIM) or "{scope} / {domain} / @@ -266,9 +446,7 @@ human_result=human_result, ) ) - self.dkim_results_combined.append( - "{0} / {1} / {2}".format(selector, domain, result) - ) + self.dkim_results_combined.append(f"{selector} / {domain} / {result}") def add_spf_result( self, @@ -285,9 +463,7 @@ human_result=human_result, ) ) - self.spf_results_combined.append( - "{0} / {1} / {2}".format(scope, domain, result) - ) + self.spf_results_combined.append(f"{scope} / {domain} / {result}") def save(self, **kwargs): # pyright: ignore[reportIncompatibleMethodOverride] self.passed_dmarc = False @@ -404,6 +580,7 @@ result_type = Text() sending_mta_ip = Ip() receiving_mx_helo = Text() + receiving_mx_hostname = Text() receiving_ip = Ip() failed_session_count = Integer() additional_information_uri = Text() @@ -445,7 +622,7 @@ receiving_mx_helo=receiving_mx_helo, receiving_ip=receiving_ip, failed_session_count=failed_session_count, - additional_information=additional_information_uri, + additional_information_uri=additional_information_uri, failure_reason_code=failure_reason_code, ) self.failure_details.append(_details) @@ -468,6 +645,19 @@ contact_info = Text() report_id = Text() policies = Nested(_SMTPTLSPolicyDoc) + # One "{policy_domain} / {policy_type}" string per policy. Kibana/ + # Grafana tables cannot terms-aggregate the subfields of an object + # array without producing a cross-product of values (issue #169), so + # dashboards aggregate these composed keywords instead. Declared to + # match what dynamic mapping produces for a string array (text + + # .keyword). + policies_combined = Text(multi=True, fields={"keyword": Keyword(ignore_above=256)}) + # One "{policy_domain} / {policy_type} / {result_type} / + # {sending_mta_ip} / {receiving_ip} / {receiving_mx_hostname}" string + # per failure detail, across all policies. + failure_details_combined = Text( + multi=True, fields={"keyword": Keyword(ignore_above=256)} + )
@@ -516,9 +706,7 @@ if not isinstance(hosts, list): hosts = [hosts] scheme = "https://" if use_ssl else "http://" - normalized_hosts = [ - host if "://" in host else "{0}{1}".format(scheme, host) for host in hosts - ] + normalized_hosts = [host if "://" in host else f"{scheme}{host}" for host in hosts] conn_params = {"hosts": normalized_hosts, "request_timeout": timeout} if use_ssl: if ssl_cert_path: @@ -561,18 +749,25 @@ for name in names: index = Index(name) try: - # Deliberately no Index.document() registration: Kibana/OpenSearch - # Dashboards/Grafana cannot terms-aggregate fields inside a - # `nested` mapping, so the dynamic `object` mapping produced by a - # bare create is load-bearing for the shipped dashboards (issue - # #169; see the *_combined fields on _AggregateReportDoc). + # Deliberately no Index.document() registration: the shipped + # dashboards cannot rebuild their detail tables on a `nested` + # mapping — Kibana/OSD visual editors do not support nested + # fields, Vega can run nested aggregations but does not render + # tables, and Grafana's nested bucket aggregation (9.4+) lacks + # reverse_nested for parent-level metrics like message_count — + # so the dynamic `object` mapping produced by a bare create is + # load-bearing for the shipped dashboards. _AggregateReportDoc + # still declares dkim_results/spf_results with Nested(...), but + # that is only the DSL's in-memory shape for building documents + # — it is never installed as a mapping. See issue #169 and the + # *_combined fields on _AggregateReportDoc. if not index.exists(): - logger.debug("Creating Elasticsearch index: {0}".format(name)) + logger.debug(f"Creating Elasticsearch index: {name}") if effective_settings: index.settings(**effective_settings) index.create() except Exception as e: - raise ElasticsearchError("Elasticsearch error: {0}".format(e.__str__()))
+ raise ElasticsearchError(f"Elasticsearch error: {e.__str__()}") @@ -581,25 +776,118 @@ def migrate_indexes( aggregate_indexes: list[str] | None = None, failure_indexes: list[str] | None = None, + smtp_tls_indexes: list[str] | None = None, ): """ - Updates index mappings + Backfills the ``dkim_results_combined``/``spf_results_combined`` fields + (added for issue #169) on aggregate report documents, and the + ``policies_combined``/``failure_details_combined`` fields on SMTP TLS + report documents, that were saved before those fields existed. - This is a no-op kept for API compatibility (``cli.py`` calls it on - startup). The only migration this function ever performed was - re-typing ``published_policy.fo`` from ``long`` to ``text``, which - applied exclusively to indices still carrying the legacy - Elasticsearch 6-era ``"doc"`` mapping type. The 8.x client can only - reach servers (Elasticsearch 8.x/9.x) whose indices were created on - Elasticsearch 7.x or later and are therefore typeless, so that - migration path is unreachable and has been removed. + For each name in ``aggregate_indexes``/``smtp_tls_indexes``, this + submits an ``update_by_query`` against the ``f"{name}*"`` index pattern + (the real indexes are date-suffixed) as a non-blocking background task + (``wait_for_completion=False``), so it never delays parsedmarc startup. + Submission is guarded by a cheap ``count`` query that only matches + documents with DKIM/SPF results (or policies/failure details) but no + combined field, so once an index is fully backfilled, later calls are a + fast no-op. Any error talking to the cluster (e.g. no indexes yet on a + fresh install, or a transient connection issue) is caught and logged as + a warning rather than raised; the backfill is simply retried on the + next startup, and the manual ``_update_by_query`` commands documented in + ``docs/source/elasticsearch.md`` remain available in the meantime. Args: aggregate_indexes (list): A list of aggregate index names - (accepted for API compatibility; unused) failure_indexes (list): A list of failure index names (accepted for API compatibility; unused) - """ + smtp_tls_indexes (list): A list of SMTP TLS index names + """ + if not aggregate_indexes and not smtp_tls_indexes: + return + + try: + client = connections.get_connection() + except Exception as e: + logger.warning( + "Skipping the dkim_results_combined/spf_results_combined/" + "policies_combined/failure_details_combined backfill: could " + f"not get an Elasticsearch connection: {e}. This will be " + "retried at the next startup." + ) + return + for name in aggregate_indexes or []: + pattern = f"{name}*" + try: + count_response = client.count( + index=pattern, + query=_COMBINED_BACKFILL_QUERY, + ignore_unavailable=True, + allow_no_indices=True, + ) + count = count_response["count"] + if not count: + continue + update_response = client.update_by_query( + index=pattern, + query=_COMBINED_BACKFILL_QUERY, + script={"source": _COMBINED_BACKFILL_SCRIPT, "lang": "painless"}, + conflicts="proceed", + wait_for_completion=False, + ignore_unavailable=True, + allow_no_indices=True, + ) + task_id = update_response.get("task") + logger.info( + "Backfilling dkim_results_combined/spf_results_combined on " + f"{count} existing documents in {pattern} (task {task_id})" + ) + except Exception as e: + logger.warning( + "Failed to check/submit the dkim_results_combined/" + f"spf_results_combined backfill for {pattern}: {e}. This " + "will be retried at the next startup; the manual " + "_update_by_query command in the documentation remains " + "available in the meantime." + ) + + for name in smtp_tls_indexes or []: + pattern = f"{name}*" + try: + count_response = client.count( + index=pattern, + query=_SMTP_TLS_COMBINED_BACKFILL_QUERY, + ignore_unavailable=True, + allow_no_indices=True, + ) + count = count_response["count"] + if not count: + continue + update_response = client.update_by_query( + index=pattern, + query=_SMTP_TLS_COMBINED_BACKFILL_QUERY, + script={ + "source": _SMTP_TLS_COMBINED_BACKFILL_SCRIPT, + "lang": "painless", + }, + conflicts="proceed", + wait_for_completion=False, + ignore_unavailable=True, + allow_no_indices=True, + ) + task_id = update_response.get("task") + logger.info( + "Backfilling policies_combined/failure_details_combined on " + f"{count} existing documents in {pattern} (task {task_id})" + ) + except Exception as e: + logger.warning( + "Failed to check/submit the policies_combined/" + f"failure_details_combined backfill for {pattern}: {e}. " + "This will be retried at the next startup; the manual " + "_update_by_query command in the documentation remains " + "available in the meantime." + ) @@ -648,11 +936,11 @@ end_date_query = Q(dict(range=dict(date_end=dict(lte=end_date)))) # pyright: ignore[reportArgumentType] if index_suffix is not None: - search_index = "dmarc_aggregate_{0}*".format(index_suffix) + search_index = f"dmarc_aggregate_{index_suffix}*" else: search_index = "dmarc_aggregate*" if index_prefix is not None: - search_index = "{0}{1}".format(index_prefix, search_index) + search_index = f"{index_prefix}{search_index}" search = Search(index=search_index) query = org_name_query & report_id_query & domain_query query = query & begin_date_query & end_date_query @@ -667,18 +955,15 @@ existing = search.execute() except Exception as error_: raise ElasticsearchError( - "Elasticsearch's search for existing report \ - error: {}".format(error_.__str__()) + f"Elasticsearch's search for existing report error: {error_.__str__()}" ) if len(existing) > 0: raise AlreadySaved( - "An aggregate report ID {0} from {1} about {2} " - "with a date range of {3} UTC to {4} UTC already " + f"An aggregate report ID {report_id} from {org_name} about {domain} " + f"with a date range of {begin_date_human} UTC to {end_date_human} UTC already " "exists in " - "Elasticsearch".format( - report_id, org_name, domain, begin_date_human, end_date_human - ) + "Elasticsearch" ) published_policy = _PublishedPolicy( domain=aggregate_report["policy_published"]["domain"], @@ -771,11 +1056,11 @@ index = "dmarc_aggregate" if index_suffix: - index = "{0}_{1}".format(index, index_suffix) + index = f"{index}_{index_suffix}" if index_prefix: - index = "{0}{1}".format(index_prefix, index) + index = f"{index_prefix}{index}" - index = "{0}-{1}".format(index, index_date) + index = f"{index}-{index_date}" index_settings = dict( number_of_shards=number_of_shards, number_of_replicas=number_of_replicas ) @@ -785,7 +1070,7 @@ try: agg_doc.save() except Exception as e: - raise ElasticsearchError("Elasticsearch error: {0}".format(e.__str__())) + raise ElasticsearchError(f"Elasticsearch error: {e.__str__()}") @@ -836,12 +1121,12 @@ arrival_date_epoch_milliseconds = int(arrival_date.timestamp() * 1000) if index_suffix is not None: - search_index = "dmarc_failure_{0}*,dmarc_forensic_{0}*".format(index_suffix) + search_index = f"dmarc_failure_{index_suffix}*,dmarc_forensic_{index_suffix}*" else: search_index = "dmarc_failure*,dmarc_forensic*" if index_prefix is not None: search_index = ",".join( - "{0}{1}".format(index_prefix, part) for part in search_index.split(",") + f"{index_prefix}{part}" for part in search_index.split(",") ) search = Search(index=search_index) q = Q(dict(match=dict(arrival_date=arrival_date_epoch_milliseconds))) # pyright: ignore[reportArgumentType] @@ -893,8 +1178,8 @@ if len(existing) > 0: raise AlreadySaved( - "A failure sample to {0} from {1} " - "with a subject of {2} and arrival date of {3} " + "A failure sample to {} from {} " + "with a subject of {} and arrival date of {} " "already exists in " "Elasticsearch".format( to_, from_, subject, failure_report["arrival_date_utc"] @@ -955,14 +1240,14 @@ index = "dmarc_failure" if index_suffix: - index = "{0}_{1}".format(index, index_suffix) + index = f"{index}_{index_suffix}" if index_prefix: - index = "{0}{1}".format(index_prefix, index) + index = f"{index_prefix}{index}" if monthly_indexes: index_date = arrival_date.strftime("%Y-%m") else: index_date = arrival_date.strftime("%Y-%m-%d") - index = "{0}-{1}".format(index, index_date) + index = f"{index}-{index_date}" index_settings = dict( number_of_shards=number_of_shards, number_of_replicas=number_of_replicas ) @@ -971,10 +1256,10 @@ try: failure_doc.save() except Exception as e: - raise ElasticsearchError("Elasticsearch error: {0}".format(e.__str__())) + raise ElasticsearchError(f"Elasticsearch error: {e.__str__()}") except KeyError as e: raise InvalidFailureReport( - "Failure report missing required field: {0}".format(e.__str__()) + f"Failure report missing required field: {e.__str__()}" ) @@ -1024,11 +1309,11 @@ end_date_query = Q(dict(match=dict(date_end=end_date))) # pyright: ignore[reportArgumentType] if index_suffix is not None: - search_index = "smtp_tls_{0}*".format(index_suffix) + search_index = f"smtp_tls_{index_suffix}*" else: search_index = "smtp_tls*" if index_prefix is not None: - search_index = "{0}{1}".format(index_prefix, search_index) + search_index = f"{index_prefix}{search_index}" search = Search(index=search_index) query = org_name_query & report_id_query query = query & begin_date_query & end_date_query @@ -1038,8 +1323,7 @@ existing = search.execute() except Exception as error_: raise ElasticsearchError( - "Elasticsearch's search for existing report \ - error: {}".format(error_.__str__()) + f"Elasticsearch's search for existing report error: {error_.__str__()}" ) if len(existing) > 0: @@ -1053,10 +1337,10 @@ index = "smtp_tls" if index_suffix: - index = "{0}_{1}".format(index, index_suffix) + index = f"{index}_{index_suffix}" if index_prefix: - index = "{0}{1}".format(index_prefix, index) - index = "{0}-{1}".format(index, index_date) + index = f"{index_prefix}{index}" + index = f"{index}-{index_date}" index_settings = dict( number_of_shards=number_of_shards, number_of_replicas=number_of_replicas ) @@ -1077,6 +1361,16 @@ policy_strings = policy["policy_strings"] if "mx_host_patterns" in policy: mx_host_patterns = policy["mx_host_patterns"] + # policies_combined/failure_details_combined: see the field + # declarations on _SMTPTLSReportDoc and issue #169. policies and + # their failure_details are object arrays with the same + # cross-product problem as dkim_results/spf_results, so dashboards + # aggregate these composed strings instead of the raw subfields. + policy_domain_combined = policy.get("policy_domain") or "none" + policy_type_combined = policy.get("policy_type") or "none" + smtp_tls_doc.policies_combined.append( + f"{policy_domain_combined} / {policy_type_combined}" + ) policy_doc = _SMTPTLSPolicyDoc( policy_domain=policy["policy_domain"], policy_type=policy["policy_type"], @@ -1097,7 +1391,12 @@ if "receiving_mx_hostname" in failure_detail: receiving_mx_hostname = failure_detail["receiving_mx_hostname"] - if "additional_information_uri" in failure_detail: + # The parser's key is additional_info_uri (see + # SMTPTLSFailureDetailsOptional in types.py); accept the + # long-form key too for dicts built by other callers. + if "additional_info_uri" in failure_detail: + additional_information_uri = failure_detail["additional_info_uri"] + elif "additional_information_uri" in failure_detail: additional_information_uri = failure_detail[ "additional_information_uri" ] @@ -1122,6 +1421,16 @@ additional_information_uri=additional_information_uri, failure_reason_code=failure_reason_code, ) + smtp_tls_doc.failure_details_combined.append( + "{} / {} / {} / {} / {} / {}".format( + policy_domain_combined, + policy_type_combined, + failure_detail.get("result_type") or "none", + sending_mta_ip or "none", + receiving_ip or "none", + receiving_mx_hostname or "none", + ) + ) smtp_tls_doc.policies.append(policy_doc) create_indexes([index], index_settings) @@ -1130,7 +1439,7 @@ try: smtp_tls_doc.save() except Exception as e: - raise ElasticsearchError("Elasticsearch error: {0}".format(e.__str__())) + raise ElasticsearchError(f"Elasticsearch error: {e.__str__()}") diff --git a/_modules/parsedmarc/opensearch.html b/_modules/parsedmarc/opensearch.html index 054a4971..64ab6760 100644 --- a/_modules/parsedmarc/opensearch.html +++ b/_modules/parsedmarc/opensearch.html @@ -5,14 +5,14 @@ - parsedmarc.opensearch — parsedmarc 10.2.4 documentation + parsedmarc.opensearch — parsedmarc 10.3.0 documentation - + @@ -119,6 +119,182 @@ +# Guard query for the dkim_results_combined/spf_results_combined backfill +# (see ``migrate_indexes``). Matches only documents that have at least one +# DKIM or SPF auth result and are missing the corresponding combined field. +# Empty arrays are invisible to ``exists``, so documents with zero +# DKIM/SPF results are correctly skipped (verified against real data; +# this also makes the query idempotent — a backfilled document no longer +# matches). Each result is matched on an OR of its ``domain``/``result`` +# subfields as defense in depth: the parsers we audited never store a +# result without both, but an empty string indexes no text tokens and is +# invisible to ``exists``, and the storage shape of every historical +# parsedmarc version can't be audited — matching either subfield costs +# nothing and cannot skip a document that has something to backfill. +_COMBINED_BACKFILL_QUERY: dict[str, Any] = { + "bool": { + "minimum_should_match": 1, + "should": [ + { + "bool": { + "must": [ + { + "bool": { + "minimum_should_match": 1, + "should": [ + {"exists": {"field": "dkim_results.domain"}}, + {"exists": {"field": "dkim_results.result"}}, + ], + } + } + ], + "must_not": [{"exists": {"field": "dkim_results_combined"}}], + } + }, + { + "bool": { + "must": [ + { + "bool": { + "minimum_should_match": 1, + "should": [ + {"exists": {"field": "spf_results.domain"}}, + {"exists": {"field": "spf_results.result"}}, + ], + } + } + ], + "must_not": [{"exists": {"field": "spf_results_combined"}}], + } + }, + ], + } +} + +# Painless script that (re)derives dkim_results_combined/spf_results_combined +# from dkim_results/spf_results, matching the format written by +# save_aggregate_report_to_opensearch(): "{selector} / {domain} / {result}" +# per DKIM result and "{scope} / {domain} / {result}" per SPF result. +_COMBINED_BACKFILL_SCRIPT = ( + "List dk = new ArrayList(); " + "def dr = ctx._source.dkim_results; " + "if (dr != null) { " + "if (!(dr instanceof List)) { dr = [dr]; } " + "for (e in dr) { " + "if (e == null) { continue; } " + 'def sel = e.selector != null ? e.selector : "none"; ' + 'def dom = e.domain != null ? e.domain : "none"; ' + 'def res = e.result != null ? e.result : "none"; ' + 'dk.add(sel + " / " + dom + " / " + res); ' + "} } " + "ctx._source.dkim_results_combined = dk; " + "List sp = new ArrayList(); " + "def sr = ctx._source.spf_results; " + "if (sr != null) { " + "if (!(sr instanceof List)) { sr = [sr]; } " + "for (e in sr) { " + "if (e == null) { continue; } " + 'def sc = e.scope != null ? e.scope : "mfrom"; ' + 'def dom = e.domain != null ? e.domain : "none"; ' + 'def res = e.result != null ? e.result : (e.results != null ? e.results : "none"); ' + 'sp.add(sc + " / " + dom + " / " + res); ' + "} } " + "ctx._source.spf_results_combined = sp;" +) + +# Guard query for the policies_combined/failure_details_combined backfill +# (see ``migrate_indexes``). Matches only SMTP TLS documents that have at +# least one policy or failure detail and are missing the corresponding +# combined field. Empty arrays are invisible to ``exists``, so documents +# with zero policies/failure details are correctly skipped (this also +# makes the query idempotent — a backfilled document no longer matches). +# Each result is matched on an OR of its relevant subfields as defense in +# depth: the parsers we audited never store a policy/failure detail +# without these fields, but an empty string indexes no text tokens and is +# invisible to ``exists``, and the storage shape of every historical +# parsedmarc version can't be audited — matching either subfield costs +# nothing and cannot skip a document that has something to backfill. +_SMTP_TLS_COMBINED_BACKFILL_QUERY: dict[str, Any] = { + "bool": { + "minimum_should_match": 1, + "should": [ + { + "bool": { + "must": [ + { + "bool": { + "minimum_should_match": 1, + "should": [ + {"exists": {"field": "policies.policy_domain"}}, + {"exists": {"field": "policies.policy_type"}}, + ], + } + } + ], + "must_not": [{"exists": {"field": "policies_combined"}}], + } + }, + { + "bool": { + "must": [ + { + "bool": { + "minimum_should_match": 1, + "should": [ + { + "exists": { + "field": "policies.failure_details.result_type" + } + }, + { + "exists": { + "field": "policies.failure_details.sending_mta_ip" + } + }, + ], + } + } + ], + "must_not": [{"exists": {"field": "failure_details_combined"}}], + } + }, + ], + } +} + +# Painless script that (re)derives policies_combined/failure_details_combined +# from policies/policies.failure_details, matching the format written by +# save_smtp_tls_report_to_opensearch(): "{policy_domain} / {policy_type}" +# per policy and "{policy_domain} / {policy_type} / {result_type} / +# {sending_mta_ip} / {receiving_ip} / {receiving_mx_hostname}" per failure +# detail. +_SMTP_TLS_COMBINED_BACKFILL_SCRIPT = ( + "List pols = new ArrayList(); " + "List dets = new ArrayList(); " + "def ps = ctx._source.policies; " + "if (ps != null) { " + "if (!(ps instanceof List)) { ps = [ps]; } " + "for (p in ps) { " + "if (p == null) { continue; } " + 'def dom = p.policy_domain != null ? p.policy_domain : "none"; ' + 'def typ = p.policy_type != null ? p.policy_type : "none"; ' + 'pols.add(dom + " / " + typ); ' + "def fds = p.failure_details; " + "if (fds != null) { " + "if (!(fds instanceof List)) { fds = [fds]; } " + "for (f in fds) { " + "if (f == null) { continue; } " + 'def rt = f.result_type != null ? f.result_type : "none"; ' + 'def smi = f.sending_mta_ip != null ? f.sending_mta_ip : "none"; ' + 'def ri = f.receiving_ip != null ? f.receiving_ip : "none"; ' + 'def rmh = f.receiving_mx_hostname != null ? f.receiving_mx_hostname : "none"; ' + 'dets.add(dom + " / " + typ + " / " + rt + " / " + smi + " / " + ri + " / " + rmh); ' + "} } } } " + "ctx._source.policies_combined = pols; " + "ctx._source.failure_details_combined = dets;" +) + + class _PolicyOverride(InnerDoc): type = Text() comment = Text() @@ -186,6 +362,11 @@ header_from = Text() envelope_from = Text() envelope_to = Text() + # Nested(...) on the two auth-result fields below is only the DSL's + # in-memory document shape; it is never installed as a mapping. + # create_indexes() deliberately skips Index.document() registration so + # these fields stay dynamic-mapped as plain `object` in the cluster + # (see the comment there and issue #169). dkim_results = Nested(_DKIMResult) spf_results = Nested(_SPFResult) # One "{selector} / {domain} / {result}" (DKIM) or "{scope} / {domain} / @@ -223,9 +404,7 @@ human_result=human_result, ) ) - self.dkim_results_combined.append( - "{0} / {1} / {2}".format(selector, domain, result) - ) + self.dkim_results_combined.append(f"{selector} / {domain} / {result}") def add_spf_result( self, @@ -242,9 +421,7 @@ human_result=human_result, ) ) - self.spf_results_combined.append( - "{0} / {1} / {2}".format(scope, domain, result) - ) + self.spf_results_combined.append(f"{scope} / {domain} / {result}") def save(self, **kwargs): # pyright: ignore[reportIncompatibleMethodOverride] self.passed_dmarc = False @@ -331,6 +508,7 @@ result_type = Text() sending_mta_ip = Ip() receiving_mx_helo = Text() + receiving_mx_hostname = Text() receiving_ip = Ip() failed_session_count = Integer() additional_information_uri = Text() @@ -366,7 +544,7 @@ receiving_mx_helo=receiving_mx_helo, receiving_ip=receiving_ip, failed_session_count=failed_session_count, - additional_information=additional_information_uri, + additional_information_uri=additional_information_uri, failure_reason_code=failure_reason_code, ) self.failure_details.append(_details) @@ -383,6 +561,19 @@ contact_info = Text() report_id = Text() policies = Nested(_SMTPTLSPolicyDoc) + # One "{policy_domain} / {policy_type}" string per policy. Kibana/ + # Grafana tables cannot terms-aggregate the subfields of an object + # array without producing a cross-product of values (issue #169), so + # dashboards aggregate these composed keywords instead. Declared to + # match what dynamic mapping produces for a string array (text + + # .keyword). + policies_combined = Text(multi=True, fields={"keyword": Keyword(ignore_above=256)}) + # One "{policy_domain} / {policy_type} / {result_type} / + # {sending_mta_ip} / {receiving_ip} / {receiving_mx_hostname}" string + # per failure detail, across all policies. + failure_details_combined = Text( + multi=True, fields={"keyword": Keyword(ignore_above=256)} + )
@@ -478,20 +669,27 @@ for name in names: index = Index(name) try: - # Deliberately no Index.document() registration: Kibana/OpenSearch - # Dashboards/Grafana cannot terms-aggregate fields inside a - # `nested` mapping, so the dynamic `object` mapping produced by a - # bare create is load-bearing for the shipped dashboards (issue - # #169; see the *_combined fields on _AggregateReportDoc). + # Deliberately no Index.document() registration: the shipped + # dashboards cannot rebuild their detail tables on a `nested` + # mapping — Kibana/OSD visual editors do not support nested + # fields, Vega can run nested aggregations but does not render + # tables, and Grafana's nested bucket aggregation (9.4+) lacks + # reverse_nested for parent-level metrics like message_count — + # so the dynamic `object` mapping produced by a bare create is + # load-bearing for the shipped dashboards. _AggregateReportDoc + # still declares dkim_results/spf_results with Nested(...), but + # that is only the DSL's in-memory shape for building documents + # — it is never installed as a mapping. See issue #169 and the + # *_combined fields on _AggregateReportDoc. if not index.exists(): - logger.debug("Creating OpenSearch index: {0}".format(name)) + logger.debug(f"Creating OpenSearch index: {name}") if settings is None: index.settings(number_of_shards=1, number_of_replicas=0) else: index.settings(**settings) index.create() except Exception as e: - raise OpenSearchError("OpenSearch error: {0}".format(e.__str__()))
+ raise OpenSearchError(f"OpenSearch error: {e.__str__()}") @@ -500,47 +698,175 @@ def migrate_indexes( aggregate_indexes: list[str] | None = None, failure_indexes: list[str] | None = None, + smtp_tls_indexes: list[str] | None = None, ): """ - Updates index mappings + Runs index migrations and backfills. + + First, the legacy ``published_policy.fo`` migration: indexes where that + field was mapped as ``long`` (data indexed by very old parsedmarc + releases under the Elasticsearch 6-era ``doc`` mapping type) are rebuilt + as a ``-v2`` index with the text/keyword shape. + + Second, the ``dkim_results_combined``/``spf_results_combined`` backfill + (added for issue #169) for aggregate report documents that were saved + before those fields existed. For each name in ``aggregate_indexes``, + this submits an ``update_by_query`` against the ``f"{name}*"`` index + pattern (the real indexes are date-suffixed) as a non-blocking + background task (``wait_for_completion=False``), so it never delays + parsedmarc startup. Submission is guarded by a cheap ``count`` query + that only matches documents with DKIM/SPF results but no combined + field, so once an index is fully backfilled, later calls are a fast + no-op. Any error talking to the cluster (e.g. no indexes yet on a + fresh install, or a transient connection issue) is caught and logged + as a warning rather than raised; the backfill is simply retried on the + next startup, and the manual ``_update_by_query`` command documented + in ``docs/source/elasticsearch.md`` remains available in the meantime. + + Third, the same treatment for the ``policies_combined``/ + ``failure_details_combined`` fields (same issue #169) on SMTP TLS + report documents, for each name in ``smtp_tls_indexes``. Args: aggregate_indexes (list): A list of aggregate index names failure_indexes (list): A list of failure index names (accepted for API compatibility; no migrations are currently needed for failure indexes) + smtp_tls_indexes (list): A list of SMTP TLS index names """ - version = 2 - if aggregate_indexes is None: - aggregate_indexes = [] - for aggregate_index_name in aggregate_indexes: - if not Index(aggregate_index_name).exists(): - continue - aggregate_index = Index(aggregate_index_name) - doc = "doc" - fo_field = "published_policy.fo" - fo = "fo" - fo_mapping = aggregate_index.get_field_mapping(fields=[fo_field]) - fo_mapping = fo_mapping[list(fo_mapping.keys())[0]]["mappings"] - if doc not in fo_mapping: - continue + if not aggregate_indexes and not smtp_tls_indexes: + return - fo_mapping = fo_mapping[doc][fo_field]["mapping"][fo] - fo_type = fo_mapping["type"] - if fo_type == "long": - new_index_name = "{0}-v{1}".format(aggregate_index_name, version) - body = { - "properties": { - "published_policy.fo": { - "type": "text", - "fields": {"keyword": {"type": "keyword", "ignore_above": 256}}, + version = 2 + for aggregate_index_name in aggregate_indexes or []: + try: + if not Index(aggregate_index_name).exists(): + continue + aggregate_index = Index(aggregate_index_name) + doc = "doc" + fo_field = "published_policy.fo" + fo = "fo" + fo_mapping = aggregate_index.get_field_mapping(fields=[fo_field]) + fo_mapping = fo_mapping[list(fo_mapping.keys())[0]]["mappings"] + if doc not in fo_mapping: + continue + + fo_mapping = fo_mapping[doc][fo_field]["mapping"][fo] + fo_type = fo_mapping["type"] + if fo_type == "long": + new_index_name = f"{aggregate_index_name}-v{version}" + body = { + "properties": { + "published_policy.fo": { + "type": "text", + "fields": { + "keyword": {"type": "keyword", "ignore_above": 256} + }, + } } } - } - Index(new_index_name).create() - Index(new_index_name).put_mapping(doc_type=doc, body=body) - reindex(connections.get_connection(), aggregate_index_name, new_index_name) - Index(aggregate_index_name).delete() + Index(new_index_name).create() + Index(new_index_name).put_mapping(doc_type=doc, body=body) + reindex( + connections.get_connection(), aggregate_index_name, new_index_name + ) + Index(aggregate_index_name).delete() + except Exception as e: + logger.warning( + "Failed the legacy published_policy.fo migration for " + f"{aggregate_index_name}: {e}. This will be retried at the " + "next startup." + ) + + try: + client = connections.get_connection() + except Exception as e: + logger.warning( + "Skipping the dkim_results_combined/spf_results_combined/" + "policies_combined/failure_details_combined backfill: could " + f"not get an OpenSearch connection: {e}. This will be retried " + "at the next startup." + ) + return + for name in aggregate_indexes or []: + pattern = f"{name}*" + try: + count_response = client.count( + index=pattern, + body={"query": _COMBINED_BACKFILL_QUERY}, + ignore_unavailable=True, + allow_no_indices=True, + ) + count = count_response["count"] + if not count: + continue + update_response = client.update_by_query( + index=pattern, + body={ + "query": _COMBINED_BACKFILL_QUERY, + "script": { + "source": _COMBINED_BACKFILL_SCRIPT, + "lang": "painless", + }, + }, + conflicts="proceed", + wait_for_completion=False, + ignore_unavailable=True, + allow_no_indices=True, + ) + task_id = update_response.get("task") + logger.info( + "Backfilling dkim_results_combined/spf_results_combined on " + f"{count} existing documents in {pattern} (task {task_id})" + ) + except Exception as e: + logger.warning( + "Failed to check/submit the dkim_results_combined/" + f"spf_results_combined backfill for {pattern}: {e}. This " + "will be retried at the next startup; the manual " + "_update_by_query command in the documentation remains " + "available in the meantime." + ) + + for name in smtp_tls_indexes or []: + pattern = f"{name}*" + try: + count_response = client.count( + index=pattern, + body={"query": _SMTP_TLS_COMBINED_BACKFILL_QUERY}, + ignore_unavailable=True, + allow_no_indices=True, + ) + count = count_response["count"] + if not count: + continue + update_response = client.update_by_query( + index=pattern, + body={ + "query": _SMTP_TLS_COMBINED_BACKFILL_QUERY, + "script": { + "source": _SMTP_TLS_COMBINED_BACKFILL_SCRIPT, + "lang": "painless", + }, + }, + conflicts="proceed", + wait_for_completion=False, + ignore_unavailable=True, + allow_no_indices=True, + ) + task_id = update_response.get("task") + logger.info( + "Backfilling policies_combined/failure_details_combined on " + f"{count} existing documents in {pattern} (task {task_id})" + ) + except Exception as e: + logger.warning( + "Failed to check/submit the policies_combined/" + f"failure_details_combined backfill for {pattern}: {e}. " + "This will be retried at the next startup; the manual " + "_update_by_query command in the documentation remains " + "available in the meantime." + ) @@ -589,11 +915,11 @@ end_date_query = Q(dict(range=dict(date_end=dict(lte=end_date)))) if index_suffix is not None: - search_index = "dmarc_aggregate_{0}*".format(index_suffix) + search_index = f"dmarc_aggregate_{index_suffix}*" else: search_index = "dmarc_aggregate*" if index_prefix is not None: - search_index = "{0}{1}".format(index_prefix, search_index) + search_index = f"{index_prefix}{search_index}" search = Search(index=search_index) query = org_name_query & report_id_query & domain_query query = query & begin_date_query & end_date_query @@ -605,18 +931,15 @@ existing = search.execute() except Exception as error_: raise OpenSearchError( - "OpenSearch's search for existing report \ - error: {}".format(error_.__str__()) + f"OpenSearch's search for existing report error: {error_.__str__()}" ) if len(existing) > 0: raise AlreadySaved( - "An aggregate report ID {0} from {1} about {2} " - "with a date range of {3} UTC to {4} UTC already " + f"An aggregate report ID {report_id} from {org_name} about {domain} " + f"with a date range of {begin_date_human} UTC to {end_date_human} UTC already " "exists in " - "OpenSearch".format( - report_id, org_name, domain, begin_date_human, end_date_human - ) + "OpenSearch" ) published_policy = _PublishedPolicy( domain=aggregate_report["policy_published"]["domain"], @@ -709,11 +1032,11 @@ index = "dmarc_aggregate" if index_suffix: - index = "{0}_{1}".format(index, index_suffix) + index = f"{index}_{index_suffix}" if index_prefix: - index = "{0}{1}".format(index_prefix, index) + index = f"{index_prefix}{index}" - index = "{0}-{1}".format(index, index_date) + index = f"{index}-{index_date}" index_settings = dict( number_of_shards=number_of_shards, number_of_replicas=number_of_replicas ) @@ -723,7 +1046,7 @@ try: agg_doc.save() except Exception as e: - raise OpenSearchError("OpenSearch error: {0}".format(e.__str__())) + raise OpenSearchError(f"OpenSearch error: {e.__str__()}") @@ -774,12 +1097,12 @@ arrival_date_epoch_milliseconds = int(arrival_date.timestamp() * 1000) if index_suffix is not None: - search_index = "dmarc_failure_{0}*,dmarc_forensic_{0}*".format(index_suffix) + search_index = f"dmarc_failure_{index_suffix}*,dmarc_forensic_{index_suffix}*" else: search_index = "dmarc_failure*,dmarc_forensic*" if index_prefix is not None: search_index = ",".join( - "{0}{1}".format(index_prefix, part) for part in search_index.split(",") + f"{index_prefix}{part}" for part in search_index.split(",") ) search = Search(index=search_index) q = Q(dict(match=dict(arrival_date=arrival_date_epoch_milliseconds))) @@ -831,8 +1154,8 @@ if len(existing) > 0: raise AlreadySaved( - "A failure sample to {0} from {1} " - "with a subject of {2} and arrival date of {3} " + "A failure sample to {} from {} " + "with a subject of {} and arrival date of {} " "already exists in " "OpenSearch".format(to_, from_, subject, failure_report["arrival_date_utc"]) ) @@ -891,14 +1214,14 @@ index = "dmarc_failure" if index_suffix: - index = "{0}_{1}".format(index, index_suffix) + index = f"{index}_{index_suffix}" if index_prefix: - index = "{0}{1}".format(index_prefix, index) + index = f"{index_prefix}{index}" if monthly_indexes: index_date = arrival_date.strftime("%Y-%m") else: index_date = arrival_date.strftime("%Y-%m-%d") - index = "{0}-{1}".format(index, index_date) + index = f"{index}-{index_date}" index_settings = dict( number_of_shards=number_of_shards, number_of_replicas=number_of_replicas ) @@ -907,10 +1230,10 @@ try: failure_doc.save() except Exception as e: - raise OpenSearchError("OpenSearch error: {0}".format(e.__str__())) + raise OpenSearchError(f"OpenSearch error: {e.__str__()}") except KeyError as e: raise InvalidFailureReport( - "Failure report missing required field: {0}".format(e.__str__()) + f"Failure report missing required field: {e.__str__()}" ) @@ -960,11 +1283,11 @@ end_date_query = Q(dict(match=dict(date_end=end_date))) if index_suffix is not None: - search_index = "smtp_tls_{0}*".format(index_suffix) + search_index = f"smtp_tls_{index_suffix}*" else: search_index = "smtp_tls*" if index_prefix is not None: - search_index = "{0}{1}".format(index_prefix, search_index) + search_index = f"{index_prefix}{search_index}" search = Search(index=search_index) query = org_name_query & report_id_query query = query & begin_date_query & end_date_query @@ -974,8 +1297,7 @@ existing = search.execute() except Exception as error_: raise OpenSearchError( - "OpenSearch's search for existing report \ - error: {}".format(error_.__str__()) + f"OpenSearch's search for existing report error: {error_.__str__()}" ) if len(existing) > 0: @@ -989,10 +1311,10 @@ index = "smtp_tls" if index_suffix: - index = "{0}_{1}".format(index, index_suffix) + index = f"{index}_{index_suffix}" if index_prefix: - index = "{0}{1}".format(index_prefix, index) - index = "{0}-{1}".format(index, index_date) + index = f"{index_prefix}{index}" + index = f"{index}-{index_date}" index_settings = dict( number_of_shards=number_of_shards, number_of_replicas=number_of_replicas ) @@ -1013,6 +1335,16 @@ policy_strings = policy["policy_strings"] if "mx_host_patterns" in policy: mx_host_patterns = policy["mx_host_patterns"] + # policies_combined/failure_details_combined: see the field + # declarations on _SMTPTLSReportDoc and issue #169. policies and + # their failure_details are object arrays with the same + # cross-product problem as dkim_results/spf_results, so dashboards + # aggregate these composed strings instead of the raw subfields. + policy_domain_combined = policy.get("policy_domain") or "none" + policy_type_combined = policy.get("policy_type") or "none" + smtp_tls_doc.policies_combined.append( + f"{policy_domain_combined} / {policy_type_combined}" + ) policy_doc = _SMTPTLSPolicyDoc( policy_domain=policy["policy_domain"], policy_type=policy["policy_type"], @@ -1033,7 +1365,12 @@ if "receiving_mx_hostname" in failure_detail: receiving_mx_hostname = failure_detail["receiving_mx_hostname"] - if "additional_information_uri" in failure_detail: + # The parser's key is additional_info_uri (see + # SMTPTLSFailureDetailsOptional in types.py); accept the + # long-form key too for dicts built by other callers. + if "additional_info_uri" in failure_detail: + additional_information_uri = failure_detail["additional_info_uri"] + elif "additional_information_uri" in failure_detail: additional_information_uri = failure_detail[ "additional_information_uri" ] @@ -1058,6 +1395,16 @@ additional_information_uri=additional_information_uri, failure_reason_code=failure_reason_code, ) + smtp_tls_doc.failure_details_combined.append( + "{} / {} / {} / {} / {} / {}".format( + policy_domain_combined, + policy_type_combined, + failure_detail.get("result_type") or "none", + sending_mta_ip or "none", + receiving_ip or "none", + receiving_mx_hostname or "none", + ) + ) smtp_tls_doc.policies.append(policy_doc) create_indexes([index], index_settings) @@ -1066,7 +1413,7 @@ try: smtp_tls_doc.save() except Exception as e: - raise OpenSearchError("OpenSearch error: {0}".format(e.__str__())) + raise OpenSearchError(f"OpenSearch error: {e.__str__()}") diff --git a/_modules/parsedmarc/splunk.html b/_modules/parsedmarc/splunk.html index 52bd3c40..877f5cef 100644 --- a/_modules/parsedmarc/splunk.html +++ b/_modules/parsedmarc/splunk.html @@ -5,14 +5,14 @@ - parsedmarc.splunk — parsedmarc 10.2.4 documentation + parsedmarc.splunk — parsedmarc 10.3.0 documentation - + @@ -133,8 +133,8 @@ data before giving up """ parsed_url = urlparse(url) - self.url = "{0}://{1}/services/collector/event/1.0".format( - parsed_url.scheme, parsed_url.netloc + self.url = ( + f"{parsed_url.scheme}://{parsed_url.netloc}/services/collector/event/1.0" ) self.access_token = access_token.lstrip("Splunk ") self.index = index @@ -149,7 +149,7 @@ self.session = httpx.Client( headers={ "User-Agent": USER_AGENT, - "Authorization": "Splunk {0}".format(self.access_token), + "Authorization": f"Splunk {self.access_token}", }, verify=self.verify, follow_redirects=True, @@ -177,7 +177,7 @@ return data = self._common_data.copy() - json_str = "" + json_lines: list[str] = [] for report in aggregate_reports: for record in report["records"]: new_report: dict[str, str | int | float | dict] = dict() @@ -216,13 +216,13 @@ ) data["time"] = timestamp data["event"] = new_report.copy() - json_str += "{0}\n".format(json.dumps(data)) + json_lines.append(f"{json.dumps(data)}\n") if not self.verify: logger.debug("Skipping certificate verification for Splunk HEC") try: response = self.session.post( - self.url, content=json_str, timeout=self.timeout + self.url, content="".join(json_lines), timeout=self.timeout ) response = response.json() except Exception as e: @@ -251,7 +251,7 @@ if len(failure_reports) < 1: return - json_str = "" + json_lines: list[str] = [] for report in failure_reports: data = self._common_data.copy() data["sourcetype"] = "dmarc:failure" @@ -262,13 +262,13 @@ ) data["time"] = timestamp data["event"] = report.copy() - json_str += "{0}\n".format(json.dumps(data)) + json_lines.append(f"{json.dumps(data)}\n") if not self.verify: logger.debug("Skipping certificate verification for Splunk HEC") try: response = self.session.post( - self.url, content=json_str, timeout=self.timeout + self.url, content="".join(json_lines), timeout=self.timeout ) response = response.json() except Exception as e: @@ -298,19 +298,19 @@ return data = self._common_data.copy() - json_str = "" + json_lines: list[str] = [] for report in reports: data["sourcetype"] = "smtp:tls" timestamp = human_timestamp_to_unix_timestamp(report["begin_date"]) data["time"] = timestamp data["event"] = report.copy() - json_str += "{0}\n".format(json.dumps(data)) + json_lines.append(f"{json.dumps(data)}\n") if not self.verify: logger.debug("Skipping certificate verification for Splunk HEC") try: response = self.session.post( - self.url, content=json_str, timeout=self.timeout + self.url, content="".join(json_lines), timeout=self.timeout ) response = response.json() except Exception as e: diff --git a/_modules/parsedmarc/types.html b/_modules/parsedmarc/types.html index faaf2094..3279dc35 100644 --- a/_modules/parsedmarc/types.html +++ b/_modules/parsedmarc/types.html @@ -5,14 +5,14 @@ - parsedmarc.types — parsedmarc 10.2.4 documentation + parsedmarc.types — parsedmarc 10.3.0 documentation - + diff --git a/_modules/parsedmarc/utils.html b/_modules/parsedmarc/utils.html index cd897c2e..b215ff60 100644 --- a/_modules/parsedmarc/utils.html +++ b/_modules/parsedmarc/utils.html @@ -5,14 +5,14 @@ - parsedmarc.utils — parsedmarc 10.2.4 documentation + parsedmarc.utils — parsedmarc 10.3.0 documentation - + @@ -340,7 +340,7 @@ """ domain = str(domain).lower() record_type = record_type.upper() - cache_key = "{0}_{1}".format(domain, record_type) + cache_key = f"{domain}_{record_type}" if cache: cached_records = cache.get(cache_key, None) if isinstance(cached_records, list): @@ -1227,7 +1227,7 @@ if len(mbox.keys()) > 0: _is_mbox = True except Exception as e: - logger.debug("Error checking for MBOX file: {0}".format(e.__str__())) + logger.debug(f"Error checking for MBOX file: {e.__str__()}") return _is_mbox @@ -1388,7 +1388,7 @@ payload = str.encode(payload) attachment["sha256"] = hashlib.sha256(payload).hexdigest() except Exception as e: - logger.debug("Unable to decode attachment: {0}".format(e.__str__())) + logger.debug(f"Unable to decode attachment: {e.__str__()}") if strip_attachment_payloads: for attachment in parsed_email["attachments"]: if "payload" in attachment: diff --git a/_sources/elasticsearch.md.txt b/_sources/elasticsearch.md.txt index 5e08b983..2d0a1a43 100644 --- a/_sources/elasticsearch.md.txt +++ b/_sources/elasticsearch.md.txt @@ -235,14 +235,32 @@ result paired, which the dashboards' alignment-detail tables aggregate on. Reports saved by older versions lack these fields and will not appear in those tables. -Running the following once per cluster backfills the fields on existing -documents. It is idempotent (documents that already have the fields are -skipped), so it is safe to re-run. It works identically on OpenSearch; -just adjust the URL and credentials. The query matches only documents -that have at least one DKIM or SPF auth result and lack the corresponding -combined field; documents with no auth results are skipped, because an -`exists` query cannot see an empty array, and for search purposes an -empty `dkim_results_combined` is identical to an absent one. +parsedmarc now backfills this automatically. On startup, it runs a cheap +count query against each configured aggregate index pattern to check for +documents that have DKIM or SPF results but are missing the corresponding +combined field. If any are found, it submits the backfill as a background +`_update_by_query` task (`wait_for_completion=false`), so startup is never +blocked on it; progress is logged, including the task ID. The check itself +is idempotent — once an index is fully backfilled, later startups see a +count of 0 and log nothing further — and it works the same way on +OpenSearch. Any error talking to the cluster (for example, no indexes yet +on a fresh install) is logged as a warning and retried on the next startup, +rather than aborting parsedmarc. + +If you upgrade the dashboards without pointing the new parsedmarc version +at the cluster, or you'd rather control when the write load happens, you +can still run the backfill manually. It is idempotent (documents that +already have the fields are skipped), so it is safe to re-run. It works +identically on OpenSearch; just adjust the URL and credentials. The query +matches only documents that have at least one DKIM or SPF auth result and +lack the corresponding combined field; documents with no auth results are +skipped, because an `exists` query cannot see an empty array, and for +search purposes an empty `dkim_results_combined` is identical to an +absent one. Each result is matched on either its `domain` or its `result` +subfield as defense in depth: an empty string indexes no text tokens and +is invisible to `exists`, and the storage shape of every historical +parsedmarc version can't be audited, so matching either subfield ensures +no backfillable document is skipped. ```bash curl -X POST "http://localhost:9200/dmarc_aggregate*/_update_by_query?conflicts=proceed&wait_for_completion=false" \ @@ -254,13 +272,33 @@ curl -X POST "http://localhost:9200/dmarc_aggregate*/_update_by_query?conflicts= "should": [ { "bool": { - "must": [{"exists": {"field": "dkim_results.domain"}}], + "must": [ + { + "bool": { + "minimum_should_match": 1, + "should": [ + {"exists": {"field": "dkim_results.domain"}}, + {"exists": {"field": "dkim_results.result"}} + ] + } + } + ], "must_not": [{"exists": {"field": "dkim_results_combined"}}] } }, { "bool": { - "must": [{"exists": {"field": "spf_results.domain"}}], + "must": [ + { + "bool": { + "minimum_should_match": 1, + "should": [ + {"exists": {"field": "spf_results.domain"}}, + {"exists": {"field": "spf_results.result"}} + ] + } + } + ], "must_not": [{"exists": {"field": "spf_results_combined"}}] } } @@ -280,6 +318,66 @@ curl -X POST "http://localhost:9200/dmarc_aggregate*/_update_by_query?conflicts= dashboards ndjson (the index pattern saved object changed too) per the import instructions above. +SMTP TLS documents have the same class of defect one level deeper: +`policies` is an object array, and each policy's `failure_details` is an +object array inside it. SMTP TLS documents now also carry +`policies_combined` and `failure_details_combined`, backfilled +automatically at startup the same way, and the equivalent manual command +is: + +```bash +curl -X POST "http://localhost:9200/smtp_tls*/_update_by_query?conflicts=proceed&wait_for_completion=false" \ + -H "Content-Type: application/json" -d ' +{ + "query": { + "bool": { + "minimum_should_match": 1, + "should": [ + { + "bool": { + "must": [ + { + "bool": { + "minimum_should_match": 1, + "should": [ + {"exists": {"field": "policies.policy_domain"}}, + {"exists": {"field": "policies.policy_type"}} + ] + } + } + ], + "must_not": [{"exists": {"field": "policies_combined"}}] + } + }, + { + "bool": { + "must": [ + { + "bool": { + "minimum_should_match": 1, + "should": [ + {"exists": {"field": "policies.failure_details.result_type"}}, + {"exists": {"field": "policies.failure_details.sending_mta_ip"}} + ] + } + } + ], + "must_not": [{"exists": {"field": "failure_details_combined"}}] + } + } + ] + } + }, + "script": { + "lang": "painless", + "source": "List pols = new ArrayList(); List dets = new ArrayList(); def ps = ctx._source.policies; if (ps != null) { if (!(ps instanceof List)) { ps = [ps]; } for (p in ps) { if (p == null) { continue; } def dom = p.policy_domain != null ? p.policy_domain : \"none\"; def typ = p.policy_type != null ? p.policy_type : \"none\"; pols.add(dom + \" / \" + typ); def fds = p.failure_details; if (fds != null) { if (!(fds instanceof List)) { fds = [fds]; } for (f in fds) { if (f == null) { continue; } def rt = f.result_type != null ? f.result_type : \"none\"; def smi = f.sending_mta_ip != null ? f.sending_mta_ip : \"none\"; def ri = f.receiving_ip != null ? f.receiving_ip : \"none\"; def rmh = f.receiving_mx_hostname != null ? f.receiving_mx_hostname : \"none\"; dets.add(dom + \" / \" + typ + \" / \" + rt + \" / \" + smi + \" / \" + ri + \" / \" + rmh); } } } } ctx._source.policies_combined = pols; ctx._source.failure_details_combined = dets;" + } +}' +``` + +It works identically on OpenSearch; just adjust the URL and credentials, same +as the aggregate command above. + ## Records retention Starting in version 5.0.0, `parsedmarc` stores data in a separate diff --git a/_sources/kibana.md.txt b/_sources/kibana.md.txt index 16b27b83..8a2be4b7 100644 --- a/_sources/kibana.md.txt +++ b/_sources/kibana.md.txt @@ -86,7 +86,19 @@ table. Each row of the DKIM details table is one real DKIM signature, shown as a combined `selector / domain / result` value; the SPF details table shows `scope / domain / result` the same way. Combining the values into one column keeps each signature's selector, domain, and result paired together, -rather than aggregating them as separate columns. +rather than aggregating them as separate columns. Because a message that +carries multiple DKIM signatures appears once per signature, summing the +messages column across rows can exceed the total number of messages. + +The "Auth result filters" panel above the details tables +provides dropdowns for the individual auth-result components — DKIM +selector, DKIM domain, DKIM result, SPF scope, SPF domain, and SPF +result — and filters the whole dashboard by them. Because components from +different signatures of the same message are indexed together, combining +two of these component filters matches documents where any signature +satisfies each condition individually, not necessarily the same signature; +the combined `selector / domain / result` (`scope / domain / result`) +column remains the per-signature source of truth. :::{note} The alignment tables (SPF details, DKIM details) and the per-IP source @@ -116,3 +128,15 @@ reporting organizations, the policy domains they report on, and the specific failure types — certificate expiry, STARTTLS not supported, STS policy fetch errors, validation failures, and similar — together with the sending and receiving MTA addresses involved. + +Like the DKIM and SPF details tables above, the "SMTP TLS domains" and +"SMTP TLS failure details" tables show one row per policy and one row per +failure detail, respectively, using combined `policy (domain / type)` and +`failure detail (domain / type / result / sending mta / receiving ip / mx)` +columns so that each policy's or failure detail's fields stay paired +together, rather than aggregating them as separate columns. The +`successful_sessions` and `failed_sessions` columns are summed per report +document, though, not per policy: when a single report carries multiple +policies, a row's session sums include the sibling policies from that +report as well as its own. Fully attributing session counts to a single +policy would require restructuring the stored documents. diff --git a/_sources/usage.md.txt b/_sources/usage.md.txt index aafc48fd..c6a93ea2 100644 --- a/_sources/usage.md.txt +++ b/_sources/usage.md.txt @@ -3,23 +3,25 @@ ## CLI help ```text -usage: parsedmarc [-h] [-c CONFIG_FILE] [--strip-attachment-payloads] [-o OUTPUT] +usage: parsedmarc [-h] [-c CONFIG_FILE] [-r] [--strip-attachment-payloads] [-o OUTPUT] [--aggregate-json-filename AGGREGATE_JSON_FILENAME] [--failure-json-filename FAILURE_JSON_FILENAME] [--smtp-tls-json-filename SMTP_TLS_JSON_FILENAME] [--aggregate-csv-filename AGGREGATE_CSV_FILENAME] [--failure-csv-filename FAILURE_CSV_FILENAME] [--smtp-tls-csv-filename SMTP_TLS_CSV_FILENAME] - [-n NAMESERVERS [NAMESERVERS ...]] [-t DNS_TIMEOUT] [--offline] [-s] [-w] [--verbose] [--debug] - [--log-file LOG_FILE] [--no-prettify-json] [-v] + [-n NAMESERVERS [NAMESERVERS ...]] [-t DNS_TIMEOUT] [--dns-retries DNS_RETRIES] [--offline] [-s] + [-w] [--verbose] [--debug] [--log-file LOG_FILE] [--no-prettify-json] [-v] [file_path ...] Parses DMARC reports positional arguments: - file_path one or more paths to aggregate or failure report files, emails, or mbox files' + file_path one or more paths to aggregate or failure report files, emails, mbox files, or directories + containing them options: -h, --help show this help message and exit -c CONFIG_FILE, --config-file CONFIG_FILE a path to a configuration file (--silent implied) + -r, --recursive search directories given as file_path recursively, and enable '**' recursion in glob patterns --strip-attachment-payloads remove attachment payloads from failure report output -o OUTPUT, --output OUTPUT @@ -40,6 +42,8 @@ options: nameservers to query -t DNS_TIMEOUT, --dns_timeout DNS_TIMEOUT number of seconds to wait for an answer from DNS (default: 2.0) + --dns-retries DNS_RETRIES + number of times to retry DNS queries on timeout or other transient errors (default: 0) --offline do not make online queries for geolocation or DNS -s, --silent only print errors -w, --warnings print warnings in addition to errors @@ -165,14 +169,22 @@ The full set of configuration options are: any configured output destination fails while saving/publishing reports (Default: `False`) - `log_file` - str: Write log messages to a file at this path - - `n_procs` - int: Number of process to run in parallel when - parsing in CLI mode (Default: `1`) + - `n_procs` - int: Number of processes to run in parallel when + parsing report files passed directly as CLI arguments + (Default: `1`) :::{note} Setting this to a number larger than one can improve performance when processing thousands of files ::: + :::{note} + `n_procs` only applies to report files passed directly on the + command line. Messages from mbox files and from mailbox + connections (IMAP, Microsoft Graph, Gmail API, Maildir) are + always processed sequentially. + ::: + - `mailbox` - `reports_folder` - str: The mailbox folder (or label for Gmail) where the incoming reports can be found diff --git a/_static/documentation_options.js b/_static/documentation_options.js index eb3f216d..d5820723 100644 --- a/_static/documentation_options.js +++ b/_static/documentation_options.js @@ -1,5 +1,5 @@ const DOCUMENTATION_OPTIONS = { - VERSION: '10.2.4', + VERSION: '10.3.0', LANGUAGE: 'en', COLLAPSE_INDEX: false, BUILDER: 'html', diff --git a/api.html b/api.html index 895b47a9..f523849a 100644 --- a/api.html +++ b/api.html @@ -6,14 +6,14 @@ - API reference — parsedmarc 10.2.4 documentation + API reference — parsedmarc 10.3.0 documentation - + @@ -872,23 +872,30 @@ remaining keys are passed through; defaults are skipped entirely.

-parsedmarc.elastic.migrate_indexes(aggregate_indexes: list[str] | None = None, failure_indexes: list[str] | None = None)[source]
-

Updates index mappings

-

This is a no-op kept for API compatibility (cli.py calls it on -startup). The only migration this function ever performed was -re-typing published_policy.fo from long to text, which -applied exclusively to indices still carrying the legacy -Elasticsearch 6-era "doc" mapping type. The 8.x client can only -reach servers (Elasticsearch 8.x/9.x) whose indices were created on -Elasticsearch 7.x or later and are therefore typeless, so that -migration path is unreachable and has been removed.

+parsedmarc.elastic.migrate_indexes(aggregate_indexes: list[str] | None = None, failure_indexes: list[str] | None = None, smtp_tls_indexes: list[str] | None = None)[source] +

Backfills the dkim_results_combined/spf_results_combined fields +(added for issue #169) on aggregate report documents, and the +policies_combined/failure_details_combined fields on SMTP TLS +report documents, that were saved before those fields existed.

+

For each name in aggregate_indexes/smtp_tls_indexes, this +submits an update_by_query against the f"{name}*" index pattern +(the real indexes are date-suffixed) as a non-blocking background task +(wait_for_completion=False), so it never delays parsedmarc startup. +Submission is guarded by a cheap count query that only matches +documents with DKIM/SPF results (or policies/failure details) but no +combined field, so once an index is fully backfilled, later calls are a +fast no-op. Any error talking to the cluster (e.g. no indexes yet on a +fresh install, or a transient connection issue) is caught and logged as +a warning rather than raised; the backfill is simply retried on the +next startup, and the manual _update_by_query commands documented in +docs/source/elasticsearch.md remain available in the meantime.

Parameters:
    -

  • aggregate_indexes (list) – A list of aggregate index names -(accepted for API compatibility; unused)

    • +

  • aggregate_indexes (list) – A list of aggregate index names

  • failure_indexes (list) – A list of failure index names (accepted for API compatibility; unused)

    • +

  • smtp_tls_indexes (list) – A list of SMTP TLS index names

@@ -1039,8 +1046,29 @@ any other settings through unchanged.

-parsedmarc.opensearch.migrate_indexes(aggregate_indexes: list[str] | None = None, failure_indexes: list[str] | None = None)[source]
-

Updates index mappings

+parsedmarc.opensearch.migrate_indexes(aggregate_indexes: list[str] | None = None, failure_indexes: list[str] | None = None, smtp_tls_indexes: list[str] | None = None)[source] +

Runs index migrations and backfills.

+

First, the legacy published_policy.fo migration: indexes where that +field was mapped as long (data indexed by very old parsedmarc +releases under the Elasticsearch 6-era doc mapping type) are rebuilt +as a -v2 index with the text/keyword shape.

+

Second, the dkim_results_combined/spf_results_combined backfill +(added for issue #169) for aggregate report documents that were saved +before those fields existed. For each name in aggregate_indexes, +this submits an update_by_query against the f"{name}*" index +pattern (the real indexes are date-suffixed) as a non-blocking +background task (wait_for_completion=False), so it never delays +parsedmarc startup. Submission is guarded by a cheap count query +that only matches documents with DKIM/SPF results but no combined +field, so once an index is fully backfilled, later calls are a fast +no-op. Any error talking to the cluster (e.g. no indexes yet on a +fresh install, or a transient connection issue) is caught and logged +as a warning rather than raised; the backfill is simply retried on the +next startup, and the manual _update_by_query command documented +in docs/source/elasticsearch.md remains available in the meantime.

+

Third, the same treatment for the policies_combined/ +failure_details_combined fields (same issue #169) on SMTP TLS +report documents, for each name in smtp_tls_indexes.

Parameters:
    @@ -1048,6 +1076,7 @@ any other settings through unchanged.

  • failure_indexes (list) – A list of failure index names (accepted for API compatibility; no migrations are currently needed for failure indexes)

    • +

  • smtp_tls_indexes (list) – A list of SMTP TLS index names

diff --git a/contributing.html b/contributing.html index d4416a33..44c9348b 100644 --- a/contributing.html +++ b/contributing.html @@ -6,14 +6,14 @@ - Contributing to parsedmarc — parsedmarc 10.2.4 documentation + Contributing to parsedmarc — parsedmarc 10.3.0 documentation - + diff --git a/davmail.html b/davmail.html index 2167436f..84fb2810 100644 --- a/davmail.html +++ b/davmail.html @@ -6,14 +6,14 @@ - Accessing an inbox using OWA/EWS — parsedmarc 10.2.4 documentation + Accessing an inbox using OWA/EWS — parsedmarc 10.3.0 documentation - + diff --git a/dmarc.html b/dmarc.html index 1d5964b1..2839521d 100644 --- a/dmarc.html +++ b/dmarc.html @@ -6,14 +6,14 @@ - Understanding DMARC — parsedmarc 10.2.4 documentation + Understanding DMARC — parsedmarc 10.3.0 documentation - + diff --git a/elasticsearch.html b/elasticsearch.html index 39b44513..3900ed3a 100644 --- a/elasticsearch.html +++ b/elasticsearch.html @@ -6,14 +6,14 @@ - Elasticsearch and Kibana — parsedmarc 10.2.4 documentation + Elasticsearch and Kibana — parsedmarc 10.3.0 documentation - + @@ -273,14 +273,31 @@ scalar string arrays that keep each auth result’s selector/scope, domain, and result paired, which the dashboards’ alignment-detail tables aggregate on. Reports saved by older versions lack these fields and will not appear in those tables.

-

Running the following once per cluster backfills the fields on existing -documents. It is idempotent (documents that already have the fields are -skipped), so it is safe to re-run. It works identically on OpenSearch; -just adjust the URL and credentials. The query matches only documents -that have at least one DKIM or SPF auth result and lack the corresponding -combined field; documents with no auth results are skipped, because an -exists query cannot see an empty array, and for search purposes an -empty dkim_results_combined is identical to an absent one.

+

parsedmarc now backfills this automatically. On startup, it runs a cheap +count query against each configured aggregate index pattern to check for +documents that have DKIM or SPF results but are missing the corresponding +combined field. If any are found, it submits the backfill as a background +_update_by_query task (wait_for_completion=false), so startup is never +blocked on it; progress is logged, including the task ID. The check itself +is idempotent — once an index is fully backfilled, later startups see a +count of 0 and log nothing further — and it works the same way on +OpenSearch. Any error talking to the cluster (for example, no indexes yet +on a fresh install) is logged as a warning and retried on the next startup, +rather than aborting parsedmarc.

+

If you upgrade the dashboards without pointing the new parsedmarc version +at the cluster, or you’d rather control when the write load happens, you +can still run the backfill manually. It is idempotent (documents that +already have the fields are skipped), so it is safe to re-run. It works +identically on OpenSearch; just adjust the URL and credentials. The query +matches only documents that have at least one DKIM or SPF auth result and +lack the corresponding combined field; documents with no auth results are +skipped, because an exists query cannot see an empty array, and for +search purposes an empty dkim_results_combined is identical to an +absent one. Each result is matched on either its domain or its result +subfield as defense in depth: an empty string indexes no text tokens and +is invisible to exists, and the storage shape of every historical +parsedmarc version can’t be audited, so matching either subfield ensures +no backfillable document is skipped.

curl -X POST "http://localhost:9200/dmarc_aggregate*/_update_by_query?conflicts=proceed&wait_for_completion=false" \
   -H "Content-Type: application/json" -d '
 {
@@ -290,13 +307,33 @@ empty dkim_results_
       "should": [
         {
           "bool": {
-            "must": [{"exists": {"field": "dkim_results.domain"}}],
+            "must": [
+              {
+                "bool": {
+                  "minimum_should_match": 1,
+                  "should": [
+                    {"exists": {"field": "dkim_results.domain"}},
+                    {"exists": {"field": "dkim_results.result"}}
+                  ]
+                }
+              }
+            ],
             "must_not": [{"exists": {"field": "dkim_results_combined"}}]
           }
         },
         {
           "bool": {
-            "must": [{"exists": {"field": "spf_results.domain"}}],
+            "must": [
+              {
+                "bool": {
+                  "minimum_should_match": 1,
+                  "should": [
+                    {"exists": {"field": "spf_results.domain"}},
+                    {"exists": {"field": "spf_results.result"}}
+                  ]
+                }
+              }
+            ],
             "must_not": [{"exists": {"field": "spf_results_combined"}}]
           }
         }
@@ -315,6 +352,63 @@ empty dkim_results_
 index_prefix/index_suffix. After backfilling, re-import the updated
 dashboards ndjson (the index pattern saved object changed too) per the
 import instructions above.

+
SMTP TLS documents have the same class of defect one level deeper:
+policies is an object array, and each policy’s failure_details is an
+object array inside it. SMTP TLS documents now also carry
+policies_combined and failure_details_combined, backfilled
+automatically at startup the same way, and the equivalent manual command
+is:

+
curl -X POST "http://localhost:9200/smtp_tls*/_update_by_query?conflicts=proceed&wait_for_completion=false" \
+  -H "Content-Type: application/json" -d '
+{
+  "query": {
+    "bool": {
+      "minimum_should_match": 1,
+      "should": [
+        {
+          "bool": {
+            "must": [
+              {
+                "bool": {
+                  "minimum_should_match": 1,
+                  "should": [
+                    {"exists": {"field": "policies.policy_domain"}},
+                    {"exists": {"field": "policies.policy_type"}}
+                  ]
+                }
+              }
+            ],
+            "must_not": [{"exists": {"field": "policies_combined"}}]
+          }
+        },
+        {
+          "bool": {
+            "must": [
+              {
+                "bool": {
+                  "minimum_should_match": 1,
+                  "should": [
+                    {"exists": {"field": "policies.failure_details.result_type"}},
+                    {"exists": {"field": "policies.failure_details.sending_mta_ip"}}
+                  ]
+                }
+              }
+            ],
+            "must_not": [{"exists": {"field": "failure_details_combined"}}]
+          }
+        }
+      ]
+    }
+  },
+  "script": {
+    "lang": "painless",
+    "source": "List pols = new ArrayList(); List dets = new ArrayList(); def ps = ctx._source.policies; if (ps != null) { if (!(ps instanceof List)) { ps = [ps]; } for (p in ps) { if (p == null) { continue; } def dom = p.policy_domain != null ? p.policy_domain : \"none\"; def typ = p.policy_type != null ? p.policy_type : \"none\"; pols.add(dom + \" / \" + typ); def fds = p.failure_details; if (fds != null) { if (!(fds instanceof List)) { fds = [fds]; } for (f in fds) { if (f == null) { continue; } def rt = f.result_type != null ? f.result_type : \"none\"; def smi = f.sending_mta_ip != null ? f.sending_mta_ip : \"none\"; def ri = f.receiving_ip != null ? f.receiving_ip : \"none\"; def rmh = f.receiving_mx_hostname != null ? f.receiving_mx_hostname : \"none\"; dets.add(dom + \" / \" + typ + \" / \" + rt + \" / \" + smi + \" / \" + ri + \" / \" + rmh); } } } } ctx._source.policies_combined = pols; ctx._source.failure_details_combined = dets;"
+  }
+}'
+

+

+
It works identically on OpenSearch; just adjust the URL and credentials, same
+as the aggregate command above.

 
 

 
Records retention

diff --git a/genindex.html b/genindex.html
index 04d38a80..1e99e0fb 100644
--- a/genindex.html
+++ b/genindex.html
@@ -5,14 +5,14 @@
 
   
   
-  Index — parsedmarc 10.2.4 documentation
+  Index — parsedmarc 10.3.0 documentation
       
       
 
   
       
       
-      
+      
       
       
     
diff --git a/index.html b/index.html
index 496346e6..48f3ee0c 100644
--- a/index.html
+++ b/index.html
@@ -6,14 +6,14 @@
   
 
   
-  parsedmarc documentation - Open source DMARC report analyzer and visualizer — parsedmarc 10.2.4 documentation
+  parsedmarc documentation - Open source DMARC report analyzer and visualizer — parsedmarc 10.3.0 documentation
       
       
 
   
       
       
-      
+      
       
       
     
diff --git a/installation.html b/installation.html
index 221ce3f6..d2b7cd84 100644
--- a/installation.html
+++ b/installation.html
@@ -6,14 +6,14 @@
   
 
   
-  Installation — parsedmarc 10.2.4 documentation
+  Installation — parsedmarc 10.3.0 documentation
       
       
 
   
       
       
-      
+      
       
       
     
diff --git a/kibana.html b/kibana.html
index 5ab73543..12d762b2 100644
--- a/kibana.html
+++ b/kibana.html
@@ -6,14 +6,14 @@
   
 
   
-  Using the Kibana dashboards — parsedmarc 10.2.4 documentation
+  Using the Kibana dashboards — parsedmarc 10.3.0 documentation
       
       
 
   
       
       
-      
+      
       
       
     
@@ -164,7 +164,18 @@ table. Each row of the DKIM details table is one real DKIM signature, shown
 as a combined selector / domain / result value; the SPF details table
 shows scope / domain / result the same way. Combining the values into one
 column keeps each signature’s selector, domain, and result paired together,
-rather than aggregating them as separate columns.

+rather than aggregating them as separate columns. Because a message that
+carries multiple DKIM signatures appears once per signature, summing the
+messages column across rows can exceed the total number of messages.

+
The “Auth result filters” panel above the details tables
+provides dropdowns for the individual auth-result components — DKIM
+selector, DKIM domain, DKIM result, SPF scope, SPF domain, and SPF
+result — and filters the whole dashboard by them. Because components from
+different signatures of the same message are indexed together, combining
+two of these component filters matches documents where any signature
+satisfies each condition individually, not necessarily the same signature;
+the combined selector / domain / result (scope / domain / result)
+column remains the per-signature source of truth.

 

 
Note

 
The alignment tables (SPF details, DKIM details) and the per-IP source
@@ -193,6 +204,17 @@ reporting organizations, the policy domains they report on, and the
 specific failure types — certificate expiry, STARTTLS not supported,
 STS policy fetch errors, validation failures, and similar — together with
 the sending and receiving MTA addresses involved.

+
Like the DKIM and SPF details tables above, the “SMTP TLS domains” and
+“SMTP TLS failure details” tables show one row per policy and one row per
+failure detail, respectively, using combined policy (domain / type) and
+failure detail (domain / type / result / sending mta / receiving ip / mx)
+columns so that each policy’s or failure detail’s fields stay paired
+together, rather than aggregating them as separate columns. The
+successful_sessions and failed_sessions columns are summed per report
+document, though, not per policy: when a single report carries multiple
+policies, a row’s session sums include the sibling policies from that
+report as well as its own. Fully attributing session counts to a single
+policy would require restructuring the stored documents.

 

 
 
diff --git a/mailing-lists.html b/mailing-lists.html
index 9eb48a7c..ff7e836d 100644
--- a/mailing-lists.html
+++ b/mailing-lists.html
@@ -6,14 +6,14 @@
   
 
   
-  What about mailing lists? — parsedmarc 10.2.4 documentation
+  What about mailing lists? — parsedmarc 10.3.0 documentation
       
       
 
   
       
       
-      
+      
       
       
     
diff --git a/objects.inv b/objects.inv
index 2779d19f..c20d9ea5 100644
Binary files a/objects.inv and b/objects.inv differ
diff --git a/opensearch.html b/opensearch.html
index d95b4663..3d5b9e01 100644
--- a/opensearch.html
+++ b/opensearch.html
@@ -6,14 +6,14 @@
   
 
   
-  OpenSearch and Grafana — parsedmarc 10.2.4 documentation
+  OpenSearch and Grafana — parsedmarc 10.3.0 documentation
       
       
 
   
       
       
-      
+      
       
       
     
diff --git a/output.html b/output.html
index 934354f3..57540b4e 100644
--- a/output.html
+++ b/output.html
@@ -6,14 +6,14 @@
   
 
   
-  Sample outputs — parsedmarc 10.2.4 documentation
+  Sample outputs — parsedmarc 10.3.0 documentation
       
       
 
   
       
       
-      
+      
       
       
     
diff --git a/py-modindex.html b/py-modindex.html
index bb30c353..a11bdc7d 100644
--- a/py-modindex.html
+++ b/py-modindex.html
@@ -5,14 +5,14 @@
 
   
   
-  Python Module Index — parsedmarc 10.2.4 documentation
+  Python Module Index — parsedmarc 10.3.0 documentation
       
       
 
   
       
       
-      
+      
       
       
     
diff --git a/search.html b/search.html
index 3b0906bc..efd021f7 100644
--- a/search.html
+++ b/search.html
@@ -5,7 +5,7 @@
 
   
   
-  Search — parsedmarc 10.2.4 documentation
+  Search — parsedmarc 10.3.0 documentation
       
       
 
@@ -13,7 +13,7 @@
     
       
       
-      
+      
       
       
     
diff --git a/searchindex.js b/searchindex.js
index 2fa851ee..cf45ff11 100644
--- a/searchindex.js
+++ b/searchindex.js
@@ -1 +1 @@
-Search.setIndex({"alltitles":{"API reference":[[0,null]],"Accessing an inbox using OWA/EWS":[[2,null]],"Backfilling the combined DKIM/SPF result fields":[[4,"backfilling-the-combined-dkim-spf-result-fields"]],"Bug reports":[[1,"bug-reports"]],"CLI help":[[12,"cli-help"]],"CSV aggregate report":[[10,"csv-aggregate-report"]],"CSV failure report":[[10,"csv-failure-report"]],"Configuration file":[[12,"configuration-file"]],"Configuring parsedmarc for DavMail":[[2,"configuring-parsedmarc-for-davmail"]],"Contents":[[5,null]],"Contributing to parsedmarc":[[1,null]],"DMARC Alignment Guide":[[3,"dmarc-alignment-guide"]],"DMARC aggregate reports":[[7,"dmarc-aggregate-reports"]],"DMARC failure reports":[[7,"dmarc-failure-reports"]],"DMARC guides":[[3,"dmarc-guides"]],"Do":[[3,"do"],[8,"do"]],"Do not":[[3,"do-not"],[8,"do-not"]],"Docker Compose example":[[12,"docker-compose-example"]],"Docker secrets (_FILE suffix)":[[12,"docker-secrets-file-suffix"]],"Elasticsearch and Kibana":[[4,null]],"Environment variable configuration":[[12,"environment-variable-configuration"]],"Examples":[[12,"examples"]],"Features":[[5,"features"]],"IP-to-country database":[[6,"ip-to-country-database"]],"Indices and tables":[[0,"indices-and-tables"]],"Installation":[[4,"installation"],[6,null],[9,"installation"]],"Installing parsedmarc":[[6,"installing-parsedmarc"]],"JSON SMTP TLS report":[[10,"json-smtp-tls-report"]],"JSON aggregate report":[[10,"json-aggregate-report"]],"JSON failure report":[[10,"json-failure-report"]],"LISTSERV":[[3,"listserv"],[8,"listserv"]],"Lookalike domains":[[3,"lookalike-domains"]],"Mailing list best practices":[[3,"mailing-list-best-practices"],[8,"mailing-list-best-practices"]],"Mailman 2":[[3,"mailman-2"],[3,"id1"],[8,"mailman-2"],[8,"id1"]],"Mailman 3":[[3,"mailman-3"],[3,"id2"],[8,"mailman-3"],[8,"id2"]],"Multi-tenant support":[[12,"multi-tenant-support"]],"OpenSearch and Grafana":[[9,null]],"Optional dependencies":[[6,"optional-dependencies"]],"Performance tuning":[[12,"performance-tuning"]],"Prerequisites":[[6,"prerequisites"]],"Python Compatibility":[[5,"python-compatibility"]],"Records retention":[[4,"records-retention"],[9,"records-retention"]],"Reloading configuration without restarting":[[12,"reloading-configuration-without-restarting"]],"Resources":[[3,"resources"]],"Running DavMail as a systemd service":[[2,"running-davmail-as-a-systemd-service"]],"Running parsedmarc as a systemd service":[[12,"running-parsedmarc-as-a-systemd-service"]],"Running without a config file (env-only mode)":[[12,"running-without-a-config-file-env-only-mode"]],"SMTP TLS reporting":[[7,"smtp-tls-reporting"]],"SPF and DMARC record validation":[[3,"spf-and-dmarc-record-validation"]],"Sample aggregate report output":[[10,"sample-aggregate-report-output"]],"Sample failure report output":[[10,"sample-failure-report-output"]],"Sample outputs":[[10,null]],"Section name mapping":[[12,"section-name-mapping"]],"Specifying the config file via environment variable":[[12,"specifying-the-config-file-via-environment-variable"]],"Splunk":[[11,null]],"Testing multiple report analyzers":[[6,"testing-multiple-report-analyzers"]],"Understanding DMARC":[[3,null]],"Upgrading Kibana index patterns":[[4,"upgrading-kibana-index-patterns"]],"Using MaxMind GeoLite2 (optional)":[[6,"using-maxmind-geolite2-optional"]],"Using Microsoft Exchange":[[6,"using-microsoft-exchange"]],"Using a web proxy":[[6,"using-a-web-proxy"]],"Using parsedmarc":[[12,null]],"Using the Kibana dashboards":[[7,null]],"What about mailing lists?":[[3,"what-about-mailing-lists"],[8,null]],"What if a sender won\u2019t support DKIM/DMARC?":[[3,"what-if-a-sender-wont-support-dkim-dmarc"]],"Workarounds":[[3,"workarounds"],[8,"workarounds"]],"parsedmarc":[[0,"module-parsedmarc"]],"parsedmarc documentation - Open source DMARC report analyzer and visualizer":[[5,null]],"parsedmarc.elastic":[[0,"module-parsedmarc.elastic"]],"parsedmarc.opensearch":[[0,"module-parsedmarc.opensearch"]],"parsedmarc.splunk":[[0,"module-parsedmarc.splunk"]],"parsedmarc.types":[[0,"module-parsedmarc.types"]],"parsedmarc.utils":[[0,"module-parsedmarc.utils"]]},"docnames":["api","contributing","davmail","dmarc","elasticsearch","index","installation","kibana","mailing-lists","opensearch","output","splunk","usage"],"envversion":{"sphinx":66,"sphinx.domains.c":3,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":9,"sphinx.domains.index":1,"sphinx.domains.javascript":3,"sphinx.domains.math":2,"sphinx.domains.python":4,"sphinx.domains.rst":2,"sphinx.domains.std":2,"sphinx.ext.todo":2,"sphinx.ext.viewcode":1},"filenames":["api.md","contributing.md","davmail.md","dmarc.md","elasticsearch.md","index.md","installation.md","kibana.md","mailing-lists.md","opensearch.md","output.md","splunk.md","usage.md"],"indexentries":{"aggregatealignment (class in parsedmarc.types)":[[0,"parsedmarc.types.AggregateAlignment",false]],"aggregateauthresultdkim (class in parsedmarc.types)":[[0,"parsedmarc.types.AggregateAuthResultDKIM",false]],"aggregateauthresults (class in parsedmarc.types)":[[0,"parsedmarc.types.AggregateAuthResults",false]],"aggregateauthresultspf (class in parsedmarc.types)":[[0,"parsedmarc.types.AggregateAuthResultSPF",false]],"aggregateidentifiers (class in parsedmarc.types)":[[0,"parsedmarc.types.AggregateIdentifiers",false]],"aggregateparsedreport (class in parsedmarc.types)":[[0,"parsedmarc.types.AggregateParsedReport",false]],"aggregatepolicyevaluated (class in parsedmarc.types)":[[0,"parsedmarc.types.AggregatePolicyEvaluated",false]],"aggregatepolicyoverridereason (class in parsedmarc.types)":[[0,"parsedmarc.types.AggregatePolicyOverrideReason",false]],"aggregatepolicypublished (class in parsedmarc.types)":[[0,"parsedmarc.types.AggregatePolicyPublished",false]],"aggregaterecord (class in parsedmarc.types)":[[0,"parsedmarc.types.AggregateRecord",false]],"aggregatereport (class in parsedmarc.types)":[[0,"parsedmarc.types.AggregateReport",false]],"aggregatereportmetadata (class in parsedmarc.types)":[[0,"parsedmarc.types.AggregateReportMetadata",false]],"alreadysaved":[[0,"parsedmarc.elastic.AlreadySaved",false],[0,"parsedmarc.opensearch.AlreadySaved",false]],"append_json() (in module parsedmarc)":[[0,"parsedmarc.append_json",false]],"close() (parsedmarc.splunk.hecclient method)":[[0,"parsedmarc.splunk.HECClient.close",false]],"configure_ipinfo_api() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.configure_ipinfo_api",false]],"convert_outlook_msg() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.convert_outlook_msg",false]],"create_indexes() (in module parsedmarc.elastic)":[[0,"parsedmarc.elastic.create_indexes",false]],"create_indexes() (in module parsedmarc.opensearch)":[[0,"parsedmarc.opensearch.create_indexes",false]],"decode_base64() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.decode_base64",false]],"downloaderror":[[0,"parsedmarc.utils.DownloadError",false]],"elasticsearcherror":[[0,"parsedmarc.elastic.ElasticsearchError",false]],"email_results() (in module parsedmarc)":[[0,"parsedmarc.email_results",false]],"email_results_via_msgraph() (in module parsedmarc)":[[0,"parsedmarc.email_results_via_msgraph",false]],"emailaddress (class in parsedmarc.types)":[[0,"parsedmarc.types.EmailAddress",false]],"emailattachment (class in parsedmarc.types)":[[0,"parsedmarc.types.EmailAttachment",false]],"emailparsererror":[[0,"parsedmarc.utils.EmailParserError",false]],"extract_report() (in module parsedmarc)":[[0,"parsedmarc.extract_report",false]],"extract_report_from_file_path() (in module parsedmarc)":[[0,"parsedmarc.extract_report_from_file_path",false]],"failureparsedreport (class in parsedmarc.types)":[[0,"parsedmarc.types.FailureParsedReport",false]],"failurereport (class in parsedmarc.types)":[[0,"parsedmarc.types.FailureReport",false]],"forensicparsedreport (in module parsedmarc.types)":[[0,"parsedmarc.types.ForensicParsedReport",false]],"forensicreport (in module parsedmarc.types)":[[0,"parsedmarc.types.ForensicReport",false]],"get_base_domain() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.get_base_domain",false]],"get_dmarc_reports_from_mailbox() (in module parsedmarc)":[[0,"parsedmarc.get_dmarc_reports_from_mailbox",false]],"get_dmarc_reports_from_mbox() (in module parsedmarc)":[[0,"parsedmarc.get_dmarc_reports_from_mbox",false]],"get_filename_safe_string() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.get_filename_safe_string",false]],"get_ip_address_country() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.get_ip_address_country",false]],"get_ip_address_db_record() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.get_ip_address_db_record",false]],"get_ip_address_info() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.get_ip_address_info",false]],"get_report_zip() (in module parsedmarc)":[[0,"parsedmarc.get_report_zip",false]],"get_reverse_dns() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.get_reverse_dns",false]],"get_service_from_reverse_dns_base_domain() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.get_service_from_reverse_dns_base_domain",false]],"hecclient (class in parsedmarc.splunk)":[[0,"parsedmarc.splunk.HECClient",false]],"human_timestamp_to_datetime() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.human_timestamp_to_datetime",false]],"human_timestamp_to_unix_timestamp() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.human_timestamp_to_unix_timestamp",false]],"invalidaggregatereport":[[0,"parsedmarc.InvalidAggregateReport",false]],"invaliddmarcreport":[[0,"parsedmarc.InvalidDMARCReport",false]],"invalidfailurereport":[[0,"parsedmarc.InvalidFailureReport",false]],"invalidforensicreport (in module parsedmarc)":[[0,"parsedmarc.InvalidForensicReport",false]],"invalidipinfoapikey":[[0,"parsedmarc.utils.InvalidIPinfoAPIKey",false]],"invalidsmtptlsreport":[[0,"parsedmarc.InvalidSMTPTLSReport",false]],"ipaddressinfo (class in parsedmarc.utils)":[[0,"parsedmarc.utils.IPAddressInfo",false]],"ipsourceinfo (class in parsedmarc.types)":[[0,"parsedmarc.types.IPSourceInfo",false]],"is_mbox() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.is_mbox",false]],"is_outlook_msg() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.is_outlook_msg",false]],"load_ip_db() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.load_ip_db",false]],"load_psl_overrides() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.load_psl_overrides",false]],"load_reverse_dns_map() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.load_reverse_dns_map",false]],"migrate_indexes() (in module parsedmarc.elastic)":[[0,"parsedmarc.elastic.migrate_indexes",false]],"migrate_indexes() (in module parsedmarc.opensearch)":[[0,"parsedmarc.opensearch.migrate_indexes",false]],"module":[[0,"module-parsedmarc",false],[0,"module-parsedmarc.elastic",false],[0,"module-parsedmarc.opensearch",false],[0,"module-parsedmarc.splunk",false],[0,"module-parsedmarc.types",false],[0,"module-parsedmarc.utils",false]],"opensearcherror":[[0,"parsedmarc.opensearch.OpenSearchError",false]],"parse_aggregate_report_file() (in module parsedmarc)":[[0,"parsedmarc.parse_aggregate_report_file",false]],"parse_aggregate_report_xml() (in module parsedmarc)":[[0,"parsedmarc.parse_aggregate_report_xml",false]],"parse_email() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.parse_email",false]],"parse_failure_report() (in module parsedmarc)":[[0,"parsedmarc.parse_failure_report",false]],"parse_forensic_report() (in module parsedmarc)":[[0,"parsedmarc.parse_forensic_report",false]],"parse_report_email() (in module parsedmarc)":[[0,"parsedmarc.parse_report_email",false]],"parse_report_file() (in module parsedmarc)":[[0,"parsedmarc.parse_report_file",false]],"parse_smtp_tls_report_json() (in module parsedmarc)":[[0,"parsedmarc.parse_smtp_tls_report_json",false]],"parsed_aggregate_reports_to_csv() (in module parsedmarc)":[[0,"parsedmarc.parsed_aggregate_reports_to_csv",false]],"parsed_aggregate_reports_to_csv_rows() (in module parsedmarc)":[[0,"parsedmarc.parsed_aggregate_reports_to_csv_rows",false]],"parsed_failure_reports_to_csv() (in module parsedmarc)":[[0,"parsedmarc.parsed_failure_reports_to_csv",false]],"parsed_failure_reports_to_csv_rows() (in module parsedmarc)":[[0,"parsedmarc.parsed_failure_reports_to_csv_rows",false]],"parsed_forensic_reports_to_csv() (in module parsedmarc)":[[0,"parsedmarc.parsed_forensic_reports_to_csv",false]],"parsed_forensic_reports_to_csv_rows() (in module parsedmarc)":[[0,"parsedmarc.parsed_forensic_reports_to_csv_rows",false]],"parsed_smtp_tls_reports_to_csv() (in module parsedmarc)":[[0,"parsedmarc.parsed_smtp_tls_reports_to_csv",false]],"parsed_smtp_tls_reports_to_csv_rows() (in module parsedmarc)":[[0,"parsedmarc.parsed_smtp_tls_reports_to_csv_rows",false]],"parsedemail (class in parsedmarc.types)":[[0,"parsedmarc.types.ParsedEmail",false]],"parsedmarc":[[0,"module-parsedmarc",false]],"parsedmarc.elastic":[[0,"module-parsedmarc.elastic",false]],"parsedmarc.opensearch":[[0,"module-parsedmarc.opensearch",false]],"parsedmarc.splunk":[[0,"module-parsedmarc.splunk",false]],"parsedmarc.types":[[0,"module-parsedmarc.types",false]],"parsedmarc.utils":[[0,"module-parsedmarc.utils",false]],"parsererror":[[0,"parsedmarc.ParserError",false]],"parsingresults (class in parsedmarc.types)":[[0,"parsedmarc.types.ParsingResults",false]],"query_dns() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.query_dns",false]],"reversednsservice (class in parsedmarc.utils)":[[0,"parsedmarc.utils.ReverseDNSService",false]],"save_aggregate_report_to_elasticsearch() (in module parsedmarc.elastic)":[[0,"parsedmarc.elastic.save_aggregate_report_to_elasticsearch",false]],"save_aggregate_report_to_opensearch() (in module parsedmarc.opensearch)":[[0,"parsedmarc.opensearch.save_aggregate_report_to_opensearch",false]],"save_aggregate_reports_to_splunk() (parsedmarc.splunk.hecclient method)":[[0,"parsedmarc.splunk.HECClient.save_aggregate_reports_to_splunk",false]],"save_failure_report_to_elasticsearch() (in module parsedmarc.elastic)":[[0,"parsedmarc.elastic.save_failure_report_to_elasticsearch",false]],"save_failure_report_to_opensearch() (in module parsedmarc.opensearch)":[[0,"parsedmarc.opensearch.save_failure_report_to_opensearch",false]],"save_failure_reports_to_splunk() (parsedmarc.splunk.hecclient method)":[[0,"parsedmarc.splunk.HECClient.save_failure_reports_to_splunk",false]],"save_forensic_report_to_elasticsearch() (in module parsedmarc.elastic)":[[0,"parsedmarc.elastic.save_forensic_report_to_elasticsearch",false]],"save_forensic_report_to_opensearch() (in module parsedmarc.opensearch)":[[0,"parsedmarc.opensearch.save_forensic_report_to_opensearch",false]],"save_forensic_reports_to_splunk() (parsedmarc.splunk.hecclient method)":[[0,"parsedmarc.splunk.HECClient.save_forensic_reports_to_splunk",false]],"save_output() (in module parsedmarc)":[[0,"parsedmarc.save_output",false]],"save_smtp_tls_report_to_elasticsearch() (in module parsedmarc.elastic)":[[0,"parsedmarc.elastic.save_smtp_tls_report_to_elasticsearch",false]],"save_smtp_tls_report_to_opensearch() (in module parsedmarc.opensearch)":[[0,"parsedmarc.opensearch.save_smtp_tls_report_to_opensearch",false]],"save_smtp_tls_reports_to_splunk() (parsedmarc.splunk.hecclient method)":[[0,"parsedmarc.splunk.HECClient.save_smtp_tls_reports_to_splunk",false]],"set_hosts() (in module parsedmarc.elastic)":[[0,"parsedmarc.elastic.set_hosts",false]],"set_hosts() (in module parsedmarc.opensearch)":[[0,"parsedmarc.opensearch.set_hosts",false]],"smtptlsfailuredetails (class in parsedmarc.types)":[[0,"parsedmarc.types.SMTPTLSFailureDetails",false]],"smtptlsfailuredetailsoptional (class in parsedmarc.types)":[[0,"parsedmarc.types.SMTPTLSFailureDetailsOptional",false]],"smtptlsparsedreport (class in parsedmarc.types)":[[0,"parsedmarc.types.SMTPTLSParsedReport",false]],"smtptlspolicy (class in parsedmarc.types)":[[0,"parsedmarc.types.SMTPTLSPolicy",false]],"smtptlspolicysummary (class in parsedmarc.types)":[[0,"parsedmarc.types.SMTPTLSPolicySummary",false]],"smtptlsreport (class in parsedmarc.types)":[[0,"parsedmarc.types.SMTPTLSReport",false]],"splunkerror":[[0,"parsedmarc.splunk.SplunkError",false]],"timestamp_to_datetime() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.timestamp_to_datetime",false]],"timestamp_to_human() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.timestamp_to_human",false]],"watch_inbox() (in module parsedmarc)":[[0,"parsedmarc.watch_inbox",false]]},"objects":{"":[[0,0,0,"-","parsedmarc"]],"parsedmarc":[[0,1,1,"","InvalidAggregateReport"],[0,1,1,"","InvalidDMARCReport"],[0,1,1,"","InvalidFailureReport"],[0,2,1,"","InvalidForensicReport"],[0,1,1,"","InvalidSMTPTLSReport"],[0,1,1,"","ParserError"],[0,3,1,"","append_json"],[0,0,0,"-","elastic"],[0,3,1,"","email_results"],[0,3,1,"","email_results_via_msgraph"],[0,3,1,"","extract_report"],[0,3,1,"","extract_report_from_file_path"],[0,3,1,"","get_dmarc_reports_from_mailbox"],[0,3,1,"","get_dmarc_reports_from_mbox"],[0,3,1,"","get_report_zip"],[0,0,0,"-","opensearch"],[0,3,1,"","parse_aggregate_report_file"],[0,3,1,"","parse_aggregate_report_xml"],[0,3,1,"","parse_failure_report"],[0,3,1,"","parse_forensic_report"],[0,3,1,"","parse_report_email"],[0,3,1,"","parse_report_file"],[0,3,1,"","parse_smtp_tls_report_json"],[0,3,1,"","parsed_aggregate_reports_to_csv"],[0,3,1,"","parsed_aggregate_reports_to_csv_rows"],[0,3,1,"","parsed_failure_reports_to_csv"],[0,3,1,"","parsed_failure_reports_to_csv_rows"],[0,3,1,"","parsed_forensic_reports_to_csv"],[0,3,1,"","parsed_forensic_reports_to_csv_rows"],[0,3,1,"","parsed_smtp_tls_reports_to_csv"],[0,3,1,"","parsed_smtp_tls_reports_to_csv_rows"],[0,3,1,"","save_output"],[0,0,0,"-","splunk"],[0,0,0,"-","types"],[0,0,0,"-","utils"],[0,3,1,"","watch_inbox"]],"parsedmarc.elastic":[[0,1,1,"","AlreadySaved"],[0,1,1,"","ElasticsearchError"],[0,3,1,"","create_indexes"],[0,3,1,"","migrate_indexes"],[0,3,1,"","save_aggregate_report_to_elasticsearch"],[0,3,1,"","save_failure_report_to_elasticsearch"],[0,3,1,"","save_forensic_report_to_elasticsearch"],[0,3,1,"","save_smtp_tls_report_to_elasticsearch"],[0,3,1,"","set_hosts"]],"parsedmarc.opensearch":[[0,1,1,"","AlreadySaved"],[0,1,1,"","OpenSearchError"],[0,3,1,"","create_indexes"],[0,3,1,"","migrate_indexes"],[0,3,1,"","save_aggregate_report_to_opensearch"],[0,3,1,"","save_failure_report_to_opensearch"],[0,3,1,"","save_forensic_report_to_opensearch"],[0,3,1,"","save_smtp_tls_report_to_opensearch"],[0,3,1,"","set_hosts"]],"parsedmarc.splunk":[[0,4,1,"","HECClient"],[0,1,1,"","SplunkError"]],"parsedmarc.splunk.HECClient":[[0,5,1,"","close"],[0,5,1,"","save_aggregate_reports_to_splunk"],[0,5,1,"","save_failure_reports_to_splunk"],[0,5,1,"","save_forensic_reports_to_splunk"],[0,5,1,"","save_smtp_tls_reports_to_splunk"]],"parsedmarc.types":[[0,4,1,"","AggregateAlignment"],[0,4,1,"","AggregateAuthResultDKIM"],[0,4,1,"","AggregateAuthResultSPF"],[0,4,1,"","AggregateAuthResults"],[0,4,1,"","AggregateIdentifiers"],[0,4,1,"","AggregateParsedReport"],[0,4,1,"","AggregatePolicyEvaluated"],[0,4,1,"","AggregatePolicyOverrideReason"],[0,4,1,"","AggregatePolicyPublished"],[0,4,1,"","AggregateRecord"],[0,4,1,"","AggregateReport"],[0,4,1,"","AggregateReportMetadata"],[0,4,1,"","EmailAddress"],[0,4,1,"","EmailAttachment"],[0,4,1,"","FailureParsedReport"],[0,4,1,"","FailureReport"],[0,2,1,"","ForensicParsedReport"],[0,2,1,"","ForensicReport"],[0,4,1,"","IPSourceInfo"],[0,4,1,"","ParsedEmail"],[0,4,1,"","ParsingResults"],[0,4,1,"","SMTPTLSFailureDetails"],[0,4,1,"","SMTPTLSFailureDetailsOptional"],[0,4,1,"","SMTPTLSParsedReport"],[0,4,1,"","SMTPTLSPolicy"],[0,4,1,"","SMTPTLSPolicySummary"],[0,4,1,"","SMTPTLSReport"]],"parsedmarc.utils":[[0,1,1,"","DownloadError"],[0,1,1,"","EmailParserError"],[0,4,1,"","IPAddressInfo"],[0,1,1,"","InvalidIPinfoAPIKey"],[0,4,1,"","ReverseDNSService"],[0,3,1,"","configure_ipinfo_api"],[0,3,1,"","convert_outlook_msg"],[0,3,1,"","decode_base64"],[0,3,1,"","get_base_domain"],[0,3,1,"","get_filename_safe_string"],[0,3,1,"","get_ip_address_country"],[0,3,1,"","get_ip_address_db_record"],[0,3,1,"","get_ip_address_info"],[0,3,1,"","get_reverse_dns"],[0,3,1,"","get_service_from_reverse_dns_base_domain"],[0,3,1,"","human_timestamp_to_datetime"],[0,3,1,"","human_timestamp_to_unix_timestamp"],[0,3,1,"","is_mbox"],[0,3,1,"","is_outlook_msg"],[0,3,1,"","load_ip_db"],[0,3,1,"","load_psl_overrides"],[0,3,1,"","load_reverse_dns_map"],[0,3,1,"","parse_email"],[0,3,1,"","query_dns"],[0,3,1,"","timestamp_to_datetime"],[0,3,1,"","timestamp_to_human"]]},"objnames":{"0":["py","module","Python module"],"1":["py","exception","Python exception"],"2":["py","attribute","Python attribute"],"3":["py","function","Python function"],"4":["py","class","Python class"],"5":["py","method","Python method"]},"objtypes":{"0":"py:module","1":"py:exception","2":"py:attribute","3":"py:function","4":"py:class","5":"py:method"},"terms":{"00z":10,"00z_exampl":10,"09t00":10,"09t23":10,"1d":12,"1g":4,"1w":12,"2017a":[3,8],"21vianet":12,"2d":12,"2k":12,"30s":12,"3d":10,"3h":12,"59z":10,"5m":[2,12],"7d":12,"A":[0,3,7,12],"AT":10,"After":[2,4,12],"All":[3,8,12],"An":[0,12],"And":0,"As":[4,7],"By":[7,12],"Do":[0,12],"Each":[7,12],"For":[4,12],"From":[3,8,12],"Further":7,"Have":12,"Here":10,"IF":12,"If":[0,3,4,6,7,8,12],"In":[0,2,3,7,8,12],"It":[2,4,7,10,12],"More":6,"Most":[7,12],"NOT":12,"No":[3,6,8],"Not":12,"On":[3,4,6,7,8,12],"Or":[4,6,12],"Some":[2,3,7,8],"That":7,"The":[0,3,4,6,7,11,12],"Then":[2,3,4,6,8,12],"There":7,"These":[7,12],"This":[0,5,6,7,10,12],"To":[2,4,6,7,9,10,12],"Under":[4,7],"What":5,"When":[0,3,5,8,12],"Where":[3,8],"While":[7,12],"With":[7,12],"YOUR":4,"You":[2,7,12],"_":12,"_attempt":0,"_cluster":12,"_input":0,"_ipdatabaserecord":0,"_serverless_rejected_set":0,"_sourc":4,"_task":4,"_update_by_queri":4,"aadst":12,"abl":6,"abort":12,"abov":[2,4,6,12],"absent":4,"accept":[0,3,4,7,8,12],"access":[0,4,5,12],"access_key_id":12,"access_token":0,"accessright":12,"accident":[3,8],"account":[6,7,12],"acm":10,"acquir":12,"acquisit":12,"across":7,"action":[3,8,12],"activ":[4,5,12],"active_primary_shard":12,"active_shard":12,"actual":[3,10],"ad":12,"add":[2,3,4,6,7,8,12],"addit":[3,8,12],"address":[0,2,3,4,6,7,8,10,12],"addresse":7,"adjust":4,"adkim":10,"admin":[3,8,12],"administr":[3,8,12],"advanc":7,"agari":5,"agent":4,"agg":7,"aggreg":[0,4,5,11,12],"aggregate_csv_filenam":[0,12],"aggregate_index":0,"aggregate_json_filenam":[0,12],"aggregate_report":0,"aggregate_top":12,"aggregate_url":12,"aggregatealign":0,"aggregateauthresult":0,"aggregateauthresultdkim":0,"aggregateauthresultspf":0,"aggregateidentifi":0,"aggregateparsedreport":0,"aggregatepolicyevalu":0,"aggregatepolicyoverridereason":0,"aggregatepolicypublish":0,"aggregaterecord":0,"aggregatereport":0,"aggregatereportmetadata":0,"aggress":12,"alia":[0,12],"alias":12,"align":[4,5,7,10],"aliv":0,"allow":[2,3,8,12],"allow_unencrypted_storag":12,"allowremot":2,"allowstringindic":7,"alon":12,"alreadi":[0,4,12],"alreadysav":0,"also":[0,2,3,4,6,7,8,12],"alter":[3,8],"altern":[5,12],"although":11,"alway":[0,2,4,12],"always_use_local_fil":[0,12],"amazon":5,"amd64":12,"amount":12,"analyt":[5,12],"analyz":12,"ani":[0,3,6,7,8,12],"anonym":10,"anoth":[6,12],"answer":[0,12],"apach":5,"api":[2,4,5,12],"api_key":[0,12],"app":12,"appear":[4,12],"append":[0,12],"append_json":0,"appendix":10,"appid":12,"appli":[0,12],"applic":[4,12],"applicationaccesspolici":12,"approach":12,"approxim":2,"apt":[2,4,6],"archiv":[0,12],"archive_fold":[0,12],"argument":12,"arm64":12,"around":12,"array":[0,4],"arraylist":4,"arriv":12,"arrival_d":10,"arrival_date_utc":[0,10],"artifact":4,"as_domain":[0,10],"as_nam":[0,10],"ask":3,"asmx":2,"asn":[0,6,10,12],"aspf":10,"assert":12,"assign":4,"associ":0,"assum":12,"assume_utc":0,"astimezon":0,"att":10,"attach":[0,3,8,10,12],"attachment_filenam":0,"attempt":[0,12],"attribut":6,"auth":[0,2,4,10,12],"auth_failur":10,"auth_method":12,"auth_mod":12,"auth_result":10,"auth_typ":[0,12],"authent":[0,2,3,4,7,12],"authentication_mechan":10,"authentication_result":10,"authentication_typ":12,"auto":2,"automat":[6,12],"avail":[6,12],"avoid":[7,12],"aw":[0,12],"aws_region":[0,12],"aws_servic":[0,12],"awssigv4":[0,12],"az":12,"azur":[5,12],"b":10,"b2c":7,"back":[0,12],"backend":[0,12],"backfil":[5,12],"backlog":12,"backward":12,"bare":12,"base":[0,2,3,4,6,7,8,10,12],"base64":0,"base_domain":[0,10],"basic":[0,2,12],"batch":12,"batch_siz":[0,12],"bcc":[0,10],"bd6e1bb5":10,"becaus":[2,3,4,7,8,12],"becom":12,"befor":[0,12],"begin_d":10,"behavior":0,"behind":6,"benefit":5,"best":[7,12],"beyond":0,"bin":[2,4,6,12],"binari":[0,12],"binaryio":0,"bind":2,"bindaddress":2,"blank":[3,8],"block":[2,12],"bodi":[0,3,8,10,12],"bool":[0,4,12],"boundari":[0,12],"box":12,"brand":[5,7],"break":[3,4,8],"broken":0,"browser":4,"bucket":12,"budget":0,"bug":[5,12],"build":6,"built":0,"bundl":[0,6,7,12],"busi":7,"button":[3,8],"byte":0,"c":[10,12],"ca":[4,12],"cach":[0,12],"cafile_path":12,"call":[0,5,12],"callabl":0,"callback":0,"came":[3,8],"can":[0,2,3,4,5,6,7,8,12],"cap":[0,12],"carri":[0,6,12],"case":[2,3,8,12],"catch":[0,12],"caught":0,"caus":[3,4,7,8,12],"cc":[0,10],"center":7,"cento":[4,6],"cert":[4,12],"cert_path":12,"certain":[0,12],"certfile_path":12,"certif":[0,4,7,12],"certificate_password":12,"certificate_path":12,"cest":10,"chain":0,"chang":[4,7,11,12],"charact":[2,12],"charset":10,"chart":7,"check":[0,2,3,4,7,12],"check_timeout":[0,12],"checkbox":4,"checkdmarc":3,"china":12,"chinacloudapi":12,"chines":7,"chmod":[2,4,12],"choos":[3,8],"chown":[2,12],"ci":7,"cisco":12,"class":0,"clean":[0,12],"clear":0,"cli":[0,5],"click":[4,7],"client":[0,2,3,4,8,12],"client_assert":12,"client_id":12,"client_secret":12,"clientassert":12,"clientsecret":12,"clientsotimeout":2,"clock":0,"close":[0,12],"cloud":[0,12],"cloudflar":[0,12],"cluster":[4,12],"cn":12,"co":4,"code":[0,4,12],"collect":[7,12],"collector":[11,12],"column":7,"com":[1,2,3,8,9,10,12],"combin":[5,7,12],"come":[0,7],"comfort":12,"comma":[6,12],"command":[2,3,6,8,12],"comment":12,"commerci":[4,5],"common":[3,4,6,8],"communiti":[3,8],"compat":[0,7,12],"complet":[3,4,12],"compli":[3,4,6,8,9],"complianc":7,"compliant":[3,8],"compon":6,"compress":5,"comput":7,"conf":6,"config":[0,2,6],"config_fil":12,"config_reload":0,"configur":[0,3,4,5,6,7,8,9],"configure_ipinfo_api":0,"confirm":12,"conflict":4,"conform":4,"connect":[0,2,4,12],"connection_str":12,"consent":12,"consid":[5,7],"consist":[0,5,10],"consol":[4,12],"constant":0,"consult":6,"consum":[7,12],"contact":[7,12],"contain":[0,7,11,12],"content":[0,3,4,8,10,11,12],"continu":4,"contrib":6,"contribut":5,"control":[0,4,12],"convent":12,"convert":[0,3,8],"convert_outlook_msg":0,"copi":[0,6,11,12],"core":[3,8],"correct":[0,6,7,12],"correspond":[4,12],"corrupt":0,"count":[2,7,10],"countri":[0,7,10,12],"country_cod":0,"cpan":6,"cr":12,"crash":[2,4,12],"creat":[0,2,3,4,6,8,12],"create_fold":0,"create_index":0,"creation":12,"creativ":6,"credenti":[4,6,12],"credentials_fil":12,"cron":6,"cross":0,"crt":4,"csr":4,"csv":[0,5,12],"csvs":12,"ctrl":12,"ctx":4,"cumul":6,"curl":4,"current":[0,2,4,12],"custom":[4,7,12],"d":[0,4,12],"daemon":[2,4,12],"daili":[0,12],"dashboard":[4,5,9,11],"dat":0,"data":[0,4,5,6,7,9,11,12],"databas":[0,12],"date":[0,3,8,10],"date_utc":10,"datetim":0,"davmail":5,"day":[0,4,9,12],"db_path":0,"dbip":[0,6,12],"dbname":12,"dce":12,"dcr":12,"dcr_aggregate_stream":12,"dcr_failure_stream":12,"dcr_immutable_id":12,"dcr_smtp_tls_stream":12,"dd":[0,12],"de":10,"dearmor":4,"deb":4,"debian":[4,5,6],"debug":12,"decemb":6,"decod":0,"decode_base64":0,"dedic":6,"def":4,"default":[0,2,4,5,6,7,12],"defens":5,"delay":[2,10,12],"deleg":12,"delegated_us":12,"delet":[0,2,4,12],"deliber":12,"delivery_result":10,"demystifi":3,"deni":12,"depend":[0,4,5,12],"deploy":[3,8,12],"deprec":[7,12],"describ":12,"descript":[2,6,12],"design":12,"destin":[0,12],"detail":[4,6,7,12],"dev":[6,12],"devel":6,"develop":5,"devicecod":12,"dict":0,"dictionari":0,"differ":[7,12],"difficult":12,"dig":0,"digest":[3,8],"dir":6,"direct":[7,12],"directori":[0,6,12],"dis":10,"disabl":[0,2,6,12],"disclaim":[3,8],"disk":[0,12],"display":[3,7,11],"display_nam":10,"disposit":[7,10],"distinguish":12,"distribut":6,"distro":6,"dk":4,"dkim":[5,7,8,10],"dkim_align":10,"dkim_domain":10,"dkim_result":[4,10],"dkim_results_combin":4,"dkim_selector":10,"dkm":3,"dmarc":[0,4,6,8,9,10,11,12],"dmarc_aggreg":[4,7],"dmarc_align":10,"dmarc_failur":4,"dmarc_moderation_act":[3,8],"dmarc_none_moderation_act":[3,8],"dmarc_quarantine_moderation_act":[3,8],"dmarcian":5,"dmarcresport":12,"dnf":6,"dns":[0,3,6,7,12],"dns_retri":0,"dns_test_address":12,"dns_timeout":[0,12],"dnspython":0,"doc":[0,9,12],"doctyp":10,"document":[0,2,4,12],"dod":12,"doe":[3,8,12],"doesn":0,"dom":4,"domain":[0,4,7,8,10,12],"domainawar":[1,3,12],"don":3,"doubl":12,"download":[0,2,4,6,12],"downloaderror":0,"dr":4,"draft":[5,10,12],"dsn":12,"dtd":10,"dummi":12,"dure":[2,12],"e":[0,2,3,4,6,8,12],"e7":10,"earlier":[0,7],"easi":[4,9],"easier":[11,12],"echo":4,"edit":[2,6,7,12],"editor":11,"effect":12,"effici":4,"either":[5,12],"elast":[4,5,7,12],"elasticsearch":[0,5,12],"elasticsearcherror":0,"elig":12,"elk":12,"els":4,"email":[0,3,5,6,7,8,10,11,12],"email_result":0,"email_results_via_msgraph":0,"emailaddress":0,"emailattach":0,"emailparsererror":0,"empti":[0,3,4,8,12],"en":[3,4,8,10],"enabl":[2,4,7,12],"enableew":2,"enablekeepal":2,"enableproxi":2,"encod":[0,10,12],"encount":0,"encrypt":[4,12],"encryptedsavedobject":4,"encryptionkey":4,"end":[0,3,4,5,12],"end_dat":10,"endpoint":[5,12],"endpoint_url":12,"enforc":[3,8],"enough":12,"enrich":6,"enrol":4,"ensur":[3,6,8],"enterpris":12,"entir":[0,3,7,8,12],"entra":12,"entri":0,"envelop":3,"envelope_from":10,"envelope_to":10,"environ":[5,6],"eof":0,"eol":5,"epel":6,"era":0,"error":[0,7,10,12],"erroritemnotfound":12,"es":[0,12],"escap":12,"especi":[7,12],"etc":[2,3,4,6,8,12],"even":[2,3,8,12],"event":[2,11,12],"ever":0,"everi":[0,2,6,7,12],"ew":5,"ex":12,"exact":[3,8],"exampl":[3,4,6,8,10],"except":[0,12],"exchang":[2,10,12],"exclud":2,"exclus":0,"execreload":12,"execstart":[2,12],"exhaust":12,"exist":[0,3,4,8,12],"exit":[0,12],"expect":12,"expiri":7,"expiringdict":0,"explain":[3,8],"explicit":[0,3,6,8,12],"export":[4,7,12],"extra":12,"extract":[0,2],"extract_report":0,"extract_report_from_file_path":0,"eye":[2,12],"f":4,"factor":2,"fail":[0,3,7,8,10,12],"fail_on_output_error":12,"failed_session_count":10,"failov":0,"failur":[0,5,11,12],"failure_csv_filenam":[0,12],"failure_detail":10,"failure_index":0,"failure_json_filenam":[0,12],"failure_report":0,"failure_top":12,"failure_url":12,"failureparsedreport":0,"failurereport":0,"fall":[0,12],"fallback":[0,12],"fals":[0,2,4,10,12],"fantast":[3,8],"faster":12,"fatal":[0,12],"featur":[4,12],"feedback":0,"feedback_report":0,"feedback_typ":10,"fetch":[0,7,12],"field":[0,5],"file":[0,2,5,6,7,11],"file_path":[0,12],"filenam":[0,12],"filename_safe_subject":10,"filepath":12,"fill":[4,6],"filter":[0,3,7,8,11,12],"final":5,"financ":12,"find":[3,7,8,12],"fine":[3,8],"finish":12,"first":[0,3,6,7,8,12],"first_strip_reply_to":[3,8],"fit":[3,8,12],"fix":[4,12],"flag":[0,2,6,12],"flat":0,"flexibl":11,"flight":12,"float":[0,12],"flush":12,"fo":[0,10],"fold":0,"folder":[0,2,12],"foldersizelimit":2,"follow":[2,4,5,12],"footer":[3,8],"forc":12,"foreground":12,"forens":[5,7,12],"forensicparsedreport":0,"forensicreport":0,"form":12,"format":[0,7,12],"former":[5,7,12],"forward":[3,7,8],"found":[0,6,12],"foundat":10,"fqdn":4,"fraud":5,"free":[6,12],"fresh":12,"freshest":12,"friend":7,"from_is_list":[3,8],"ftp_proxi":6,"full":12,"fulli":[3,8,12],"function":0,"g":[0,2,3,4,6,8,12],"gateway":2,"gb":4,"gcc":[6,12],"gdpr":[4,9],"gelf":[5,12],"general":[3,6,8,12],"generat":[3,4,8,10],"geoip":[6,12],"geoipupd":6,"geolite2":5,"geoloc":[0,12],"get":[0,2,4,6,12],"get_base_domain":0,"get_dmarc_reports_from_mailbox":0,"get_dmarc_reports_from_mbox":0,"get_filename_safe_str":0,"get_ip_address_countri":0,"get_ip_address_db_record":0,"get_ip_address_info":0,"get_report_zip":0,"get_reverse_dn":0,"get_service_from_reverse_dns_base_domain":0,"ghcr":12,"github":[1,6,10,12],"give":[0,4],"given":[0,12],"glass":7,"global":12,"gmail":[5,7,12],"gmail_api":12,"go":[0,3,8],"goe":[3,8],"googl":[7,12],"googleapi":12,"got":12,"gov":12,"govern":12,"gpg":4,"grafana":5,"grant":12,"graph":[0,2,5,7,12],"graph_url":12,"graylog":5,"group":[2,7,12],"guid":[4,5],"guidanc":12,"gzip":[0,5],"h":[0,4,12],"hamburg":4,"hand":[3,8],"handl":[5,12],"handler":7,"happen":0,"hard":12,"has_defect":10,"hasn":12,"head":10,"header":[0,3,7,8,10,12],"header_from":10,"headless":2,"health":12,"healthcar":12,"heap":4,"heavi":[4,12],"hec":[0,11,12],"hecclient":0,"hectokengoesher":12,"help":5,"hh":0,"hierarchi":12,"high":[7,12],"higher":[3,8],"histor":12,"histori":12,"hit":[0,12],"home":6,"hop":10,"host":[0,2,3,4,5,8,12],"hostnam":[0,12],"hour":[0,12],"hover":7,"href":10,"html":[3,4,8,10],"http":[0,2,4,5,6,10,11,12],"http_proxi":6,"https":[0,1,2,3,4,6,8,9,12],"https_proxi":6,"httpx":12,"human":[0,7],"human_timestamp":0,"human_timestamp_to_datetim":0,"human_timestamp_to_unix_timestamp":0,"hup":12,"icon":7,"id":[3,4,8,10,12],"ideal":[3,8],"idempot":4,"ident":[3,4,8,12],"identifi":10,"idl":[0,2,12],"ignor":[0,12],"imag":12,"imap":[0,2,5,12],"imap_password":12,"imapalwaysapproxmsgs":2,"imapautoexpung":2,"imapcli":5,"imapidledelay":2,"imapport":2,"immedi":[2,12],"immut":12,"impli":12,"import":[4,7,12],"improv":12,"inbox":[0,3,5,8,12],"inc":10,"includ":[0,3,4,6,8,12],"include_list_post_head":[3,8],"include_rfc2369_head":[3,8],"include_sender_head":[3,8],"include_spam_trash":12,"incom":[7,12],"incorrect":12,"increas":[4,12],"increment":12,"indefinit":12,"indent":12,"index":[0,5,7,9,11,12],"index_prefix":[0,4,12],"index_prefix_domain_map":12,"index_suffix":[0,4,12],"indic":[3,5],"individu":12,"industri":12,"inform":[0,4,7,12],"infrequ":12,"ingest":12,"ini":[2,12],"initi":0,"inner":12,"input":0,"input_":0,"inspect":12,"instal":[2,5,12],"installed_app":12,"instanc":12,"instanceof":4,"instead":[0,3,4,6,8,12],"instruct":4,"int":[0,12],"intend":[3,8],"interact":[2,4,12],"interakt":10,"interfer":[3,8],"interpret":[0,6],"interrupt":12,"interval":12,"interval_begin":10,"interval_end":10,"invalid":[0,12],"invalidaggregatereport":0,"invaliddmarcreport":0,"invalidfailurereport":0,"invalidforensicreport":0,"invalidipinfoapikey":0,"invalidsmtptlsreport":0,"involv":7,"io":[0,12],"ip":[0,3,4,7,12],"ip_address":[0,10],"ip_db_path":[0,6,12],"ip_db_url":12,"ipaddressinfo":0,"ipinfo":[0,6,12],"ipinfo_api_token":12,"ipinfo_url":12,"ipsourceinfo":0,"ipv4":0,"ipv6":0,"is_mbox":0,"is_outlook_msg":0,"iso":[0,12],"issu":1,"item":[0,12],"java":2,"job":[3,6,8],"joe":[3,8],"journalctl":[2,12],"jre":2,"json":[0,4,5,12],"june":5,"junk":12,"just":[4,7],"jvm":4,"jwt":12,"kafka":[5,12],"kb4099855":6,"kb4134118":6,"kb4295699":6,"keep":[0,4,7,12],"keep_al":0,"keepal":2,"kept":0,"key":[0,3,4,6,12],"keyfile_path":12,"keyout":4,"keyr":4,"keystor":4,"kibana":[5,11],"kill":12,"killsign":12,"kind":12,"know":3,"known":[0,3,7,8,12],"kubernet":12,"kwarg":0,"l4":12,"l5":12,"label":12,"lack":4,"lang":4,"languag":[3,8],"larg":[2,12],"larger":12,"last":6,"later":[0,4,6,12],"latest":[2,4,9,12],"layer":0,"layout":11,"leak":7,"least":[4,6,12],"leav":3,"left":[0,7],"legaci":[0,5],"legal":[3,8],"legitim":[7,12],"less":12,"level":[0,3,4,12],"lf":12,"libemail":6,"libpq":12,"librari":12,"libxml2":6,"libxslt":6,"licens":6,"life":5,"lifetim":0,"lifetimetimeout":0,"like":[0,3,6,8,12],"limit":[0,2,12],"line":[3,8,12],"link":[3,4,7,8],"linux":[3,6,8],"list":[0,2,4,5,7,12],"listen":[2,12],"lite":[0,6,12],"live":[7,12],"ll":[3,8],"load":[0,4,12],"load_ip_db":0,"load_psl_overrid":0,"load_reverse_dns_map":0,"local":[0,2,4,6,10,12],"local_file_path":0,"local_psl_overrides_path":12,"local_reverse_dns_map_path":12,"localhost":[4,12],"locat":[7,12],"log":[0,2,5,12],"log_analyt":12,"log_fil":12,"logger":12,"login":[4,12],"logstash":4,"long":[0,3,12],"longer":[3,6,8],"look":[0,3,7],"lookup":[0,12],"loop":[0,12],"loopback":2,"loss":0,"lot":7,"low":12,"lower":12,"lua":10,"m":[0,6,12],"m365":12,"maco":6,"magnifi":7,"mail":[0,5,6,10,12],"mail_bcc":0,"mail_cc":0,"mail_from":0,"mail_to":0,"mailbox":[0,7,12],"mailbox_check_timeout":12,"mailbox_connect":0,"mailboxconnect":0,"maildir":12,"maildir_cr":12,"maildir_path":12,"mailer":10,"mailrelay":10,"mailsuit":12,"mailto":6,"main":4,"mainpid":12,"maintain":5,"make":[0,3,4,6,8,9,12],"malici":[7,12],"manag":[4,7,12],"mandatori":12,"manual":12,"map":0,"mariadb":12,"market":7,"massiv":12,"match":[0,4,11,12],"max_ag":10,"max_shards_per_nod":12,"maximum":4,"maxmind":[0,5,12],"may":[5,7,12],"mbox":[0,12],"mean":12,"mechan":3,"member":[3,8],"memori":12,"mention":7,"menu":[4,7],"merg":0,"messag":[0,2,3,4,6,7,8,10,12],"message_id":10,"meta":10,"method":12,"metric":7,"mfrom":[4,10],"microsoft":[0,2,5,10,12],"microsoftgraph":12,"microsoftonlin":12,"mid":12,"might":[0,3,7,8],"migrat":[0,7,12],"migrate_index":0,"mime":10,"min":0,"minim":12,"minimum":[4,12],"minimum_should_match":4,"minut":[0,2,12],"mirror":0,"miss":[6,12],"mitig":[3,8],"mix":0,"mm":[0,12],"mmdb":[0,6,12],"mobil":[3,8],"mode":[0,2,4,6,10],"modern":[2,3,8],"modifi":[0,3,8,12],"modul":[0,5,6,12],"mon":10,"monitor":[3,12],"month":[0,12],"monthly_index":[0,12],"mous":7,"move":[0,4,12],"ms":[0,10,12],"msal":12,"msg":[0,6],"msg_byte":0,"msg_date":0,"msg_footer":[3,8],"msg_header":[3,8],"msgconvert":[0,6],"msgraph":12,"msgraphconnect":0,"mta":7,"much":12,"multi":[2,5],"multipl":[0,12],"mung":[3,8],"must":[2,3,4,8,12],"must_not":4,"mutual":[4,12],"mv":4,"mx":10,"n":[0,10,12],"n_proc":12,"naiv":0,"name":[0,3,4,7,10,11],"nameserv":[0,12],"nano":[2,12],"nation":12,"navig":[3,8],"ncontent":10,"ndate":10,"ndjson":[4,7],"need":[0,2,3,4,6,7,8,12],"neither":12,"nelson":[3,8],"net":[2,12],"network":[0,2,4,12],"never":12,"new":[0,2,4,5,6,7,12],"newer":6,"newest":[2,12],"newkey":4,"news":3,"next":[0,12],"nfrom":10,"nmessag":10,"nmime":10,"node":4,"nologin":6,"non":[0,3,4,8,12],"nonameserv":0,"none":[0,3,4,10,12],"noproxyfor":2,"norepli":[3,10],"normal":[0,10,12],"normalize_timespan_threshold_hour":0,"normalized_timespan":10,"nosecureimap":2,"notabl":7,"note":12,"noth":12,"notic":12,"now":[4,6,7],"nsubject":10,"nto":10,"null":[4,6,10],"number":[0,12],"number_of_replica":[0,12],"number_of_shard":[0,12],"nwettbewerb":10,"nx":10,"o":[2,4,12],"oR":6,"oauth2":12,"oauth2_port":12,"object":[0,4,7],"observ":[7,12],"occur":[0,7],"occurr":11,"oct":10,"offic":2,"office365":2,"offici":12,"offlin":[0,6,12],"offset":[0,12],"often":[7,12],"old":7,"older":[4,6,10,12],"oldest":[2,12],"ole":[0,6],"omit":12,"onc":[4,12],"ondmarc":5,"one":[0,3,4,5,6,7,8,12],"onli":[0,2,3,4,6,7,8],"onlin":[0,2,12],"onto":0,"oor":0,"op":0,"open":[0,3],"opendn":12,"opensearch":[4,5,7,12],"opensearch_dashboard":7,"opensearcherror":0,"openssl":4,"oper":12,"opt":[2,6,12],"option":[0,2,3,4,5,8,11,12],"order":12,"org":[0,6,9,10,12],"org_email":10,"org_extra_contact_info":10,"org_nam":10,"organiz":[2,5,7,12],"organization_nam":10,"origin":[3,8,12],"original_envelope_id":10,"original_mail_from":10,"original_rcpt_to":10,"original_timespan_second":10,"os":0,"oserror":0,"otherwis":[0,12],"outdat":7,"outgo":[3,8,12],"outlook":[0,2,6,12],"output":[0,5,12],"output_directori":0,"outsid":12,"overal":0,"overrid":[0,6,12],"overwrit":[0,4],"owa":[5,12],"owned":6,"ownership":6,"p":[3,10],"p12":4,"pack":4,"packag":[0,4,6],"packet":0,"pad":0,"page":[3,4,6,7,8],"paginate_messag":12,"painless":4,"pair":[4,7],"pan":10,"parallel":12,"paramet":[0,12],"parent":7,"pars":[0,3,5,6,10,12],"parse_aggregate_report_fil":0,"parse_aggregate_report_xml":0,"parse_email":0,"parse_failure_report":0,"parse_forensic_report":0,"parse_report_email":0,"parse_report_fil":0,"parse_smtp_tls_report_json":0,"parsed_aggregate_reports_to_csv":0,"parsed_aggregate_reports_to_csv_row":0,"parsed_failure_reports_to_csv":0,"parsed_failure_reports_to_csv_row":0,"parsed_forensic_reports_to_csv":0,"parsed_forensic_reports_to_csv_row":0,"parsed_sampl":10,"parsed_smtp_tls_reports_to_csv":0,"parsed_smtp_tls_reports_to_csv_row":0,"parsedemail":0,"parsedmarc":[4,9,10,11],"parsedmarc_":12,"parsedmarc_config_fil":12,"parsedmarc_debug":12,"parsedmarc_elasticsearch_":12,"parsedmarc_elasticsearch_host":12,"parsedmarc_elasticsearch_ssl":12,"parsedmarc_gelf_":12,"parsedmarc_general_":12,"parsedmarc_general_debug":12,"parsedmarc_general_ipinfo_api_token":12,"parsedmarc_general_ipinfo_url":12,"parsedmarc_general_offlin":12,"parsedmarc_general_save_aggreg":12,"parsedmarc_general_save_failur":12,"parsedmarc_gmail_api_":12,"parsedmarc_gmail_api_credentials_file_fil":12,"parsedmarc_imap_":12,"parsedmarc_imap_host":12,"parsedmarc_imap_password":12,"parsedmarc_imap_password_fil":12,"parsedmarc_imap_us":12,"parsedmarc_kafka_":12,"parsedmarc_log_analytics_":12,"parsedmarc_mailbox_":12,"parsedmarc_mailbox_watch":12,"parsedmarc_maildir_":12,"parsedmarc_msgraph_":12,"parsedmarc_opensearch_":12,"parsedmarc_s3_":12,"parsedmarc_smtp_":12,"parsedmarc_splunk_hec_":12,"parsedmarc_splunk_hec_index":12,"parsedmarc_splunk_hec_token":12,"parsedmarc_splunk_hec_url":12,"parsedmarc_syslog_":12,"parsedmarc_webhook_":12,"parser":0,"parsererror":0,"parsingresult":0,"part":[3,4,7,8,12],"particular":[7,12],"pass":[0,3,7,10,12],"passag":7,"passsword":12,"password":[0,4,6,12],"paste":[4,11],"patch":6,"path":[0,4,6,12],"pathlik":0,"pattern":[0,5,7],"payload":[0,12],"pct":10,"peak":12,"pem":12,"per":[0,4,7,12],"percentag":7,"perform":[0,2,5],"period":12,"perl":[0,6],"permiss":[4,12],"persist":12,"peter":10,"pick":[6,12],"pickup":6,"pid":12,"pie":7,"pin":12,"pip":[6,12],"pkcs12":12,"place":[0,4,7,12],"plain":[0,12],"plaintext":[3,8],"platform":[3,6,8,12],"pleas":[1,5,12],"plug":12,"plus":[7,12],"point":[6,12],"polici":[3,7,8,10,12],"policy_domain":10,"policy_evalu":10,"policy_override_com":10,"policy_override_reason":10,"policy_publish":10,"policy_str":10,"policy_typ":10,"policyscopegroupid":12,"poll":[0,2,12],"popul":0,"port":[0,2,12],"portal":12,"posit":[0,12],"posix":0,"possibl":12,"post":[3,4,8,12],"poster":[3,8],"postgr":12,"postgresql":[5,12],"postorius":[3,8],"powershel":12,"ppa":6,"practic":12,"pre":[6,12],"prebuilt":12,"predict":12,"prefer":[2,6,12],"prefix":[0,3,8,12],"premad":[5,11],"prepend":0,"prerequisit":5,"present":12,"pressur":12,"pretti":12,"prettifi":12,"previous":[0,2,4,6,12],"pri":[2,12],"primari":0,"print":12,"printabl":10,"prioriti":12,"privaci":[3,6,7,8,12],"privat":12,"probe":0,"problem":12,"proc":12,"proceed":4,"process":[0,2,5,6,12],"produc":[0,10],"program":12,"programdata":6,"progress":[4,12],"project":[0,2,3,5,11,12],"prompt":4,"proofpoint":5,"properti":2,"protect":[2,3,5,8,12],"protocol":12,"provid":[0,4,7,12],"provis":12,"prox":6,"proxi":2,"proxyhost":2,"proxypassword":2,"proxyport":2,"proxyus":2,"psl":[0,12],"psl_overrid":0,"psl_overrides_path":0,"psl_overrides_url":[0,12],"psycopg":12,"public":[0,3,10,12],"public_suffix_list":0,"publicbaseurl":4,"publicsuffix":0,"publish":[3,12],"published_polici":0,"pull":12,"purpos":4,"put":[4,12],"py":0,"python":[0,4,6],"python3":6,"qo":4,"quarantin":[3,8],"queri":[0,4,12],"query_dn":0,"quick":0,"quickstart":12,"quit":12,"quot":[10,12],"quota":[0,12],"r":[2,10,12],"rais":[0,12],"ram":[4,12],"rate":[0,12],"rather":[3,7,8,12],"ratio":7,"raw":12,"re":[0,4,6,12],"reach":[0,12],"reachabl":12,"read":[0,12],"readabl":[0,12],"readwrit":12,"real":7,"realli":3,"reason":[0,2,4,5,12],"receiv":[0,7,10,12],"receiveddatetim":12,"receiving_ip":10,"receiving_mx_hostnam":10,"recent":0,"recipi":7,"recogn":7,"recommend":12,"recommended_dns_nameserv":0,"record":[0,5,6,10,12],"record_typ":0,"redact":12,"redi":12,"reduc":[6,12],"refer":[4,5,7],"referenc":12,"refresh":[6,12],"refresh_interv":12,"refus":4,"regard":12,"regardless":[0,10,12],"region":[0,12],"region_nam":12,"regist":[6,12],"registr":12,"regul":[4,6,9,12],"regular":[3,8],"reject":[0,3,8,12],"relat":[3,12],"relay":[3,8],"releas":[4,6],"reli":[6,7],"reliabl":12,"reload":[0,2,4],"remain":[0,7,12],"remot":2,"remov":[0,3,4,8,12],"render":7,"repeat":[0,3,8],"replac":[0,3,4,8,12],"repli":[2,3,8],"replic":12,"replica":[0,12],"reply_goes_to_list":[3,8],"reply_to":10,"replyto":[3,8],"repopul":0,"report":[0,4,11,12],"report_id":10,"report_metadata":10,"report_typ":0,"reported_domain":10,"reports_fold":[0,12],"repositori":[6,11],"req":4,"request":[0,2,4,12],"requir":[0,2,3,4,5,6,7,8,12],"require_encrypt":0,"res":4,"reserv":12,"resid":12,"resolv":[0,12],"resort":6,"resourc":[0,4,5,12],"respons":[0,12],"rest":[0,12],"restart":[2,3,4,6,8],"restartsec":[2,12],"restor":4,"restrict":12,"restrictaccess":12,"result":[0,5,7,10,12],"result_typ":10,"resum":12,"retain":[3,8,12],"retent":5,"retri":[0,12],"retriev":2,"retry_attempt":12,"retry_delay":12,"return":[0,4],"revers":[0,6,7,12],"reverse_dn":[0,10],"reverse_dns_base_domain":0,"reverse_dns_map":0,"reverse_dns_map_path":0,"reverse_dns_map_url":[0,12],"reversednsservic":0,"review":7,"rewrit":[0,3,8],"rfc":[0,3,5,8,10],"rfc2369":[3,8],"rfc822":2,"rhel":[4,5,6],"right":[4,7],"rm":4,"ro":0,"rocki":6,"rollup":6,"root":[2,12],"rough":12,"row":7,"rpm":4,"rpt":[5,7],"rsa":4,"rua":[5,6],"ruf":[5,6,7,12],"rule":[7,12],"run":[0,4,5,6],"runtimeerror":12,"rw":[2,12],"s":[0,2,3,4,6,7,8,10,12],"s3":[5,12],"safe":[0,4],"safer":12,"sampl":[0,5,7,12],"sample_headers_on":10,"save":[0,4,6,7,12],"save_aggreg":12,"save_aggregate_report_to_elasticsearch":0,"save_aggregate_report_to_opensearch":0,"save_aggregate_reports_to_splunk":0,"save_failur":12,"save_failure_report_to_elasticsearch":0,"save_failure_report_to_opensearch":0,"save_failure_reports_to_splunk":0,"save_forens":12,"save_forensic_report_to_elasticsearch":0,"save_forensic_report_to_opensearch":0,"save_forensic_reports_to_splunk":0,"save_output":0,"save_smtp_tl":12,"save_smtp_tls_report_to_elasticsearch":0,"save_smtp_tls_report_to_opensearch":0,"save_smtp_tls_reports_to_splunk":0,"sbin":6,"sc":4,"scalar":4,"schedul":[6,12],"schema":[5,10,12],"scheme":0,"scope":[4,7,10,12],"script":[4,6],"scrub_nondigest":[3,8],"sdk":12,"search":[0,3,4,8,12],"second":[0,2,12],"secret_access_key":12,"section":4,"secur":[0,4,12],"see":[2,3,4,6,7,12],"seek":0,"segment":7,"sel":4,"select":0,"selector":[4,7,10],"self":[4,5],"send":[0,2,3,4,5,7,8,11,12],"sender":[5,7,8],"sending_mta_ip":10,"sendmail":[0,12],"sensit":12,"sent":[0,3,8,12],"sentinel":5,"separ":[0,3,4,6,7,9,11,12],"sequenc":0,"serial":12,"server":[0,2,3,4,5,6,7,10,12],"server_ip":4,"serverless":[0,12],"servernameon":10,"servic":[0,3,4,5,6,7,8,10],"service_account":12,"service_account_us":12,"session":[0,7],"set":[0,2,3,4,6,7,8,9,12],"set_host":0,"setup":[4,6,9,12],"shard":[0,12],"share":[4,6,7,12],"sharealik":6,"sharepoint":10,"shell":6,"ship":[6,12],"short":12,"shot":12,"shouldn":[3,8],"show":[2,7,12],"shown":[6,7,12],"shutdown":[0,12],"side":[7,12],"sighup":[0,6,12],"sigkil":12,"sign":[0,3,4,6,12],"signal":12,"signatur":[3,7,8],"sigterm":[0,12],"sigv4":[0,12],"silent":[6,12],"similar":7,"simpl":5,"simplifi":0,"sinc":[0,6,7,12],"singl":[0,12],"sink":12,"sister":3,"six":12,"size":[2,4],"skel":6,"skip":[0,4,12],"skip_certificate_verif":[0,12],"slight":11,"slow":0,"small":[4,12],"smaller":12,"smtp":[0,3,5,12],"smtp_tls":[0,12],"smtp_tls_csv_filenam":[0,12],"smtp_tls_json_filenam":[0,12],"smtp_tls_report":0,"smtp_tls_url":12,"smtptlsfailuredetail":0,"smtptlsfailuredetailsopt":0,"smtptlsparsedreport":0,"smtptlspolici":0,"smtptlspolicysummari":0,"smtptlsreport":0,"socket":2,"solut":6,"somehow":12,"someon":4,"sometim":12,"sort":12,"sourc":[0,3,4,6,7,10],"source_as_domain":10,"source_as_nam":10,"source_asn":10,"source_base_domain":10,"source_countri":10,"source_ip_address":10,"source_nam":10,"source_reverse_dn":10,"source_typ":10,"sourceforg":2,"sovereign":12,"sp":[3,4,10],"spam":12,"special":12,"specif":[3,6,7,12],"specifi":[2,3],"spf":[5,7,10],"spf_align":10,"spf_domain":10,"spf_result":[4,10],"spf_results_combin":4,"spf_scope":10,"splunk":[5,12],"splunk_hec":12,"splunkerror":0,"splunkhec":12,"sponsor":5,"spoof":[3,8],"spurious":12,"sr":4,"ss":0,"ssl":[0,2,4,12],"ssl_cert_path":0,"stabl":4,"stack":[4,7,12],"standard":[0,5,6,10],"start":[0,2,4,7,9,11,12],"starttl":[7,12],"startup":[0,6],"static":12,"status":[2,12],"stay":7,"stdout":12,"step":[3,4,6,8,12],"still":[0,3,8,10,12],"stop":12,"storag":[0,12],"store":[2,4,9,12],"str":[0,12],"straight":12,"stream":12,"string":[0,4,7,12],"strip":[0,3,8,12],"strip_attachment_payload":[0,12],"strong":12,"structur":5,"sts":[7,10,12],"stsv1":10,"style":0,"subdomain":[0,3,12],"subject":[0,3,8,10,12],"subject_prefix":[3,8],"subsidiari":7,"substitut":6,"success":12,"successful_session_count":10,"sudo":[2,4,6,12],"suffici":12,"suffix":0,"suggest":7,"suit":12,"suitabl":0,"summari":[3,8,12],"supervis":12,"suppli":[0,7,12],"support":[2,4,5,7,10,11],"sure":4,"surfac":[7,12],"sw50zxjha3rpdmugv2v0dgjld2vyymvylcocymvyc2ljahq":10,"switch":7,"syslog":[2,5,12],"system":[2,3,4,6,8,12],"systemctl":[2,4,12],"systemd":5,"systemdr":6,"t":[0,5,7,8,10,12],"tab":[3,4,8],"tabl":[4,5,7,12],"tag":6,"take":[0,12],"target":[0,2,12],"task":[4,6],"tbi":10,"tcp":12,"tee":4,"tell":[3,7,8],"templat":[3,8],"temporari":7,"tenant":5,"tenant_id":12,"term":6,"test":[0,10,12],"text":[0,10],"thank":10,"therebi":[3,8],"therefor":0,"thousand":12,"three":7,"throughput":12,"tier":12,"time":[0,2,4,6,7,12],"timeout":[0,2,12],"timeoutstopsec":12,"timespan":0,"timespan_requires_norm":10,"timestamp":[0,12],"timestamp_to_datetim":0,"timestamp_to_human":0,"timezon":10,"tld":3,"tls":[0,5,12],"to_domain":10,"to_utc":0,"togeth":[7,12],"token":[0,4,12],"token_fil":12,"tool":12,"top":[3,7,12],"topic":12,"touch":[3,8],"tracker":1,"trade":12,"tradit":[3,8],"traffic":7,"trail":12,"transfer":10,"transient":0,"transpar":5,"transport":[4,12],"trash":12,"treat":0,"tri":[0,12],"troubleshoot":12,"true":[0,2,4,10,12],"trust":12,"truststor":4,"tsvb":7,"tuesday":6,"tune":5,"two":6,"txt":[0,12],"type":[4,5,7,10,12],"typeless":0,"typic":12,"typo":12,"u":[2,6,12],"ubuntu":[4,6],"udp":[0,12],"ui":[3,8],"unchang":[0,12],"uncondit":[3,8],"underlying":[0,12],"underneath":7,"underscor":12,"understand":[5,7],"unencrypt":12,"unexpir":12,"unfortun":[3,8],"unit":[0,2,12],"unix":0,"unknown":0,"unless":[6,12],"unreach":[0,12],"unread":12,"unrel":6,"unsubscrib":[3,8],"unsuit":12,"unus":0,"unzip":2,"updat":[0,4,6,12],"upersecur":12,"upgrad":[2,5,6,12],"upload":12,"upper":7,"uppercas":12,"uri":[6,12],"url":[0,2,4,12],"us":[10,12],"usabl":12,"usag":12,"use":[0,3,4,5,8,10],"use_ssl":0,"user":[0,2,3,4,6,7,8,10,12],"user_ag":10,"useradd":[2,6],"usernam":[0,12],"usernamepassword":12,"usesystemproxi":2,"usr":[4,6],"utc":[0,12],"utf":10,"util":5,"v":12,"valid":[0,7,10,12],"valimail":5,"valu":[0,3,4,7,8,12],"var":[3,8,12],"variabl":5,"variant":12,"various":6,"vendor":3,"venv":[6,12],"verbatim":12,"verbos":12,"veri":[4,7,12],"verif":[0,4,12],"verifi":0,"verification_mod":4,"version":[0,2,4,5,9,10,11,12],"vew":2,"via":[0,2],"view":[7,12],"vim":4,"virtualenv":6,"visual":[4,7,9],"volum":[7,12],"vulner":3,"w":[0,12],"w3c":10,"wait":[0,12],"wait_for_complet":4,"wall":0,"want":[2,12],"wantedbi":[2,12],"warn":12,"watch":[0,2,4,6,12],"watch_inbox":0,"watcher":[0,12],"way":[0,4,7],"web":[2,4],"webdav":2,"webhook":[5,12],"webmail":[3,7,8],"week":[0,6,12],"well":[2,12],"wettbewerb":10,"wget":4,"whalensolut":12,"wheel":12,"whenev":[0,2,12],"wherea":7,"wherev":12,"whether":[0,12],"whi":[3,7,12],"whole":0,"whose":[0,12],"wide":[6,10,12],"wiki":10,"will":[0,2,3,4,6,7,8,12],"win":12,"window":[6,12],"within":0,"without":[3,4,6,7,8],"won":5,"work":[2,3,4,5,6,7,8,12],"worker":12,"workstat":2,"worst":[3,12],"worth":12,"wrap":[3,8],"wrapper":12,"write":[0,12],"written":12,"www":[4,6,12],"x":[0,4,7,10],"x509":4,"xennn":10,"xml":[0,11],"xml_schema":10,"xms4g":4,"xmx4g":4,"xpack":4,"xxxx":4,"y":[4,6],"yahoo":7,"yaml":12,"year":12,"yes":[3,8],"yet":[3,12],"yml":4,"yyyi":[0,12],"z":12,"zero":12,"zip":[0,2,5,12],"\u00fcbersicht":10},"titles":["API reference","Contributing to parsedmarc","Accessing an inbox using OWA/EWS","Understanding DMARC","Elasticsearch and Kibana","parsedmarc documentation - Open source DMARC report analyzer and visualizer","Installation","Using the Kibana dashboards","What about mailing lists?","OpenSearch and Grafana","Sample outputs","Splunk","Using parsedmarc"],"titleterms":{"Do":[3,8],"What":[3,8],"_file":12,"access":2,"aggreg":[7,10],"align":3,"analyz":[5,6],"api":0,"backfil":4,"best":[3,8],"bug":1,"cli":12,"combin":4,"compat":5,"compos":12,"config":12,"configur":[2,12],"content":5,"contribut":1,"countri":6,"csv":10,"dashboard":7,"databas":6,"davmail":2,"depend":6,"dkim":[3,4],"dmarc":[3,5,7],"docker":12,"document":5,"domain":3,"elast":0,"elasticsearch":4,"env":12,"environ":12,"ew":2,"exampl":12,"exchang":6,"failur":[7,10],"featur":5,"field":4,"file":12,"geolite2":6,"grafana":9,"guid":3,"help":12,"inbox":2,"index":4,"indic":0,"instal":[4,6,9],"ip":6,"json":10,"kibana":[4,7],"list":[3,8],"listserv":[3,8],"lookalik":3,"mail":[3,8],"mailman":[3,8],"map":12,"maxmind":6,"microsoft":6,"mode":12,"multi":12,"multipl":6,"name":12,"onli":12,"open":5,"opensearch":[0,9],"option":6,"output":10,"owa":2,"parsedmarc":[0,1,2,5,6,12],"pattern":4,"perform":12,"practic":[3,8],"prerequisit":6,"proxi":6,"python":5,"record":[3,4,9],"refer":0,"reload":12,"report":[1,5,6,7,10],"resourc":3,"restart":12,"result":4,"retent":[4,9],"run":[2,12],"sampl":10,"secret":12,"section":12,"sender":3,"servic":[2,12],"smtp":[7,10],"sourc":5,"specifi":12,"spf":[3,4],"splunk":[0,11],"suffix":12,"support":[3,12],"systemd":[2,12],"t":3,"tabl":0,"tenant":12,"test":6,"tls":[7,10],"tune":12,"type":0,"understand":3,"upgrad":4,"use":[2,6,7,12],"util":0,"valid":3,"variabl":12,"via":12,"visual":5,"web":6,"without":12,"won":3,"workaround":[3,8]}})
\ No newline at end of file
+Search.setIndex({"alltitles":{"API reference":[[0,null]],"Accessing an inbox using OWA/EWS":[[2,null]],"Backfilling the combined DKIM/SPF result fields":[[4,"backfilling-the-combined-dkim-spf-result-fields"]],"Bug reports":[[1,"bug-reports"]],"CLI help":[[12,"cli-help"]],"CSV aggregate report":[[10,"csv-aggregate-report"]],"CSV failure report":[[10,"csv-failure-report"]],"Configuration file":[[12,"configuration-file"]],"Configuring parsedmarc for DavMail":[[2,"configuring-parsedmarc-for-davmail"]],"Contents":[[5,null]],"Contributing to parsedmarc":[[1,null]],"DMARC Alignment Guide":[[3,"dmarc-alignment-guide"]],"DMARC aggregate reports":[[7,"dmarc-aggregate-reports"]],"DMARC failure reports":[[7,"dmarc-failure-reports"]],"DMARC guides":[[3,"dmarc-guides"]],"Do":[[3,"do"],[8,"do"]],"Do not":[[3,"do-not"],[8,"do-not"]],"Docker Compose example":[[12,"docker-compose-example"]],"Docker secrets (_FILE suffix)":[[12,"docker-secrets-file-suffix"]],"Elasticsearch and Kibana":[[4,null]],"Environment variable configuration":[[12,"environment-variable-configuration"]],"Examples":[[12,"examples"]],"Features":[[5,"features"]],"IP-to-country database":[[6,"ip-to-country-database"]],"Indices and tables":[[0,"indices-and-tables"]],"Installation":[[4,"installation"],[6,null],[9,"installation"]],"Installing parsedmarc":[[6,"installing-parsedmarc"]],"JSON SMTP TLS report":[[10,"json-smtp-tls-report"]],"JSON aggregate report":[[10,"json-aggregate-report"]],"JSON failure report":[[10,"json-failure-report"]],"LISTSERV":[[3,"listserv"],[8,"listserv"]],"Lookalike domains":[[3,"lookalike-domains"]],"Mailing list best practices":[[3,"mailing-list-best-practices"],[8,"mailing-list-best-practices"]],"Mailman 2":[[3,"mailman-2"],[3,"id1"],[8,"mailman-2"],[8,"id1"]],"Mailman 3":[[3,"mailman-3"],[3,"id2"],[8,"mailman-3"],[8,"id2"]],"Multi-tenant support":[[12,"multi-tenant-support"]],"OpenSearch and Grafana":[[9,null]],"Optional dependencies":[[6,"optional-dependencies"]],"Performance tuning":[[12,"performance-tuning"]],"Prerequisites":[[6,"prerequisites"]],"Python Compatibility":[[5,"python-compatibility"]],"Records retention":[[4,"records-retention"],[9,"records-retention"]],"Reloading configuration without restarting":[[12,"reloading-configuration-without-restarting"]],"Resources":[[3,"resources"]],"Running DavMail as a systemd service":[[2,"running-davmail-as-a-systemd-service"]],"Running parsedmarc as a systemd service":[[12,"running-parsedmarc-as-a-systemd-service"]],"Running without a config file (env-only mode)":[[12,"running-without-a-config-file-env-only-mode"]],"SMTP TLS reporting":[[7,"smtp-tls-reporting"]],"SPF and DMARC record validation":[[3,"spf-and-dmarc-record-validation"]],"Sample aggregate report output":[[10,"sample-aggregate-report-output"]],"Sample failure report output":[[10,"sample-failure-report-output"]],"Sample outputs":[[10,null]],"Section name mapping":[[12,"section-name-mapping"]],"Specifying the config file via environment variable":[[12,"specifying-the-config-file-via-environment-variable"]],"Splunk":[[11,null]],"Testing multiple report analyzers":[[6,"testing-multiple-report-analyzers"]],"Understanding DMARC":[[3,null]],"Upgrading Kibana index patterns":[[4,"upgrading-kibana-index-patterns"]],"Using MaxMind GeoLite2 (optional)":[[6,"using-maxmind-geolite2-optional"]],"Using Microsoft Exchange":[[6,"using-microsoft-exchange"]],"Using a web proxy":[[6,"using-a-web-proxy"]],"Using parsedmarc":[[12,null]],"Using the Kibana dashboards":[[7,null]],"What about mailing lists?":[[3,"what-about-mailing-lists"],[8,null]],"What if a sender won\u2019t support DKIM/DMARC?":[[3,"what-if-a-sender-wont-support-dkim-dmarc"]],"Workarounds":[[3,"workarounds"],[8,"workarounds"]],"parsedmarc":[[0,"module-parsedmarc"]],"parsedmarc documentation - Open source DMARC report analyzer and visualizer":[[5,null]],"parsedmarc.elastic":[[0,"module-parsedmarc.elastic"]],"parsedmarc.opensearch":[[0,"module-parsedmarc.opensearch"]],"parsedmarc.splunk":[[0,"module-parsedmarc.splunk"]],"parsedmarc.types":[[0,"module-parsedmarc.types"]],"parsedmarc.utils":[[0,"module-parsedmarc.utils"]]},"docnames":["api","contributing","davmail","dmarc","elasticsearch","index","installation","kibana","mailing-lists","opensearch","output","splunk","usage"],"envversion":{"sphinx":66,"sphinx.domains.c":3,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":9,"sphinx.domains.index":1,"sphinx.domains.javascript":3,"sphinx.domains.math":2,"sphinx.domains.python":4,"sphinx.domains.rst":2,"sphinx.domains.std":2,"sphinx.ext.todo":2,"sphinx.ext.viewcode":1},"filenames":["api.md","contributing.md","davmail.md","dmarc.md","elasticsearch.md","index.md","installation.md","kibana.md","mailing-lists.md","opensearch.md","output.md","splunk.md","usage.md"],"indexentries":{"aggregatealignment (class in parsedmarc.types)":[[0,"parsedmarc.types.AggregateAlignment",false]],"aggregateauthresultdkim (class in parsedmarc.types)":[[0,"parsedmarc.types.AggregateAuthResultDKIM",false]],"aggregateauthresults (class in parsedmarc.types)":[[0,"parsedmarc.types.AggregateAuthResults",false]],"aggregateauthresultspf (class in parsedmarc.types)":[[0,"parsedmarc.types.AggregateAuthResultSPF",false]],"aggregateidentifiers (class in parsedmarc.types)":[[0,"parsedmarc.types.AggregateIdentifiers",false]],"aggregateparsedreport (class in parsedmarc.types)":[[0,"parsedmarc.types.AggregateParsedReport",false]],"aggregatepolicyevaluated (class in parsedmarc.types)":[[0,"parsedmarc.types.AggregatePolicyEvaluated",false]],"aggregatepolicyoverridereason (class in parsedmarc.types)":[[0,"parsedmarc.types.AggregatePolicyOverrideReason",false]],"aggregatepolicypublished (class in parsedmarc.types)":[[0,"parsedmarc.types.AggregatePolicyPublished",false]],"aggregaterecord (class in parsedmarc.types)":[[0,"parsedmarc.types.AggregateRecord",false]],"aggregatereport (class in parsedmarc.types)":[[0,"parsedmarc.types.AggregateReport",false]],"aggregatereportmetadata (class in parsedmarc.types)":[[0,"parsedmarc.types.AggregateReportMetadata",false]],"alreadysaved":[[0,"parsedmarc.elastic.AlreadySaved",false],[0,"parsedmarc.opensearch.AlreadySaved",false]],"append_json() (in module parsedmarc)":[[0,"parsedmarc.append_json",false]],"close() (parsedmarc.splunk.hecclient method)":[[0,"parsedmarc.splunk.HECClient.close",false]],"configure_ipinfo_api() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.configure_ipinfo_api",false]],"convert_outlook_msg() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.convert_outlook_msg",false]],"create_indexes() (in module parsedmarc.elastic)":[[0,"parsedmarc.elastic.create_indexes",false]],"create_indexes() (in module parsedmarc.opensearch)":[[0,"parsedmarc.opensearch.create_indexes",false]],"decode_base64() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.decode_base64",false]],"downloaderror":[[0,"parsedmarc.utils.DownloadError",false]],"elasticsearcherror":[[0,"parsedmarc.elastic.ElasticsearchError",false]],"email_results() (in module parsedmarc)":[[0,"parsedmarc.email_results",false]],"email_results_via_msgraph() (in module parsedmarc)":[[0,"parsedmarc.email_results_via_msgraph",false]],"emailaddress (class in parsedmarc.types)":[[0,"parsedmarc.types.EmailAddress",false]],"emailattachment (class in parsedmarc.types)":[[0,"parsedmarc.types.EmailAttachment",false]],"emailparsererror":[[0,"parsedmarc.utils.EmailParserError",false]],"extract_report() (in module parsedmarc)":[[0,"parsedmarc.extract_report",false]],"extract_report_from_file_path() (in module parsedmarc)":[[0,"parsedmarc.extract_report_from_file_path",false]],"failureparsedreport (class in parsedmarc.types)":[[0,"parsedmarc.types.FailureParsedReport",false]],"failurereport (class in parsedmarc.types)":[[0,"parsedmarc.types.FailureReport",false]],"forensicparsedreport (in module parsedmarc.types)":[[0,"parsedmarc.types.ForensicParsedReport",false]],"forensicreport (in module parsedmarc.types)":[[0,"parsedmarc.types.ForensicReport",false]],"get_base_domain() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.get_base_domain",false]],"get_dmarc_reports_from_mailbox() (in module parsedmarc)":[[0,"parsedmarc.get_dmarc_reports_from_mailbox",false]],"get_dmarc_reports_from_mbox() (in module parsedmarc)":[[0,"parsedmarc.get_dmarc_reports_from_mbox",false]],"get_filename_safe_string() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.get_filename_safe_string",false]],"get_ip_address_country() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.get_ip_address_country",false]],"get_ip_address_db_record() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.get_ip_address_db_record",false]],"get_ip_address_info() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.get_ip_address_info",false]],"get_report_zip() (in module parsedmarc)":[[0,"parsedmarc.get_report_zip",false]],"get_reverse_dns() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.get_reverse_dns",false]],"get_service_from_reverse_dns_base_domain() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.get_service_from_reverse_dns_base_domain",false]],"hecclient (class in parsedmarc.splunk)":[[0,"parsedmarc.splunk.HECClient",false]],"human_timestamp_to_datetime() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.human_timestamp_to_datetime",false]],"human_timestamp_to_unix_timestamp() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.human_timestamp_to_unix_timestamp",false]],"invalidaggregatereport":[[0,"parsedmarc.InvalidAggregateReport",false]],"invaliddmarcreport":[[0,"parsedmarc.InvalidDMARCReport",false]],"invalidfailurereport":[[0,"parsedmarc.InvalidFailureReport",false]],"invalidforensicreport (in module parsedmarc)":[[0,"parsedmarc.InvalidForensicReport",false]],"invalidipinfoapikey":[[0,"parsedmarc.utils.InvalidIPinfoAPIKey",false]],"invalidsmtptlsreport":[[0,"parsedmarc.InvalidSMTPTLSReport",false]],"ipaddressinfo (class in parsedmarc.utils)":[[0,"parsedmarc.utils.IPAddressInfo",false]],"ipsourceinfo (class in parsedmarc.types)":[[0,"parsedmarc.types.IPSourceInfo",false]],"is_mbox() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.is_mbox",false]],"is_outlook_msg() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.is_outlook_msg",false]],"load_ip_db() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.load_ip_db",false]],"load_psl_overrides() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.load_psl_overrides",false]],"load_reverse_dns_map() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.load_reverse_dns_map",false]],"migrate_indexes() (in module parsedmarc.elastic)":[[0,"parsedmarc.elastic.migrate_indexes",false]],"migrate_indexes() (in module parsedmarc.opensearch)":[[0,"parsedmarc.opensearch.migrate_indexes",false]],"module":[[0,"module-parsedmarc",false],[0,"module-parsedmarc.elastic",false],[0,"module-parsedmarc.opensearch",false],[0,"module-parsedmarc.splunk",false],[0,"module-parsedmarc.types",false],[0,"module-parsedmarc.utils",false]],"opensearcherror":[[0,"parsedmarc.opensearch.OpenSearchError",false]],"parse_aggregate_report_file() (in module parsedmarc)":[[0,"parsedmarc.parse_aggregate_report_file",false]],"parse_aggregate_report_xml() (in module parsedmarc)":[[0,"parsedmarc.parse_aggregate_report_xml",false]],"parse_email() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.parse_email",false]],"parse_failure_report() (in module parsedmarc)":[[0,"parsedmarc.parse_failure_report",false]],"parse_forensic_report() (in module parsedmarc)":[[0,"parsedmarc.parse_forensic_report",false]],"parse_report_email() (in module parsedmarc)":[[0,"parsedmarc.parse_report_email",false]],"parse_report_file() (in module parsedmarc)":[[0,"parsedmarc.parse_report_file",false]],"parse_smtp_tls_report_json() (in module parsedmarc)":[[0,"parsedmarc.parse_smtp_tls_report_json",false]],"parsed_aggregate_reports_to_csv() (in module parsedmarc)":[[0,"parsedmarc.parsed_aggregate_reports_to_csv",false]],"parsed_aggregate_reports_to_csv_rows() (in module parsedmarc)":[[0,"parsedmarc.parsed_aggregate_reports_to_csv_rows",false]],"parsed_failure_reports_to_csv() (in module parsedmarc)":[[0,"parsedmarc.parsed_failure_reports_to_csv",false]],"parsed_failure_reports_to_csv_rows() (in module parsedmarc)":[[0,"parsedmarc.parsed_failure_reports_to_csv_rows",false]],"parsed_forensic_reports_to_csv() (in module parsedmarc)":[[0,"parsedmarc.parsed_forensic_reports_to_csv",false]],"parsed_forensic_reports_to_csv_rows() (in module parsedmarc)":[[0,"parsedmarc.parsed_forensic_reports_to_csv_rows",false]],"parsed_smtp_tls_reports_to_csv() (in module parsedmarc)":[[0,"parsedmarc.parsed_smtp_tls_reports_to_csv",false]],"parsed_smtp_tls_reports_to_csv_rows() (in module parsedmarc)":[[0,"parsedmarc.parsed_smtp_tls_reports_to_csv_rows",false]],"parsedemail (class in parsedmarc.types)":[[0,"parsedmarc.types.ParsedEmail",false]],"parsedmarc":[[0,"module-parsedmarc",false]],"parsedmarc.elastic":[[0,"module-parsedmarc.elastic",false]],"parsedmarc.opensearch":[[0,"module-parsedmarc.opensearch",false]],"parsedmarc.splunk":[[0,"module-parsedmarc.splunk",false]],"parsedmarc.types":[[0,"module-parsedmarc.types",false]],"parsedmarc.utils":[[0,"module-parsedmarc.utils",false]],"parsererror":[[0,"parsedmarc.ParserError",false]],"parsingresults (class in parsedmarc.types)":[[0,"parsedmarc.types.ParsingResults",false]],"query_dns() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.query_dns",false]],"reversednsservice (class in parsedmarc.utils)":[[0,"parsedmarc.utils.ReverseDNSService",false]],"save_aggregate_report_to_elasticsearch() (in module parsedmarc.elastic)":[[0,"parsedmarc.elastic.save_aggregate_report_to_elasticsearch",false]],"save_aggregate_report_to_opensearch() (in module parsedmarc.opensearch)":[[0,"parsedmarc.opensearch.save_aggregate_report_to_opensearch",false]],"save_aggregate_reports_to_splunk() (parsedmarc.splunk.hecclient method)":[[0,"parsedmarc.splunk.HECClient.save_aggregate_reports_to_splunk",false]],"save_failure_report_to_elasticsearch() (in module parsedmarc.elastic)":[[0,"parsedmarc.elastic.save_failure_report_to_elasticsearch",false]],"save_failure_report_to_opensearch() (in module parsedmarc.opensearch)":[[0,"parsedmarc.opensearch.save_failure_report_to_opensearch",false]],"save_failure_reports_to_splunk() (parsedmarc.splunk.hecclient method)":[[0,"parsedmarc.splunk.HECClient.save_failure_reports_to_splunk",false]],"save_forensic_report_to_elasticsearch() (in module parsedmarc.elastic)":[[0,"parsedmarc.elastic.save_forensic_report_to_elasticsearch",false]],"save_forensic_report_to_opensearch() (in module parsedmarc.opensearch)":[[0,"parsedmarc.opensearch.save_forensic_report_to_opensearch",false]],"save_forensic_reports_to_splunk() (parsedmarc.splunk.hecclient method)":[[0,"parsedmarc.splunk.HECClient.save_forensic_reports_to_splunk",false]],"save_output() (in module parsedmarc)":[[0,"parsedmarc.save_output",false]],"save_smtp_tls_report_to_elasticsearch() (in module parsedmarc.elastic)":[[0,"parsedmarc.elastic.save_smtp_tls_report_to_elasticsearch",false]],"save_smtp_tls_report_to_opensearch() (in module parsedmarc.opensearch)":[[0,"parsedmarc.opensearch.save_smtp_tls_report_to_opensearch",false]],"save_smtp_tls_reports_to_splunk() (parsedmarc.splunk.hecclient method)":[[0,"parsedmarc.splunk.HECClient.save_smtp_tls_reports_to_splunk",false]],"set_hosts() (in module parsedmarc.elastic)":[[0,"parsedmarc.elastic.set_hosts",false]],"set_hosts() (in module parsedmarc.opensearch)":[[0,"parsedmarc.opensearch.set_hosts",false]],"smtptlsfailuredetails (class in parsedmarc.types)":[[0,"parsedmarc.types.SMTPTLSFailureDetails",false]],"smtptlsfailuredetailsoptional (class in parsedmarc.types)":[[0,"parsedmarc.types.SMTPTLSFailureDetailsOptional",false]],"smtptlsparsedreport (class in parsedmarc.types)":[[0,"parsedmarc.types.SMTPTLSParsedReport",false]],"smtptlspolicy (class in parsedmarc.types)":[[0,"parsedmarc.types.SMTPTLSPolicy",false]],"smtptlspolicysummary (class in parsedmarc.types)":[[0,"parsedmarc.types.SMTPTLSPolicySummary",false]],"smtptlsreport (class in parsedmarc.types)":[[0,"parsedmarc.types.SMTPTLSReport",false]],"splunkerror":[[0,"parsedmarc.splunk.SplunkError",false]],"timestamp_to_datetime() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.timestamp_to_datetime",false]],"timestamp_to_human() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.timestamp_to_human",false]],"watch_inbox() (in module parsedmarc)":[[0,"parsedmarc.watch_inbox",false]]},"objects":{"":[[0,0,0,"-","parsedmarc"]],"parsedmarc":[[0,1,1,"","InvalidAggregateReport"],[0,1,1,"","InvalidDMARCReport"],[0,1,1,"","InvalidFailureReport"],[0,2,1,"","InvalidForensicReport"],[0,1,1,"","InvalidSMTPTLSReport"],[0,1,1,"","ParserError"],[0,3,1,"","append_json"],[0,0,0,"-","elastic"],[0,3,1,"","email_results"],[0,3,1,"","email_results_via_msgraph"],[0,3,1,"","extract_report"],[0,3,1,"","extract_report_from_file_path"],[0,3,1,"","get_dmarc_reports_from_mailbox"],[0,3,1,"","get_dmarc_reports_from_mbox"],[0,3,1,"","get_report_zip"],[0,0,0,"-","opensearch"],[0,3,1,"","parse_aggregate_report_file"],[0,3,1,"","parse_aggregate_report_xml"],[0,3,1,"","parse_failure_report"],[0,3,1,"","parse_forensic_report"],[0,3,1,"","parse_report_email"],[0,3,1,"","parse_report_file"],[0,3,1,"","parse_smtp_tls_report_json"],[0,3,1,"","parsed_aggregate_reports_to_csv"],[0,3,1,"","parsed_aggregate_reports_to_csv_rows"],[0,3,1,"","parsed_failure_reports_to_csv"],[0,3,1,"","parsed_failure_reports_to_csv_rows"],[0,3,1,"","parsed_forensic_reports_to_csv"],[0,3,1,"","parsed_forensic_reports_to_csv_rows"],[0,3,1,"","parsed_smtp_tls_reports_to_csv"],[0,3,1,"","parsed_smtp_tls_reports_to_csv_rows"],[0,3,1,"","save_output"],[0,0,0,"-","splunk"],[0,0,0,"-","types"],[0,0,0,"-","utils"],[0,3,1,"","watch_inbox"]],"parsedmarc.elastic":[[0,1,1,"","AlreadySaved"],[0,1,1,"","ElasticsearchError"],[0,3,1,"","create_indexes"],[0,3,1,"","migrate_indexes"],[0,3,1,"","save_aggregate_report_to_elasticsearch"],[0,3,1,"","save_failure_report_to_elasticsearch"],[0,3,1,"","save_forensic_report_to_elasticsearch"],[0,3,1,"","save_smtp_tls_report_to_elasticsearch"],[0,3,1,"","set_hosts"]],"parsedmarc.opensearch":[[0,1,1,"","AlreadySaved"],[0,1,1,"","OpenSearchError"],[0,3,1,"","create_indexes"],[0,3,1,"","migrate_indexes"],[0,3,1,"","save_aggregate_report_to_opensearch"],[0,3,1,"","save_failure_report_to_opensearch"],[0,3,1,"","save_forensic_report_to_opensearch"],[0,3,1,"","save_smtp_tls_report_to_opensearch"],[0,3,1,"","set_hosts"]],"parsedmarc.splunk":[[0,4,1,"","HECClient"],[0,1,1,"","SplunkError"]],"parsedmarc.splunk.HECClient":[[0,5,1,"","close"],[0,5,1,"","save_aggregate_reports_to_splunk"],[0,5,1,"","save_failure_reports_to_splunk"],[0,5,1,"","save_forensic_reports_to_splunk"],[0,5,1,"","save_smtp_tls_reports_to_splunk"]],"parsedmarc.types":[[0,4,1,"","AggregateAlignment"],[0,4,1,"","AggregateAuthResultDKIM"],[0,4,1,"","AggregateAuthResultSPF"],[0,4,1,"","AggregateAuthResults"],[0,4,1,"","AggregateIdentifiers"],[0,4,1,"","AggregateParsedReport"],[0,4,1,"","AggregatePolicyEvaluated"],[0,4,1,"","AggregatePolicyOverrideReason"],[0,4,1,"","AggregatePolicyPublished"],[0,4,1,"","AggregateRecord"],[0,4,1,"","AggregateReport"],[0,4,1,"","AggregateReportMetadata"],[0,4,1,"","EmailAddress"],[0,4,1,"","EmailAttachment"],[0,4,1,"","FailureParsedReport"],[0,4,1,"","FailureReport"],[0,2,1,"","ForensicParsedReport"],[0,2,1,"","ForensicReport"],[0,4,1,"","IPSourceInfo"],[0,4,1,"","ParsedEmail"],[0,4,1,"","ParsingResults"],[0,4,1,"","SMTPTLSFailureDetails"],[0,4,1,"","SMTPTLSFailureDetailsOptional"],[0,4,1,"","SMTPTLSParsedReport"],[0,4,1,"","SMTPTLSPolicy"],[0,4,1,"","SMTPTLSPolicySummary"],[0,4,1,"","SMTPTLSReport"]],"parsedmarc.utils":[[0,1,1,"","DownloadError"],[0,1,1,"","EmailParserError"],[0,4,1,"","IPAddressInfo"],[0,1,1,"","InvalidIPinfoAPIKey"],[0,4,1,"","ReverseDNSService"],[0,3,1,"","configure_ipinfo_api"],[0,3,1,"","convert_outlook_msg"],[0,3,1,"","decode_base64"],[0,3,1,"","get_base_domain"],[0,3,1,"","get_filename_safe_string"],[0,3,1,"","get_ip_address_country"],[0,3,1,"","get_ip_address_db_record"],[0,3,1,"","get_ip_address_info"],[0,3,1,"","get_reverse_dns"],[0,3,1,"","get_service_from_reverse_dns_base_domain"],[0,3,1,"","human_timestamp_to_datetime"],[0,3,1,"","human_timestamp_to_unix_timestamp"],[0,3,1,"","is_mbox"],[0,3,1,"","is_outlook_msg"],[0,3,1,"","load_ip_db"],[0,3,1,"","load_psl_overrides"],[0,3,1,"","load_reverse_dns_map"],[0,3,1,"","parse_email"],[0,3,1,"","query_dns"],[0,3,1,"","timestamp_to_datetime"],[0,3,1,"","timestamp_to_human"]]},"objnames":{"0":["py","module","Python module"],"1":["py","exception","Python exception"],"2":["py","attribute","Python attribute"],"3":["py","function","Python function"],"4":["py","class","Python class"],"5":["py","method","Python method"]},"objtypes":{"0":"py:module","1":"py:exception","2":"py:attribute","3":"py:function","4":"py:class","5":"py:method"},"terms":{"00z":10,"00z_exampl":10,"09t00":10,"09t23":10,"1d":12,"1g":4,"1w":12,"2017a":[3,8],"21vianet":12,"2d":12,"2k":12,"30s":12,"3d":10,"3h":12,"59z":10,"5m":[2,12],"7d":12,"A":[0,3,7,12],"AT":10,"After":[2,4,12],"All":[3,8,12],"An":[0,12],"And":0,"As":[4,7],"By":[7,12],"Do":[0,12],"Each":[4,7,12],"For":[0,4,12],"From":[3,8,12],"Further":7,"Have":12,"Here":10,"IF":12,"If":[0,3,4,6,7,8,12],"In":[0,2,3,7,8,12],"It":[2,4,7,10,12],"More":6,"Most":[7,12],"NOT":12,"No":[3,6,8],"Not":12,"On":[3,4,6,7,8,12],"Or":[4,6,12],"Some":[2,3,7,8],"That":7,"The":[0,3,4,6,7,11,12],"Then":[2,3,4,6,8,12],"There":7,"These":[7,12],"This":[5,6,7,10,12],"To":[2,4,6,7,9,10,12],"Under":[4,7],"What":5,"When":[0,3,5,8,12],"Where":[3,8],"While":[7,12],"With":[7,12],"YOUR":4,"You":[2,7,12],"_":12,"_attempt":0,"_cluster":12,"_input":0,"_ipdatabaserecord":0,"_serverless_rejected_set":0,"_sourc":4,"_task":4,"_update_by_queri":[0,4],"aadst":12,"abl":6,"abort":[4,12],"abov":[2,4,6,7,12],"absent":4,"accept":[0,3,4,7,8,12],"access":[0,4,5,12],"access_key_id":12,"access_token":0,"accessright":12,"accident":[3,8],"account":[6,7,12],"acm":10,"acquir":12,"acquisit":12,"across":7,"action":[3,8,12],"activ":[4,5,12],"active_primary_shard":12,"active_shard":12,"actual":[3,10],"ad":12,"add":[0,2,3,4,6,7,8,12],"addit":[3,8,12],"address":[0,2,3,4,6,7,8,10,12],"addresse":7,"adjust":4,"adkim":10,"admin":[3,8,12],"administr":[3,8,12],"advanc":7,"agari":5,"agent":4,"agg":7,"aggreg":[0,4,5,11,12],"aggregate_csv_filenam":[0,12],"aggregate_index":0,"aggregate_json_filenam":[0,12],"aggregate_report":0,"aggregate_top":12,"aggregate_url":12,"aggregatealign":0,"aggregateauthresult":0,"aggregateauthresultdkim":0,"aggregateauthresultspf":0,"aggregateidentifi":0,"aggregateparsedreport":0,"aggregatepolicyevalu":0,"aggregatepolicyoverridereason":0,"aggregatepolicypublish":0,"aggregaterecord":0,"aggregatereport":0,"aggregatereportmetadata":0,"aggress":12,"alia":[0,12],"alias":12,"align":[4,5,7,10],"aliv":0,"allow":[2,3,8,12],"allow_unencrypted_storag":12,"allowremot":2,"allowstringindic":7,"alon":12,"alreadi":[0,4,12],"alreadysav":0,"also":[0,2,3,4,6,7,8,12],"alter":[3,8],"altern":[5,12],"although":11,"alway":[0,2,4,12],"always_use_local_fil":[0,12],"amazon":5,"amd64":12,"amount":12,"analyt":[5,12],"analyz":12,"ani":[0,3,4,6,7,8,12],"anonym":10,"anoth":[6,12],"answer":[0,12],"apach":5,"api":[2,4,5,12],"api_key":[0,12],"app":12,"appear":[4,7,12],"append":[0,12],"append_json":0,"appendix":10,"appid":12,"appli":12,"applic":[4,12],"applicationaccesspolici":12,"approach":12,"approxim":2,"apt":[2,4,6],"archiv":[0,12],"archive_fold":[0,12],"argument":12,"arm64":12,"around":12,"array":[0,4],"arraylist":4,"arriv":12,"arrival_d":10,"arrival_date_utc":[0,10],"artifact":4,"as_domain":[0,10],"as_nam":[0,10],"ask":3,"asmx":2,"asn":[0,6,10,12],"aspf":10,"assert":12,"assign":4,"associ":0,"assum":12,"assume_utc":0,"astimezon":0,"att":10,"attach":[0,3,8,10,12],"attachment_filenam":0,"attempt":[0,12],"attribut":[6,7],"audit":4,"auth":[0,2,4,7,10,12],"auth_failur":10,"auth_method":12,"auth_mod":12,"auth_result":10,"auth_typ":[0,12],"authent":[0,2,3,4,7,12],"authentication_mechan":10,"authentication_result":10,"authentication_typ":12,"auto":2,"automat":[4,6,12],"avail":[0,6,12],"avoid":[7,12],"aw":[0,12],"aws_region":[0,12],"aws_servic":[0,12],"awssigv4":[0,12],"az":12,"azur":[5,12],"b":10,"b2c":7,"back":[0,12],"backend":[0,12],"backfil":[0,5,12],"background":[0,4],"backlog":12,"backward":12,"bare":12,"base":[0,2,3,4,6,7,8,10,12],"base64":0,"base_domain":[0,10],"basic":[0,2,12],"batch":12,"batch_siz":[0,12],"bcc":[0,10],"bd6e1bb5":10,"becaus":[2,3,4,7,8,12],"becom":12,"befor":[0,12],"begin_d":10,"behavior":0,"behind":6,"benefit":5,"best":[7,12],"beyond":0,"bin":[2,4,6,12],"binari":[0,12],"binaryio":0,"bind":2,"bindaddress":2,"blank":[3,8],"block":[0,2,4,12],"bodi":[0,3,8,10,12],"bool":[0,4,12],"boundari":[0,12],"box":12,"brand":[5,7],"break":[3,4,8],"broken":0,"browser":4,"bucket":12,"budget":0,"bug":[5,12],"build":6,"built":0,"bundl":[0,6,7,12],"busi":7,"button":[3,8],"byte":0,"c":[10,12],"ca":[4,12],"cach":[0,12],"cafile_path":12,"call":[0,5,12],"callabl":0,"callback":0,"came":[3,8],"can":[0,2,3,4,5,6,7,8,12],"cap":[0,12],"carri":[0,4,6,7,12],"case":[2,3,8,12],"catch":[0,12],"caught":0,"caus":[3,4,7,8,12],"cc":[0,10],"center":7,"cento":[4,6],"cert":[4,12],"cert_path":12,"certain":[0,12],"certfile_path":12,"certif":[0,4,7,12],"certificate_password":12,"certificate_path":12,"cest":10,"chain":0,"chang":[4,7,11,12],"charact":[2,12],"charset":10,"chart":7,"cheap":[0,4],"check":[0,2,3,4,7,12],"check_timeout":[0,12],"checkbox":4,"checkdmarc":3,"china":12,"chinacloudapi":12,"chines":7,"chmod":[2,4,12],"choos":[3,8],"chown":[2,12],"ci":7,"cisco":12,"class":[0,4],"clean":[0,12],"clear":0,"cli":[0,5],"click":[4,7],"client":[2,3,4,8,12],"client_assert":12,"client_id":12,"client_secret":12,"clientassert":12,"clientsecret":12,"clientsotimeout":2,"clock":0,"close":[0,12],"cloud":[0,12],"cloudflar":[0,12],"cluster":[0,4,12],"cn":12,"co":4,"code":[0,4,12],"collect":[7,12],"collector":[11,12],"column":7,"com":[1,2,3,8,9,10,12],"combin":[0,5,7,12],"come":[0,7],"comfort":12,"comma":[6,12],"command":[0,2,3,4,6,8,12],"comment":12,"commerci":[4,5],"common":[3,4,6,8],"communiti":[3,8],"compat":[0,7,12],"complet":[3,4,12],"compli":[3,4,6,8,9],"complianc":7,"compliant":[3,8],"compon":[6,7],"compress":5,"comput":7,"condit":7,"conf":6,"config":[0,2,6],"config_fil":12,"config_reload":0,"configur":[0,3,4,5,6,7,8,9],"configure_ipinfo_api":0,"confirm":12,"conflict":4,"conform":4,"connect":[0,2,4,12],"connection_str":12,"consent":12,"consid":[5,7],"consist":[0,5,10],"consol":[4,12],"constant":0,"consult":6,"consum":[7,12],"contact":[7,12],"contain":[0,7,11,12],"content":[0,3,4,8,10,11,12],"continu":4,"contrib":6,"contribut":5,"control":[0,4,12],"convent":12,"convert":[0,3,8],"convert_outlook_msg":0,"copi":[0,6,11,12],"core":[3,8],"correct":[0,6,7,12],"correspond":[4,12],"corrupt":0,"count":[0,2,4,7,10],"countri":[0,7,10,12],"country_cod":0,"cpan":6,"cr":12,"crash":[2,4,12],"creat":[0,2,3,4,6,8,12],"create_fold":0,"create_index":0,"creation":12,"creativ":6,"credenti":[4,6,12],"credentials_fil":12,"cron":6,"cross":0,"crt":4,"csr":4,"csv":[0,5,12],"csvs":12,"ctrl":12,"ctx":4,"cumul":6,"curl":4,"current":[0,2,4,12],"custom":[4,7,12],"d":[0,4,12],"daemon":[2,4,12],"daili":[0,12],"dashboard":[4,5,9,11],"dat":0,"data":[0,4,5,6,7,9,11,12],"databas":[0,12],"date":[0,3,8,10],"date_utc":10,"datetim":0,"davmail":5,"day":[0,4,9,12],"db_path":0,"dbip":[0,6,12],"dbname":12,"dce":12,"dcr":12,"dcr_aggregate_stream":12,"dcr_failure_stream":12,"dcr_immutable_id":12,"dcr_smtp_tls_stream":12,"dd":[0,12],"de":10,"dearmor":4,"deb":4,"debian":[4,5,6],"debug":12,"decemb":6,"decod":0,"decode_base64":0,"dedic":6,"deeper":4,"def":4,"default":[0,2,4,5,6,7,12],"defect":4,"defens":[4,5],"delay":[0,2,10,12],"deleg":12,"delegated_us":12,"delet":[0,2,4,12],"deliber":12,"delivery_result":10,"demystifi":3,"deni":12,"depend":[0,4,5,12],"deploy":[3,8,12],"deprec":[7,12],"depth":4,"describ":12,"descript":[2,6,12],"design":12,"destin":[0,12],"det":4,"detail":[0,4,6,7,12],"dev":[6,12],"devel":6,"develop":5,"devicecod":12,"dict":0,"dictionari":0,"differ":[7,12],"difficult":12,"dig":0,"digest":[3,8],"dir":6,"direct":[7,12],"directori":[0,6,12],"dis":10,"disabl":[0,2,6,12],"disclaim":[3,8],"disk":[0,12],"display":[3,7,11],"display_nam":10,"disposit":[7,10],"distinguish":12,"distribut":6,"distro":6,"dk":4,"dkim":[0,5,7,8,10],"dkim_align":10,"dkim_domain":10,"dkim_result":[4,10],"dkim_results_combin":[0,4],"dkim_selector":10,"dkm":3,"dmarc":[0,4,6,8,9,10,11,12],"dmarc_aggreg":[4,7],"dmarc_align":10,"dmarc_failur":4,"dmarc_moderation_act":[3,8],"dmarc_none_moderation_act":[3,8],"dmarc_quarantine_moderation_act":[3,8],"dmarcian":5,"dmarcresport":12,"dnf":6,"dns":[0,3,6,7,12],"dns_retri":[0,12],"dns_test_address":12,"dns_timeout":[0,12],"dnspython":0,"doc":[0,9,12],"doctyp":10,"document":[0,2,4,7,12],"dod":12,"doe":[3,8,12],"doesn":0,"dom":4,"domain":[0,4,7,8,10,12],"domainawar":[1,3,12],"don":3,"doubl":12,"download":[0,2,4,6,12],"downloaderror":0,"dr":4,"draft":[5,10,12],"dropdown":7,"dsn":12,"dtd":10,"dummi":12,"dure":[2,12],"e":[0,2,3,4,6,8,12],"e7":10,"earlier":[0,7],"easi":[4,9],"easier":[11,12],"echo":4,"edit":[2,6,7,12],"editor":11,"effect":12,"effici":4,"either":[4,5,12],"elast":[4,5,7,12],"elasticsearch":[0,5,12],"elasticsearcherror":0,"elig":12,"elk":12,"els":4,"email":[0,3,5,6,7,8,10,11,12],"email_result":0,"email_results_via_msgraph":0,"emailaddress":0,"emailattach":0,"emailparsererror":0,"empti":[0,3,4,8,12],"en":[3,4,8,10],"enabl":[2,4,7,12],"enableew":2,"enablekeepal":2,"enableproxi":2,"encod":[0,10,12],"encount":0,"encrypt":[4,12],"encryptedsavedobject":4,"encryptionkey":4,"end":[0,3,4,5,12],"end_dat":10,"endpoint":[5,12],"endpoint_url":12,"enforc":[3,8],"enough":12,"enrich":6,"enrol":4,"ensur":[3,4,6,8],"enterpris":12,"entir":[0,3,7,8,12],"entra":12,"entri":0,"envelop":3,"envelope_from":10,"envelope_to":10,"environ":[5,6],"eof":0,"eol":5,"epel":6,"equival":4,"era":0,"error":[0,4,7,10,12],"erroritemnotfound":12,"es":[0,12],"escap":12,"especi":[7,12],"etc":[2,3,4,6,8,12],"even":[2,3,8,12],"event":[2,11,12],"everi":[0,2,4,6,7,12],"ew":5,"ex":12,"exact":[3,8],"exampl":[3,4,6,8,10],"exceed":7,"except":[0,12],"exchang":[2,10,12],"exclud":2,"execreload":12,"execstart":[2,12],"exhaust":12,"exist":[0,3,4,8,12],"exit":[0,12],"expect":12,"expiri":7,"expiringdict":0,"explain":[3,8],"explicit":[0,3,6,8,12],"export":[4,7,12],"extra":12,"extract":[0,2],"extract_report":0,"extract_report_from_file_path":0,"eye":[2,12],"f":[0,4],"factor":2,"fail":[0,3,7,8,10,12],"fail_on_output_error":12,"failed_sess":7,"failed_session_count":10,"failov":0,"failur":[0,5,11,12],"failure_csv_filenam":[0,12],"failure_detail":[4,10],"failure_details_combin":[0,4],"failure_index":0,"failure_json_filenam":[0,12],"failure_report":0,"failure_top":12,"failure_url":12,"failureparsedreport":0,"failurereport":0,"fall":[0,12],"fallback":[0,12],"fals":[0,2,4,10,12],"fantast":[3,8],"fast":0,"faster":12,"fatal":[0,12],"fds":4,"featur":[4,12],"feedback":0,"feedback_report":0,"feedback_typ":10,"fetch":[0,7,12],"field":[0,5,7],"file":[0,2,5,6,7,11],"file_path":[0,12],"filenam":[0,12],"filename_safe_subject":10,"filepath":12,"fill":[4,6],"filter":[0,3,7,8,11,12],"final":5,"financ":12,"find":[3,7,8,12],"fine":[3,8],"finish":12,"first":[0,3,6,7,8,12],"first_strip_reply_to":[3,8],"fit":[3,8,12],"fix":[4,12],"flag":[0,2,6,12],"flat":0,"flexibl":11,"flight":12,"float":[0,12],"flush":12,"fo":[0,10],"fold":0,"folder":[0,2,12],"foldersizelimit":2,"follow":[2,4,5,12],"footer":[3,8],"forc":12,"foreground":12,"forens":[5,7,12],"forensicparsedreport":0,"forensicreport":0,"form":12,"format":[0,7,12],"former":[5,7,12],"forward":[3,7,8],"found":[0,4,6,12],"foundat":10,"fqdn":4,"fraud":5,"free":[6,12],"fresh":[0,4,12],"freshest":12,"friend":7,"from_is_list":[3,8],"ftp_proxi":6,"full":12,"fulli":[0,3,4,7,8,12],"function":0,"g":[0,2,3,4,6,8,12],"gateway":2,"gb":4,"gcc":[6,12],"gdpr":[4,9],"gelf":[5,12],"general":[3,6,8,12],"generat":[3,4,8,10],"geoip":[6,12],"geoipupd":6,"geolite2":5,"geoloc":[0,12],"get":[0,2,4,6,12],"get_base_domain":0,"get_dmarc_reports_from_mailbox":0,"get_dmarc_reports_from_mbox":0,"get_filename_safe_str":0,"get_ip_address_countri":0,"get_ip_address_db_record":0,"get_ip_address_info":0,"get_report_zip":0,"get_reverse_dn":0,"get_service_from_reverse_dns_base_domain":0,"ghcr":12,"github":[1,6,10,12],"give":[0,4],"given":[0,12],"glass":7,"glob":12,"global":12,"gmail":[5,7,12],"gmail_api":12,"go":[0,3,8],"goe":[3,8],"googl":[7,12],"googleapi":12,"got":12,"gov":12,"govern":12,"gpg":4,"grafana":5,"grant":12,"graph":[0,2,5,7,12],"graph_url":12,"graylog":5,"group":[2,7,12],"guard":0,"guid":[4,5],"guidanc":12,"gzip":[0,5],"h":[0,4,12],"hamburg":4,"hand":[3,8],"handl":[5,12],"handler":7,"happen":[0,4],"hard":12,"has_defect":10,"hasn":12,"head":10,"header":[0,3,7,8,10,12],"header_from":10,"headless":2,"health":12,"healthcar":12,"heap":4,"heavi":[4,12],"hec":[0,11,12],"hecclient":0,"hectokengoesher":12,"help":5,"hh":0,"hierarchi":12,"high":[7,12],"higher":[3,8],"histor":[4,12],"histori":12,"hit":[0,12],"home":6,"hop":10,"host":[0,2,3,4,5,8,12],"hostnam":[0,12],"hour":[0,12],"hover":7,"href":10,"html":[3,4,8,10],"http":[0,2,4,5,6,10,11,12],"http_proxi":6,"https":[0,1,2,3,4,6,8,9,12],"https_proxi":6,"httpx":12,"human":[0,7],"human_timestamp":0,"human_timestamp_to_datetim":0,"human_timestamp_to_unix_timestamp":0,"hup":12,"icon":7,"id":[3,4,8,10,12],"ideal":[3,8],"idempot":4,"ident":[3,4,8,12],"identifi":10,"idl":[0,2,12],"ignor":[0,12],"imag":12,"imap":[0,2,5,12],"imap_password":12,"imapalwaysapproxmsgs":2,"imapautoexpung":2,"imapcli":5,"imapidledelay":2,"imapport":2,"immedi":[2,12],"immut":12,"impli":12,"import":[4,7,12],"improv":12,"inbox":[0,3,5,8,12],"inc":10,"includ":[0,3,4,6,7,8,12],"include_list_post_head":[3,8],"include_rfc2369_head":[3,8],"include_sender_head":[3,8],"include_spam_trash":12,"incom":[7,12],"incorrect":12,"increas":[4,12],"increment":12,"indefinit":12,"indent":12,"index":[0,5,7,9,11,12],"index_prefix":[0,4,12],"index_prefix_domain_map":12,"index_suffix":[0,4,12],"indic":[3,5],"individu":[7,12],"industri":12,"inform":[0,4,7,12],"infrequ":12,"ingest":12,"ini":[2,12],"initi":0,"inner":12,"input":0,"input_":0,"insid":4,"inspect":12,"instal":[0,2,5,12],"installed_app":12,"instanc":12,"instanceof":4,"instead":[0,3,4,6,8,12],"instruct":4,"int":[0,12],"intend":[3,8],"interact":[2,4,12],"interakt":10,"interfer":[3,8],"interpret":[0,6],"interrupt":12,"interval":12,"interval_begin":10,"interval_end":10,"invalid":[0,12],"invalidaggregatereport":0,"invaliddmarcreport":0,"invalidfailurereport":0,"invalidforensicreport":0,"invalidipinfoapikey":0,"invalidsmtptlsreport":0,"invis":4,"involv":7,"io":[0,12],"ip":[0,3,4,7,12],"ip_address":[0,10],"ip_db_path":[0,6,12],"ip_db_url":12,"ipaddressinfo":0,"ipinfo":[0,6,12],"ipinfo_api_token":12,"ipinfo_url":12,"ipsourceinfo":0,"ipv4":0,"ipv6":0,"is_mbox":0,"is_outlook_msg":0,"iso":[0,12],"issu":[0,1],"item":[0,12],"java":2,"job":[3,6,8],"joe":[3,8],"journalctl":[2,12],"jre":2,"json":[0,4,5,12],"june":5,"junk":12,"just":[4,7],"jvm":4,"jwt":12,"kafka":[5,12],"kb4099855":6,"kb4134118":6,"kb4295699":6,"keep":[0,4,7,12],"keep_al":0,"keepal":2,"key":[0,3,4,6,12],"keyfile_path":12,"keyout":4,"keyr":4,"keystor":4,"keyword":0,"kibana":[5,11],"kill":12,"killsign":12,"kind":12,"know":3,"known":[0,3,7,8,12],"kubernet":12,"kwarg":0,"l4":12,"l5":12,"label":12,"lack":4,"lang":4,"languag":[3,8],"larg":[2,12],"larger":12,"last":6,"later":[0,4,6,12],"latest":[2,4,9,12],"layer":0,"layout":11,"leak":7,"least":[4,6,12],"leav":3,"left":[0,7],"legaci":[0,5],"legal":[3,8],"legitim":[7,12],"less":12,"level":[0,3,4,12],"lf":12,"libemail":6,"libpq":12,"librari":12,"libxml2":6,"libxslt":6,"licens":6,"life":5,"lifetim":0,"lifetimetimeout":0,"like":[0,3,6,7,8,12],"limit":[0,2,12],"line":[3,8,12],"link":[3,4,7,8],"linux":[3,6,8],"list":[0,2,4,5,7,12],"listen":[2,12],"lite":[0,6,12],"live":[7,12],"ll":[3,8],"load":[0,4,12],"load_ip_db":0,"load_psl_overrid":0,"load_reverse_dns_map":0,"local":[0,2,4,6,10,12],"local_file_path":0,"local_psl_overrides_path":12,"local_reverse_dns_map_path":12,"localhost":[4,12],"locat":[7,12],"log":[0,2,4,5,12],"log_analyt":12,"log_fil":12,"logger":12,"login":[4,12],"logstash":4,"long":[0,3,12],"longer":[3,6,8],"look":[0,3,7],"lookup":[0,12],"loop":[0,12],"loopback":2,"loss":0,"lot":7,"low":12,"lower":12,"lua":10,"m":[0,6,12],"m365":12,"maco":6,"magnifi":7,"mail":[0,5,6,10,12],"mail_bcc":0,"mail_cc":0,"mail_from":0,"mail_to":0,"mailbox":[0,7,12],"mailbox_check_timeout":12,"mailbox_connect":0,"mailboxconnect":0,"maildir":12,"maildir_cr":12,"maildir_path":12,"mailer":10,"mailrelay":10,"mailsuit":12,"mailto":6,"main":4,"mainpid":12,"maintain":5,"make":[0,3,4,6,8,9,12],"malici":[7,12],"manag":[4,7,12],"mandatori":12,"manual":[0,4,12],"map":0,"mariadb":12,"market":7,"massiv":12,"match":[0,4,7,11,12],"max_ag":10,"max_shards_per_nod":12,"maximum":4,"maxmind":[0,5,12],"may":[5,7,12],"mbox":[0,12],"md":0,"mean":12,"meantim":0,"mechan":3,"member":[3,8],"memori":12,"mention":7,"menu":[4,7],"merg":0,"messag":[0,2,3,4,6,7,8,10,12],"message_id":10,"meta":10,"method":12,"metric":7,"mfrom":[4,10],"microsoft":[0,2,5,10,12],"microsoftgraph":12,"microsoftonlin":12,"mid":12,"might":[0,3,7,8],"migrat":[0,7,12],"migrate_index":0,"mime":10,"min":0,"minim":12,"minimum":[4,12],"minimum_should_match":4,"minut":[0,2,12],"mirror":0,"miss":[4,6,12],"mitig":[3,8],"mix":0,"mm":[0,12],"mmdb":[0,6,12],"mobil":[3,8],"mode":[0,2,4,6,10],"modern":[2,3,8],"modifi":[0,3,8,12],"modul":[0,5,6,12],"mon":10,"monitor":[3,12],"month":[0,12],"monthly_index":[0,12],"mous":7,"move":[0,4,12],"ms":[0,10,12],"msal":12,"msg":[0,6],"msg_byte":0,"msg_date":0,"msg_footer":[3,8],"msg_header":[3,8],"msgconvert":[0,6],"msgraph":12,"msgraphconnect":0,"mta":7,"much":12,"multi":[2,5],"multipl":[0,7,12],"mung":[3,8],"must":[2,3,4,8,12],"must_not":4,"mutual":[4,12],"mv":4,"mx":[7,10],"n":[0,10,12],"n_proc":12,"naiv":0,"name":[0,3,4,7,10,11],"nameserv":[0,12],"nano":[2,12],"nation":12,"navig":[3,8],"ncontent":10,"ndate":10,"ndjson":[4,7],"necessarili":7,"need":[0,2,3,4,6,7,8,12],"neither":12,"nelson":[3,8],"net":[2,12],"network":[0,2,4,12],"never":[0,4,12],"new":[0,2,4,5,6,7,12],"newer":6,"newest":[2,12],"newkey":4,"news":3,"next":[0,4,12],"nfrom":10,"nmessag":10,"nmime":10,"node":4,"nologin":6,"non":[0,3,4,8,12],"nonameserv":0,"none":[0,3,4,10,12],"noproxyfor":2,"norepli":[3,10],"normal":[0,10,12],"normalize_timespan_threshold_hour":0,"normalized_timespan":10,"nosecureimap":2,"notabl":7,"note":12,"noth":[4,12],"notic":12,"now":[4,6,7],"nsubject":10,"nto":10,"null":[4,6,10],"number":[0,7,12],"number_of_replica":[0,12],"number_of_shard":[0,12],"nwettbewerb":10,"nx":10,"o":[2,4,12],"oR":6,"oauth2":12,"oauth2_port":12,"object":[0,4,7],"observ":[7,12],"occur":[0,7],"occurr":11,"oct":10,"offic":2,"office365":2,"offici":12,"offlin":[0,6,12],"offset":[0,12],"often":[7,12],"old":[0,7],"older":[4,6,10,12],"oldest":[2,12],"ole":[0,6],"omit":12,"onc":[0,4,7,12],"ondmarc":5,"one":[0,3,4,5,6,7,8,12],"onli":[0,2,3,4,6,7,8],"onlin":[0,2,12],"onto":0,"oor":0,"op":0,"open":[0,3],"opendn":12,"opensearch":[4,5,7,12],"opensearch_dashboard":7,"opensearcherror":0,"openssl":4,"oper":12,"opt":[2,6,12],"option":[0,2,3,4,5,8,11,12],"order":12,"org":[0,6,9,10,12],"org_email":10,"org_extra_contact_info":10,"org_nam":10,"organiz":[2,5,7,12],"organization_nam":10,"origin":[3,8,12],"original_envelope_id":10,"original_mail_from":10,"original_rcpt_to":10,"original_timespan_second":10,"os":0,"oserror":0,"otherwis":[0,12],"outdat":7,"outgo":[3,8,12],"outlook":[0,2,6,12],"output":[0,5,12],"output_directori":0,"outsid":12,"overal":0,"overrid":[0,6,12],"overwrit":[0,4],"owa":[5,12],"owned":6,"ownership":6,"p":[3,4,10],"p12":4,"pack":4,"packag":[0,4,6],"packet":0,"pad":0,"page":[3,4,6,7,8],"paginate_messag":12,"painless":4,"pair":[4,7],"pan":10,"panel":7,"parallel":12,"paramet":[0,12],"parent":7,"pars":[0,3,5,6,10,12],"parse_aggregate_report_fil":0,"parse_aggregate_report_xml":0,"parse_email":0,"parse_failure_report":0,"parse_forensic_report":0,"parse_report_email":0,"parse_report_fil":0,"parse_smtp_tls_report_json":0,"parsed_aggregate_reports_to_csv":0,"parsed_aggregate_reports_to_csv_row":0,"parsed_failure_reports_to_csv":0,"parsed_failure_reports_to_csv_row":0,"parsed_forensic_reports_to_csv":0,"parsed_forensic_reports_to_csv_row":0,"parsed_sampl":10,"parsed_smtp_tls_reports_to_csv":0,"parsed_smtp_tls_reports_to_csv_row":0,"parsedemail":0,"parsedmarc":[4,9,10,11],"parsedmarc_":12,"parsedmarc_config_fil":12,"parsedmarc_debug":12,"parsedmarc_elasticsearch_":12,"parsedmarc_elasticsearch_host":12,"parsedmarc_elasticsearch_ssl":12,"parsedmarc_gelf_":12,"parsedmarc_general_":12,"parsedmarc_general_debug":12,"parsedmarc_general_ipinfo_api_token":12,"parsedmarc_general_ipinfo_url":12,"parsedmarc_general_offlin":12,"parsedmarc_general_save_aggreg":12,"parsedmarc_general_save_failur":12,"parsedmarc_gmail_api_":12,"parsedmarc_gmail_api_credentials_file_fil":12,"parsedmarc_imap_":12,"parsedmarc_imap_host":12,"parsedmarc_imap_password":12,"parsedmarc_imap_password_fil":12,"parsedmarc_imap_us":12,"parsedmarc_kafka_":12,"parsedmarc_log_analytics_":12,"parsedmarc_mailbox_":12,"parsedmarc_mailbox_watch":12,"parsedmarc_maildir_":12,"parsedmarc_msgraph_":12,"parsedmarc_opensearch_":12,"parsedmarc_s3_":12,"parsedmarc_smtp_":12,"parsedmarc_splunk_hec_":12,"parsedmarc_splunk_hec_index":12,"parsedmarc_splunk_hec_token":12,"parsedmarc_splunk_hec_url":12,"parsedmarc_syslog_":12,"parsedmarc_webhook_":12,"parser":0,"parsererror":0,"parsingresult":0,"part":[3,4,7,8,12],"particular":[7,12],"pass":[0,3,7,10,12],"passag":7,"passsword":12,"password":[0,4,6,12],"paste":[4,11],"patch":6,"path":[0,4,6,12],"pathlik":0,"pattern":[0,5,7,12],"payload":[0,12],"pct":10,"peak":12,"pem":12,"per":[0,4,7,12],"percentag":7,"perform":[2,5],"period":12,"perl":[0,6],"permiss":[4,12],"persist":12,"peter":10,"pick":[6,12],"pickup":6,"pid":12,"pie":7,"pin":12,"pip":[6,12],"pkcs12":12,"place":[0,4,7,12],"plain":[0,12],"plaintext":[3,8],"platform":[3,6,8,12],"pleas":[1,5,12],"plug":12,"plus":[7,12],"point":[4,6,12],"pol":4,"polici":[0,3,4,7,8,10,12],"policies_combin":[0,4],"policy_domain":[4,10],"policy_evalu":10,"policy_override_com":10,"policy_override_reason":10,"policy_publish":10,"policy_str":10,"policy_typ":[4,10],"policyscopegroupid":12,"poll":[0,2,12],"popul":0,"port":[0,2,12],"portal":12,"posit":[0,12],"posix":0,"possibl":12,"post":[3,4,8,12],"poster":[3,8],"postgr":12,"postgresql":[5,12],"postorius":[3,8],"powershel":12,"ppa":6,"practic":12,"pre":[6,12],"prebuilt":12,"predict":12,"prefer":[2,6,12],"prefix":[0,3,8,12],"premad":[5,11],"prepend":0,"prerequisit":5,"present":12,"pressur":12,"pretti":12,"prettifi":12,"previous":[0,2,4,6,12],"pri":[2,12],"primari":0,"print":12,"printabl":10,"prioriti":12,"privaci":[3,6,7,8,12],"privat":12,"probe":0,"problem":12,"proc":12,"proceed":4,"process":[0,2,5,6,12],"produc":[0,10],"program":12,"programdata":6,"progress":[4,12],"project":[0,2,3,5,11,12],"prompt":4,"proofpoint":5,"properti":2,"protect":[2,3,5,8,12],"protocol":12,"provid":[0,4,7,12],"provis":12,"prox":6,"proxi":2,"proxyhost":2,"proxypassword":2,"proxyport":2,"proxyus":2,"ps":4,"psl":[0,12],"psl_overrid":0,"psl_overrides_path":0,"psl_overrides_url":[0,12],"psycopg":12,"public":[0,3,10,12],"public_suffix_list":0,"publicbaseurl":4,"publicsuffix":0,"publish":[3,12],"published_polici":0,"pull":12,"purpos":4,"put":[4,12],"python":[0,4,6],"python3":6,"qo":4,"quarantin":[3,8],"queri":[0,4,12],"query_dn":0,"quick":0,"quickstart":12,"quit":12,"quot":[10,12],"quota":[0,12],"r":[2,10,12],"rais":[0,12],"ram":[4,12],"rate":[0,12],"rather":[0,3,4,7,8,12],"ratio":7,"raw":12,"re":[4,6,12],"reach":12,"reachabl":12,"read":[0,12],"readabl":[0,12],"readwrit":12,"real":[0,7],"realli":3,"reason":[0,2,4,5,12],"rebuilt":0,"receiv":[0,7,10,12],"receiveddatetim":12,"receiving_ip":[4,10],"receiving_mx_hostnam":[4,10],"recent":0,"recipi":7,"recogn":7,"recommend":12,"recommended_dns_nameserv":0,"record":[0,5,6,10,12],"record_typ":0,"recurs":12,"redact":12,"redi":12,"reduc":[6,12],"refer":[4,5,7],"referenc":12,"refresh":[6,12],"refresh_interv":12,"refus":4,"regard":12,"regardless":[0,10,12],"region":[0,12],"region_nam":12,"regist":[6,12],"registr":12,"regul":[4,6,9,12],"regular":[3,8],"reject":[0,3,8,12],"relat":[3,12],"relay":[3,8],"releas":[0,4,6],"reli":[6,7],"reliabl":12,"reload":[0,2,4],"remain":[0,7,12],"remot":2,"remov":[0,3,4,8,12],"render":7,"repeat":[0,3,8],"replac":[0,3,4,8,12],"repli":[2,3,8],"replic":12,"replica":[0,12],"reply_goes_to_list":[3,8],"reply_to":10,"replyto":[3,8],"repopul":0,"report":[0,4,11,12],"report_id":10,"report_metadata":10,"report_typ":0,"reported_domain":10,"reports_fold":[0,12],"repositori":[6,11],"req":4,"request":[0,2,4,12],"requir":[0,2,3,4,5,6,7,8,12],"require_encrypt":0,"res":4,"reserv":12,"resid":12,"resolv":[0,12],"resort":6,"resourc":[0,4,5,12],"respect":7,"respons":[0,12],"rest":[0,12],"restart":[2,3,4,6,8],"restartsec":[2,12],"restor":4,"restrict":12,"restrictaccess":12,"restructur":7,"result":[0,5,7,10,12],"result_typ":[4,10],"resum":12,"retain":[3,8,12],"retent":5,"retri":[0,4,12],"retriev":2,"retry_attempt":12,"retry_delay":12,"return":[0,4],"revers":[0,6,7,12],"reverse_dn":[0,10],"reverse_dns_base_domain":0,"reverse_dns_map":0,"reverse_dns_map_path":0,"reverse_dns_map_url":[0,12],"reversednsservic":0,"review":7,"rewrit":[0,3,8],"rfc":[0,3,5,8,10],"rfc2369":[3,8],"rfc822":2,"rhel":[4,5,6],"ri":4,"right":[4,7],"rm":4,"rmh":4,"ro":0,"rocki":6,"rollup":6,"root":[2,12],"rough":12,"row":7,"rpm":4,"rpt":[5,7],"rsa":4,"rt":4,"rua":[5,6],"ruf":[5,6,7,12],"rule":[7,12],"run":[0,4,5,6],"runtimeerror":12,"rw":[2,12],"s":[0,2,3,4,6,7,8,10,12],"s3":[5,12],"safe":[0,4],"safer":12,"sampl":[0,5,7,12],"sample_headers_on":10,"satisfi":7,"save":[0,4,6,7,12],"save_aggreg":12,"save_aggregate_report_to_elasticsearch":0,"save_aggregate_report_to_opensearch":0,"save_aggregate_reports_to_splunk":0,"save_failur":12,"save_failure_report_to_elasticsearch":0,"save_failure_report_to_opensearch":0,"save_failure_reports_to_splunk":0,"save_forens":12,"save_forensic_report_to_elasticsearch":0,"save_forensic_report_to_opensearch":0,"save_forensic_reports_to_splunk":0,"save_output":0,"save_smtp_tl":12,"save_smtp_tls_report_to_elasticsearch":0,"save_smtp_tls_report_to_opensearch":0,"save_smtp_tls_reports_to_splunk":0,"sbin":6,"sc":4,"scalar":4,"schedul":[6,12],"schema":[5,10,12],"scheme":0,"scope":[4,7,10,12],"script":[4,6],"scrub_nondigest":[3,8],"sdk":12,"search":[0,3,4,8,12],"second":[0,2,12],"secret_access_key":12,"section":4,"secur":[0,4,12],"see":[2,3,4,6,7,12],"seek":0,"segment":7,"sel":4,"select":0,"selector":[4,7,10],"self":[4,5],"send":[0,2,3,4,5,7,8,11,12],"sender":[5,7,8],"sending_mta_ip":[4,10],"sendmail":[0,12],"sensit":12,"sent":[0,3,8,12],"sentinel":5,"separ":[0,3,4,6,7,9,11,12],"sequenc":0,"sequenti":12,"serial":12,"server":[0,2,3,4,5,6,7,10,12],"server_ip":4,"serverless":[0,12],"servernameon":10,"servic":[0,3,4,5,6,7,8,10],"service_account":12,"service_account_us":12,"session":[0,7],"set":[0,2,3,4,6,7,8,9,12],"set_host":0,"setup":[4,6,9,12],"shape":[0,4],"shard":[0,12],"share":[4,6,7,12],"sharealik":6,"sharepoint":10,"shell":6,"ship":[6,12],"short":12,"shot":12,"shouldn":[3,8],"show":[2,7,12],"shown":[6,7,12],"shutdown":[0,12],"sibl":7,"side":[7,12],"sighup":[0,6,12],"sigkil":12,"sign":[0,3,4,6,12],"signal":12,"signatur":[3,7,8],"sigterm":[0,12],"sigv4":[0,12],"silent":[6,12],"similar":7,"simpl":5,"simpli":0,"simplifi":0,"sinc":[0,6,7,12],"singl":[0,7,12],"sink":12,"sister":3,"six":12,"size":[2,4],"skel":6,"skip":[0,4,12],"skip_certificate_verif":[0,12],"slight":11,"slow":0,"small":[4,12],"smaller":12,"smi":4,"smtp":[0,3,4,5,12],"smtp_tls":[0,4,12],"smtp_tls_csv_filenam":[0,12],"smtp_tls_index":0,"smtp_tls_json_filenam":[0,12],"smtp_tls_report":0,"smtp_tls_url":12,"smtptlsfailuredetail":0,"smtptlsfailuredetailsopt":0,"smtptlsparsedreport":0,"smtptlspolici":0,"smtptlspolicysummari":0,"smtptlsreport":0,"socket":2,"solut":6,"somehow":12,"someon":4,"sometim":12,"sort":12,"sourc":[0,3,4,6,7,10],"source_as_domain":10,"source_as_nam":10,"source_asn":10,"source_base_domain":10,"source_countri":10,"source_ip_address":10,"source_nam":10,"source_reverse_dn":10,"source_typ":10,"sourceforg":2,"sovereign":12,"sp":[3,4,10],"spam":12,"special":12,"specif":[3,6,7,12],"specifi":[2,3],"spf":[0,5,7,10],"spf_align":10,"spf_domain":10,"spf_result":[4,10],"spf_results_combin":[0,4],"spf_scope":10,"splunk":[5,12],"splunk_hec":12,"splunkerror":0,"splunkhec":12,"sponsor":5,"spoof":[3,8],"spurious":12,"sr":4,"ss":0,"ssl":[0,2,4,12],"ssl_cert_path":0,"stabl":4,"stack":[4,7,12],"standard":[0,5,6,10],"start":[0,2,4,7,9,11,12],"starttl":[7,12],"startup":[0,4,6],"static":12,"status":[2,12],"stay":7,"stdout":12,"step":[3,4,6,8,12],"still":[0,3,4,8,10,12],"stop":12,"storag":[0,4,12],"store":[2,4,7,9,12],"str":[0,12],"straight":12,"stream":12,"string":[0,4,7,12],"strip":[0,3,8,12],"strip_attachment_payload":[0,12],"strong":12,"structur":5,"sts":[7,10,12],"stsv1":10,"style":0,"subdomain":[0,3,12],"subfield":4,"subject":[0,3,8,10,12],"subject_prefix":[3,8],"submiss":0,"submit":[0,4],"subsidiari":7,"substitut":6,"success":12,"successful_sess":7,"successful_session_count":10,"sudo":[2,4,6,12],"suffici":12,"suffix":0,"suggest":7,"suit":12,"suitabl":0,"sum":7,"summari":[3,8,12],"supervis":12,"suppli":[0,7,12],"support":[2,4,5,7,10,11],"sure":4,"surfac":[7,12],"sw50zxjha3rpdmugv2v0dgjld2vyymvylcocymvyc2ljahq":10,"switch":7,"syslog":[2,5,12],"system":[2,3,4,6,8,12],"systemctl":[2,4,12],"systemd":5,"systemdr":6,"t":[0,4,5,7,8,10,12],"tab":[3,4,8],"tabl":[4,5,7,12],"tag":6,"take":[0,12],"talk":[0,4],"target":[0,2,12],"task":[0,4,6],"tbi":10,"tcp":12,"tee":4,"tell":[3,7,8],"templat":[3,8],"temporari":7,"tenant":5,"tenant_id":12,"term":6,"test":[0,10,12],"text":[0,4,10],"thank":10,"therebi":[3,8],"third":0,"though":7,"thousand":12,"three":7,"throughput":12,"tier":12,"time":[0,2,4,6,7,12],"timeout":[0,2,12],"timeoutstopsec":12,"timespan":0,"timespan_requires_norm":10,"timestamp":[0,12],"timestamp_to_datetim":0,"timestamp_to_human":0,"timezon":10,"tld":3,"tls":[0,4,5,12],"to_domain":10,"to_utc":0,"togeth":[7,12],"token":[0,4,12],"token_fil":12,"tool":12,"top":[3,7,12],"topic":12,"total":7,"touch":[3,8],"tracker":1,"trade":12,"tradit":[3,8],"traffic":7,"trail":12,"transfer":10,"transient":[0,12],"transpar":5,"transport":[4,12],"trash":12,"treat":0,"treatment":0,"tri":[0,12],"troubleshoot":12,"true":[0,2,4,10,12],"trust":12,"truststor":4,"truth":7,"tsvb":7,"tuesday":6,"tune":5,"two":[6,7],"txt":[0,12],"typ":4,"type":[4,5,7,10,12],"typic":12,"typo":12,"u":[2,6,12],"ubuntu":[4,6],"udp":[0,12],"ui":[3,8],"unchang":[0,12],"uncondit":[3,8],"underlying":[0,12],"underneath":7,"underscor":12,"understand":[5,7],"unencrypt":12,"unexpir":12,"unfortun":[3,8],"unit":[0,2,12],"unix":0,"unknown":0,"unless":[6,12],"unreach":12,"unread":12,"unrel":6,"unsubscrib":[3,8],"unsuit":12,"unus":0,"unzip":2,"updat":[4,6,12],"update_by_queri":0,"upersecur":12,"upgrad":[2,5,6,12],"upload":12,"upper":7,"uppercas":12,"uri":[6,12],"url":[0,2,4,12],"us":[10,12],"usabl":12,"usag":12,"use":[0,3,4,5,8,10],"use_ssl":0,"user":[0,2,3,4,6,7,8,10,12],"user_ag":10,"useradd":[2,6],"usernam":[0,12],"usernamepassword":12,"usesystemproxi":2,"usr":[4,6],"utc":[0,12],"utf":10,"util":5,"v":12,"v2":0,"valid":[0,7,10,12],"valimail":5,"valu":[0,3,4,7,8,12],"var":[3,8,12],"variabl":5,"variant":12,"various":6,"vendor":3,"venv":[6,12],"verbatim":12,"verbos":12,"veri":[0,4,7,12],"verif":[0,4,12],"verifi":0,"verification_mod":4,"version":[0,2,4,5,9,10,11,12],"vew":2,"via":[0,2],"view":[7,12],"vim":4,"virtualenv":6,"visual":[4,7,9],"volum":[7,12],"vulner":3,"w":[0,12],"w3c":10,"wait":[0,12],"wait_for_complet":[0,4],"wall":0,"want":[2,12],"wantedbi":[2,12],"warn":[0,4,12],"watch":[0,2,4,6,12],"watch_inbox":0,"watcher":[0,12],"way":[0,4,7],"web":[2,4],"webdav":2,"webhook":[5,12],"webmail":[3,7,8],"week":[0,6,12],"well":[2,7,12],"wettbewerb":10,"wget":4,"whalensolut":12,"wheel":12,"whenev":[0,2,12],"wherea":7,"wherev":12,"whether":[0,12],"whi":[3,7,12],"whole":[0,7],"whose":12,"wide":[6,10,12],"wiki":10,"will":[0,2,3,4,6,7,8,12],"win":12,"window":[6,12],"within":0,"without":[3,4,6,7,8],"won":5,"work":[2,3,4,5,6,7,8,12],"worker":12,"workstat":2,"worst":[3,12],"worth":12,"wrap":[3,8],"wrapper":12,"write":[0,4,12],"written":12,"www":[4,6,12],"x":[4,7,10],"x509":4,"xennn":10,"xml":[0,11],"xml_schema":10,"xms4g":4,"xmx4g":4,"xpack":4,"xxxx":4,"y":[4,6],"yahoo":7,"yaml":12,"year":12,"yes":[3,8],"yet":[0,3,4,12],"yml":4,"yyyi":[0,12],"z":12,"zero":12,"zip":[0,2,5,12],"\u00fcbersicht":10},"titles":["API reference","Contributing to parsedmarc","Accessing an inbox using OWA/EWS","Understanding DMARC","Elasticsearch and Kibana","parsedmarc documentation - Open source DMARC report analyzer and visualizer","Installation","Using the Kibana dashboards","What about mailing lists?","OpenSearch and Grafana","Sample outputs","Splunk","Using parsedmarc"],"titleterms":{"Do":[3,8],"What":[3,8],"_file":12,"access":2,"aggreg":[7,10],"align":3,"analyz":[5,6],"api":0,"backfil":4,"best":[3,8],"bug":1,"cli":12,"combin":4,"compat":5,"compos":12,"config":12,"configur":[2,12],"content":5,"contribut":1,"countri":6,"csv":10,"dashboard":7,"databas":6,"davmail":2,"depend":6,"dkim":[3,4],"dmarc":[3,5,7],"docker":12,"document":5,"domain":3,"elast":0,"elasticsearch":4,"env":12,"environ":12,"ew":2,"exampl":12,"exchang":6,"failur":[7,10],"featur":5,"field":4,"file":12,"geolite2":6,"grafana":9,"guid":3,"help":12,"inbox":2,"index":4,"indic":0,"instal":[4,6,9],"ip":6,"json":10,"kibana":[4,7],"list":[3,8],"listserv":[3,8],"lookalik":3,"mail":[3,8],"mailman":[3,8],"map":12,"maxmind":6,"microsoft":6,"mode":12,"multi":12,"multipl":6,"name":12,"onli":12,"open":5,"opensearch":[0,9],"option":6,"output":10,"owa":2,"parsedmarc":[0,1,2,5,6,12],"pattern":4,"perform":12,"practic":[3,8],"prerequisit":6,"proxi":6,"python":5,"record":[3,4,9],"refer":0,"reload":12,"report":[1,5,6,7,10],"resourc":3,"restart":12,"result":4,"retent":[4,9],"run":[2,12],"sampl":10,"secret":12,"section":12,"sender":3,"servic":[2,12],"smtp":[7,10],"sourc":5,"specifi":12,"spf":[3,4],"splunk":[0,11],"suffix":12,"support":[3,12],"systemd":[2,12],"t":3,"tabl":0,"tenant":12,"test":6,"tls":[7,10],"tune":12,"type":0,"understand":3,"upgrad":4,"use":[2,6,7,12],"util":0,"valid":3,"variabl":12,"via":12,"visual":5,"web":6,"without":12,"won":3,"workaround":[3,8]}})
\ No newline at end of file
diff --git a/splunk.html b/splunk.html
index 773a34c2..0ff33af4 100644
--- a/splunk.html
+++ b/splunk.html
@@ -6,14 +6,14 @@
   
 
   
-  Splunk — parsedmarc 10.2.4 documentation
+  Splunk — parsedmarc 10.3.0 documentation
       
       
 
   
       
       
-      
+      
       
       
     
diff --git a/usage.html b/usage.html
index ccaa4ec6..e4d0148f 100644
--- a/usage.html
+++ b/usage.html
@@ -6,14 +6,14 @@
   
 
   
-  Using parsedmarc — parsedmarc 10.2.4 documentation
+  Using parsedmarc — parsedmarc 10.3.0 documentation
       
       
 
   
       
       
-      
+      
       
       
     
@@ -104,23 +104,25 @@
 
Using parsedmarc

 

 
CLI help

-
usage: parsedmarc [-h] [-c CONFIG_FILE] [--strip-attachment-payloads] [-o OUTPUT]
+
usage: parsedmarc [-h] [-c CONFIG_FILE] [-r] [--strip-attachment-payloads] [-o OUTPUT]
                   [--aggregate-json-filename AGGREGATE_JSON_FILENAME] [--failure-json-filename FAILURE_JSON_FILENAME]
                   [--smtp-tls-json-filename SMTP_TLS_JSON_FILENAME] [--aggregate-csv-filename AGGREGATE_CSV_FILENAME]
                   [--failure-csv-filename FAILURE_CSV_FILENAME] [--smtp-tls-csv-filename SMTP_TLS_CSV_FILENAME]
-                  [-n NAMESERVERS [NAMESERVERS ...]] [-t DNS_TIMEOUT] [--offline] [-s] [-w] [--verbose] [--debug]
-                  [--log-file LOG_FILE] [--no-prettify-json] [-v]
+                  [-n NAMESERVERS [NAMESERVERS ...]] [-t DNS_TIMEOUT] [--dns-retries DNS_RETRIES] [--offline] [-s]
+                  [-w] [--verbose] [--debug] [--log-file LOG_FILE] [--no-prettify-json] [-v]
                   [file_path ...]
 
 Parses DMARC reports
 
 positional arguments:
-  file_path             one or more paths to aggregate or failure report files, emails, or mbox files'
+  file_path             one or more paths to aggregate or failure report files, emails, mbox files, or directories
+                        containing them
 
 options:
   -h, --help            show this help message and exit
   -c CONFIG_FILE, --config-file CONFIG_FILE
                         a path to a configuration file (--silent implied)
+  -r, --recursive       search directories given as file_path recursively, and enable '**' recursion in glob patterns
   --strip-attachment-payloads
                         remove attachment payloads from failure report output
   -o OUTPUT, --output OUTPUT
@@ -141,6 +143,8 @@ options:
                         nameservers to query
   -t DNS_TIMEOUT, --dns_timeout DNS_TIMEOUT
                         number of seconds to wait for an answer from DNS (default: 2.0)
+  --dns-retries DNS_RETRIES
+                        number of times to retry DNS queries on timeout or other transient errors (default: 0)
   --offline             do not make online queries for geolocation or DNS
   -s, --silent          only print errors
   -w, --warnings        print warnings in addition to errors
@@ -264,13 +268,21 @@ DNS resolvers (Default: False)

 
  • log_file - str: Write log messages to a file at this path
    • 
-
  • n_procs - int: Number of process to run in parallel when
-parsing in CLI mode (Default: 1)
    
+
  • n_procs - int: Number of processes to run in parallel when
+parsing report files passed directly as CLI arguments
+(Default: 1)
    
 
    
 
    Note
    
 
    Setting this to a number larger than one can improve
 performance when processing thousands of files
    
 
    
+
    
+
    Note
    
+
    n_procs only applies to report files passed directly on the
+command line. Messages from mbox files and from mailbox
+connections (IMAP, Microsoft Graph, Gmail API, Maildir) are
+always processed sequentially.
    
+
    
 
    •