diff --git a/_modules/index.html b/_modules/index.html index fe413f7..f1e6516 100644 --- a/_modules/index.html +++ b/_modules/index.html @@ -5,14 +5,14 @@ - Overview: module code — parsedmarc 9.11.2 documentation + Overview: module code — parsedmarc 10.0.0 documentation - + diff --git a/_modules/parsedmarc.html b/_modules/parsedmarc.html index a60b860..939ef00 100644 --- a/_modules/parsedmarc.html +++ b/_modules/parsedmarc.html @@ -5,14 +5,14 @@ - parsedmarc — parsedmarc 9.11.2 documentation + parsedmarc — parsedmarc 10.0.0 documentation - + @@ -134,7 +134,8 @@ ) from parsedmarc.types import ( AggregateReport, - ForensicReport, + FailureReport, + ForensicReport as ForensicReport, ParsedReport, ParsingResults, SMTPTLSReport, @@ -155,10 +156,33 @@ xml_header_regex = re.compile(r"^<\?xml .*?>", re.MULTILINE) xml_schema_regex = re.compile(r"</??xs:schema.*>", re.MULTILINE) text_report_regex = re.compile(r"\s*([a-zA-Z\s]+):\s(.+)", re.MULTILINE) +# Captures the value of any xmlns (default or prefixed) declaration so the +# RFC 9990 namespace can be detected before xmltodict drops it. +xml_namespace_regex = re.compile( + r"""xmlns(?::[a-zA-Z_][\w.-]*)?\s*=\s*["']([^"']+)["']""" +) + +# The XML namespace assigned to DMARC aggregate reports by RFC 9990. +RFC_9990_NAMESPACE = "urn:ietf:params:xml:ns:dmarc-2.0" + +# PolicyOverrideType enumeration from RFC 9990. Compared to RFC 7489, +# `policy_test_mode` was added (emitted when t=y suppresses enforcement) +# and `forwarded` / `sampled_out` were removed. +RFC_9990_POLICY_OVERRIDE_TYPES = frozenset( + { + "local_policy", + "mailing_list", + "other", + "policy_test_mode", + "trusted_forwarder", + } +) +RFC_7489_REMOVED_POLICY_OVERRIDE_TYPES = frozenset({"forwarded", "sampled_out"}) MAGIC_ZIP = b"\x50\x4b\x03\x04" MAGIC_GZIP = b"\x1f\x8b" MAGIC_XML = b"\x3c\x3f\x78\x6d\x6c\x20" +MAGIC_XML_TAG = b"\x3c" # '<' - XML starting with an element tag (no declaration) MAGIC_JSON = b"\7b" EMAIL_SAMPLE_CONTENT_TYPES = ( @@ -205,13 +229,37 @@ -
-[docs] -class InvalidForensicReport(InvalidDMARCReport): - """Raised when an invalid DMARC forensic report is encountered"""
+
+[docs] +class InvalidFailureReport(InvalidDMARCReport): + """Raised when an invalid DMARC failure report is encountered"""
+# Backward-compatible alias +InvalidForensicReport = InvalidFailureReport + + +def _text(value: Any) -> Optional[str]: + """Unwrap a possibly-langAttrString value parsed by xmltodict. + + RFC 9990 changed several aggregate-report elements (extra_contact_info, + error, comment, human_result) to type ``langAttrString`` — an + xs:simpleContent string with an optional ``lang`` attribute. When the + attribute is present, xmltodict parses the element as + ``{"#text": "...", "@lang": "en"}`` instead of a plain string. Returns + the text payload for both shapes, ``None`` for unset values, and leaves + other scalar shapes untouched so callers can preserve whatever the + reporter sent. + """ + if value is None: + return None + if isinstance(value, dict): + text = value.get("#text") + return None if text is None else str(text) + return value + + def _bucket_interval_by_day( begin: datetime, end: datetime, @@ -404,6 +452,7 @@ nameservers: Optional[list[str]] = None, dns_timeout: float = DEFAULT_DNS_TIMEOUT, dns_retries: int = DEFAULT_DNS_MAX_RETRIES, + is_rfc_9990: bool = False, ) -> dict[str, Any]: """ Converts a record from a DMARC aggregate report into a more consistent @@ -453,8 +502,6 @@ } if "disposition" in policy_evaluated: new_policy_evaluated["disposition"] = policy_evaluated["disposition"] - if new_policy_evaluated["disposition"].strip().lower() == "pass": - new_policy_evaluated["disposition"] = "none" if "dkim" in policy_evaluated: new_policy_evaluated["dkim"] = policy_evaluated["dkim"] if "spf" in policy_evaluated: @@ -479,8 +526,27 @@ else: reasons = [policy_evaluated["reason"]] for reason in reasons: - if "comment" not in reason: - reason["comment"] = None + # `comment` is langAttrString in RFC 9990 — unwrap {"#text": ..., "@lang": ...} + reason["comment"] = _text(reason.get("comment")) + reason_type = reason.get("type") + if is_rfc_9990 and reason_type in RFC_7489_REMOVED_POLICY_OVERRIDE_TYPES: + logger.warning( + "Policy override reason type %r was removed in RFC 9990; " + "expected one of %s", + reason_type, + sorted(RFC_9990_POLICY_OVERRIDE_TYPES), + ) + elif ( + is_rfc_9990 + and reason_type is not None + and reason_type not in RFC_9990_POLICY_OVERRIDE_TYPES + ): + logger.warning( + "Unknown policy override reason type %r per RFC 9990; " + "expected one of %s", + reason_type, + sorted(RFC_9990_POLICY_OVERRIDE_TYPES), + ) new_policy_evaluated["policy_override_reasons"] = reasons new_record["policy_evaluated"] = new_policy_evaluated if "identities" in record: @@ -510,11 +576,18 @@ if "selector" in result and result["selector"] is not None: new_result["selector"] = result["selector"] else: + if is_rfc_9990: + logger.warning( + "DKIM auth result for %r is missing the 'selector' " + "element, which is REQUIRED by RFC 9990", + result["domain"], + ) new_result["selector"] = "none" if "result" in result and result["result"] is not None: new_result["result"] = result["result"] else: new_result["result"] = "none" + new_result["human_result"] = _text(result.get("human_result")) new_record["auth_results"]["dkim"].append(new_result) if not isinstance(auth_results["spf"], list): @@ -530,6 +603,7 @@ new_result["result"] = result["result"] else: new_result["result"] = "none" + new_result["human_result"] = _text(result.get("human_result")) new_record["auth_results"]["spf"].append(new_result) if "envelope_from" not in new_record["identifiers"]: @@ -813,6 +887,21 @@ # Parse XML and recover from errors if isinstance(xml, bytes): xml = xml.decode(errors="ignore") + + # Detect the XML namespace before any rewriting strips it. The dmarc-2.0 + # namespace is one of the indicators for an RFC 9990 report but it is + # NOT a reliable sole discriminator: the <version> element value is + # ambiguous (RFC 9990's appendix sample uses <version>1.0</version> + # inside the dmarc-2.0 namespace), and real-world reporters frequently + # emit RFC 9990-shaped reports without declaring the namespace at all. + # The final `is_rfc_9990` decision is made post-parse so that + # RFC 9990-only fields (np, testing, discovery_method, generator, + # human_result) can also vote it in. + xml_namespace: Optional[str] = None + namespace_match = xml_namespace_regex.search(xml) + if namespace_match: + xml_namespace = namespace_match.group(1) + try: xmltodict.parse(xml)["feedback"] except Exception as e: @@ -836,16 +925,24 @@ report = xmltodict.parse(xml)["feedback"] report_metadata = report["report_metadata"] + # <email> is xs:string in both RFC 7489 and RFC 9990, but defensive + # parsing in the wild: some reporters emit it with an xml:lang or + # similar attribute, which xmltodict turns into a dict. if isinstance(report_metadata.get("email"), dict): - logger.debug( - "Discarding malformed <email> in report_metadata: %r", - report_metadata["email"], - ) - report_metadata["email"] = None + unwrapped = _text(report_metadata["email"]) + if unwrapped is None: + logger.debug( + "Discarding malformed <email> in report_metadata: %r", + report_metadata["email"], + ) + report_metadata["email"] = unwrapped schema = "draft" if "version" in report: schema = report["version"] - new_report: dict[str, Any] = {"xml_schema": schema} + new_report: dict[str, Any] = { + "xml_schema": schema, + "xml_namespace": xml_namespace, + } new_report_metadata: dict[str, Any] = {} if report_metadata["org_name"] is None: if report_metadata["email"] is not None: @@ -866,9 +963,9 @@ ) new_report_metadata["org_name"] = org_name new_report_metadata["org_email"] = report_metadata["email"] - extra = None - if "extra_contact_info" in report_metadata: - extra = report_metadata["extra_contact_info"] + # extra_contact_info is langAttrString in RFC 9990 (xs:string in + # RFC 7489) — unwrap {"#text": ..., "@lang": ...} if present. + extra = _text(report_metadata.get("extra_contact_info")) new_report_metadata["org_extra_contact_info"] = extra new_report_metadata["report_id"] = report_metadata["report_id"] report_id = new_report_metadata["report_id"] @@ -896,17 +993,38 @@ new_report_metadata["end_date"], to_utc=True ) + # <error> is langAttrString in RFC 9990 (xs:string in RFC 7489) and + # was cardinality-narrowed from "unbounded" to "1" in RFC 9990, but + # the parser still accepts a list for backward compatibility with + # RFC 7489 reports that carry multiple errors. if "error" in report["report_metadata"]: - if not isinstance(report["report_metadata"]["error"], list): - errors = [report["report_metadata"]["error"]] - else: - errors = report["report_metadata"]["error"] + raw_errors = report["report_metadata"]["error"] + if not isinstance(raw_errors, list): + raw_errors = [raw_errors] + errors = [text for text in (_text(e) for e in raw_errors) if text] new_report_metadata["errors"] = errors + # <generator> is a plain xs:string in RFC 9990 but apply _text() so + # a malformed reporter that decorates it with attributes still + # yields a string instead of breaking downstream consumers. + generator = _text(report_metadata.get("generator")) + new_report_metadata["generator"] = generator new_report["report_metadata"] = new_report_metadata records = [] policy_published = report["policy_published"] if type(policy_published) is list: policy_published = policy_published[0] + + # Final RFC 9990 detection: the dmarc-2.0 XML namespace OR any + # RFC 9990-only field. Real-world reporters that follow the schema + # without declaring the namespace still get RFC 9990-aware + # warnings (missing DKIM selector, removed override-reason types, + # etc.) and a truthful audit trail in `xml_namespace`. + rfc_9990_only_policy_fields = {"np", "testing", "discovery_method"} + is_rfc_9990 = ( + xml_namespace == RFC_9990_NAMESPACE + or "generator" in report_metadata + or any(f in policy_published for f in rfc_9990_only_policy_fields) + ) new_policy_published: dict[str, Any] = {} new_policy_published["domain"] = policy_published["domain"] adkim = "r" @@ -925,16 +1043,39 @@ if policy_published["sp"] is not None: sp = policy_published["sp"] new_policy_published["sp"] = sp - pct = "100" + pct = None if "pct" in policy_published: if policy_published["pct"] is not None: pct = policy_published["pct"] new_policy_published["pct"] = pct - fo = "0" + fo = None if "fo" in policy_published: if policy_published["fo"] is not None: fo = policy_published["fo"] new_policy_published["fo"] = fo + np_ = None + if "np" in policy_published: + if policy_published["np"] is not None: + np_ = policy_published["np"] + if np_ not in ("none", "quarantine", "reject"): + logger.warning("Invalid np value: {0}".format(np_)) + new_policy_published["np"] = np_ + testing = None + if "testing" in policy_published: + if policy_published["testing"] is not None: + testing = policy_published["testing"] + if testing not in ("n", "y"): + logger.warning("Invalid testing value: {0}".format(testing)) + new_policy_published["testing"] = testing + discovery_method = None + if "discovery_method" in policy_published: + if policy_published["discovery_method"] is not None: + discovery_method = policy_published["discovery_method"] + if discovery_method not in ("psl", "treewalk"): + logger.warning( + "Invalid discovery_method value: {0}".format(discovery_method) + ) + new_policy_published["discovery_method"] = discovery_method new_report["policy_published"] = new_policy_published if type(report["record"]) is list: @@ -954,6 +1095,7 @@ nameservers=nameservers, dns_timeout=timeout, dns_retries=retries, + is_rfc_9990=is_rfc_9990, ) _append_parsed_record( parsed_record=report_record, @@ -976,6 +1118,7 @@ nameservers=nameservers, dns_timeout=timeout, dns_retries=retries, + is_rfc_9990=is_rfc_9990, ) _append_parsed_record( parsed_record=report_record, @@ -1072,6 +1215,7 @@ ) elif ( header[: len(MAGIC_XML)] == MAGIC_XML + or header[: len(MAGIC_XML_TAG)] == MAGIC_XML_TAG or header[: len(MAGIC_JSON)] == MAGIC_JSON ): report = file_object.read().decode(errors="ignore") @@ -1210,6 +1354,9 @@ sp = report["policy_published"]["sp"] pct = report["policy_published"]["pct"] fo = report["policy_published"]["fo"] + np_ = report["policy_published"].get("np", None) + testing = report["policy_published"].get("testing", None) + discovery_method = report["policy_published"].get("discovery_method", None) report_dict: dict[str, Any] = dict( xml_schema=xml_schema, @@ -1226,8 +1373,11 @@ aspf=aspf, p=p, sp=sp, + np=np_, pct=pct, fo=fo, + testing=testing, + discovery_method=discovery_method, ) for record in report["records"]: @@ -1329,8 +1479,11 @@ "aspf", "p", "sp", + "np", "pct", "fo", + "testing", + "discovery_method", "source_ip_address", "source_country", "source_reverse_dns", @@ -1372,9 +1525,9 @@ -
-[docs] -def parse_forensic_report( +
+[docs] +def parse_failure_report( feedback_report: str, sample: str, msg_date: datetime, @@ -1388,9 +1541,9 @@ dns_timeout: float = DEFAULT_DNS_TIMEOUT, dns_retries: int = DEFAULT_DNS_MAX_RETRIES, strip_attachment_payloads: bool = False, -) -> ForensicReport: +) -> FailureReport: """ - Converts a DMARC forensic report and sample to a dict + Converts a DMARC failure report and sample to a dict Args: feedback_report (str): A message's feedback report as a string @@ -1407,7 +1560,7 @@ dns_retries (int): Number of times to retry DNS queries on timeout or other transient errors strip_attachment_payloads (bool): Remove attachment payloads from - forensic report results + failure report results Returns: dict: A parsed report and sample @@ -1423,7 +1576,7 @@ if "arrival_date" not in parsed_report: if msg_date is None: - raise InvalidForensicReport("Forensic sample is not a valid email") + raise InvalidFailureReport("Failure sample is not a valid email") parsed_report["arrival_date"] = msg_date.isoformat() if "version" not in parsed_report: @@ -1465,21 +1618,37 @@ parsed_report["source"] = parsed_report_source del parsed_report["source_ip"] + # Identity-Alignment is REQUIRED per RFC 9991 §4. Default silently for + # backward compatibility with pre-9991 reporters, but log so the + # offending reporter is visible. Values are CFWS-separated per the + # ABNF, so each mechanism is stripped after splitting. if "identity_alignment" not in parsed_report: + logger.warning( + "Failure report missing required 'Identity-Alignment' " + "field (RFC 9991 §4); defaulting to no aligned mechanisms" + ) parsed_report["authentication_mechanisms"] = [] - elif parsed_report["identity_alignment"] == "none": - parsed_report["authentication_mechanisms"] = [] - del parsed_report["identity_alignment"] else: - auth_mechanisms = parsed_report["identity_alignment"] - auth_mechanisms = auth_mechanisms.split(",") - parsed_report["authentication_mechanisms"] = auth_mechanisms + raw_alignment = parsed_report["identity_alignment"].strip() + if raw_alignment.lower() == "none": + parsed_report["authentication_mechanisms"] = [] + else: + parsed_report["authentication_mechanisms"] = [ + m.strip() for m in raw_alignment.split(",") if m.strip() + ] del parsed_report["identity_alignment"] + # Auth-Failure is REQUIRED per RFC 9991 §4. Comma-separated per ABNF + # so strip each token. if "auth_failure" not in parsed_report: + logger.warning( + "Failure report missing required 'Auth-Failure' field " + "(RFC 9991 §4); defaulting to 'dmarc'" + ) parsed_report["auth_failure"] = "dmarc" - auth_failure = parsed_report["auth_failure"].split(",") - parsed_report["auth_failure"] = auth_failure + parsed_report["auth_failure"] = [ + f.strip() for f in parsed_report["auth_failure"].split(",") if f.strip() + ] optional_fields = [ "original_envelope_id", @@ -1510,30 +1679,30 @@ parsed_report["sample"] = sample parsed_report["parsed_sample"] = parsed_sample - return cast(ForensicReport, parsed_report) + return cast(FailureReport, parsed_report) except KeyError as error: - raise InvalidForensicReport("Missing value: {0}".format(error.__str__())) + raise InvalidFailureReport("Missing value: {0}".format(error.__str__())) except Exception as error: - raise InvalidForensicReport("Unexpected error: {0}".format(error.__str__()))
+ raise InvalidFailureReport("Unexpected error: {0}".format(error.__str__()))
-
-[docs] -def parsed_forensic_reports_to_csv_rows( - reports: Union[ForensicReport, list[ForensicReport]], +
+[docs] +def parsed_failure_reports_to_csv_rows( + reports: Union[FailureReport, list[FailureReport]], ) -> list[dict[str, Any]]: """ - Converts one or more parsed forensic reports to a list of dicts in flat CSV + Converts one or more parsed failure reports to a list of dicts in flat CSV format Args: - reports: A parsed forensic report or list of parsed forensic reports + reports: A parsed failure report or list of parsed failure reports Returns: - list: Parsed forensic report data as a list of dicts in flat CSV format + list: Parsed failure report data as a list of dicts in flat CSV format """ if isinstance(reports, dict): reports = [reports] @@ -1564,20 +1733,20 @@ -
-[docs] -def parsed_forensic_reports_to_csv( - reports: Union[ForensicReport, list[ForensicReport]], +
+[docs] +def parsed_failure_reports_to_csv( + reports: Union[FailureReport, list[FailureReport]], ) -> str: """ - Converts one or more parsed forensic reports to flat CSV format, including + Converts one or more parsed failure reports to flat CSV format, including headers Args: - reports: A parsed forensic report or list of parsed forensic reports + reports: A parsed failure report or list of parsed failure reports Returns: - str: Parsed forensic report data in flat CSV format, including headers + str: Parsed failure report data in flat CSV format, including headers """ fields = [ "feedback_type", @@ -1612,7 +1781,7 @@ csv_writer = DictWriter(csv_file, fieldnames=fields) csv_writer.writeheader() - rows = parsed_forensic_reports_to_csv_rows(reports) + rows = parsed_failure_reports_to_csv_rows(reports) for row in rows: new_row: dict[str, Any] = {} @@ -1656,13 +1825,13 @@ dns_retries (int): Number of times to retry DNS queries on timeout or other transient errors strip_attachment_payloads (bool): Remove attachment payloads from - forensic report results + failure report results keep_alive (callable): keep alive function normalize_timespan_threshold_hours (float): Normalize timespans beyond this Returns: dict: - * ``report_type``: ``aggregate`` or ``forensic`` + * ``report_type``: ``aggregate`` or ``failure`` * ``report``: The parsed report """ result: Optional[ParsedReport] = None @@ -1806,7 +1975,7 @@ if feedback_report and sample: try: - forensic_report = parse_forensic_report( + failure_report = parse_failure_report( feedback_report, sample, msg_date, @@ -1820,17 +1989,17 @@ dns_retries=dns_retries, strip_attachment_payloads=strip_attachment_payloads, ) - except InvalidForensicReport as e: + except InvalidFailureReport as e: error = ( 'Message with subject "{0}" ' "is not a valid " - "forensic DMARC report: {1}".format(subject, e) + "failure DMARC report: {1}".format(subject, e) ) - raise InvalidForensicReport(error) + raise InvalidFailureReport(error) except Exception as e: - raise InvalidForensicReport(e.__str__()) + raise InvalidFailureReport(e.__str__()) - result = {"report_type": "forensic", "report": forensic_report} + result = {"report_type": "failure", "report": failure_report} return result if result is None: @@ -1858,7 +2027,7 @@ keep_alive: Optional[Callable] = None, normalize_timespan_threshold_hours: float = 24, ) -> ParsedReport: - """Parses a DMARC aggregate or forensic file at the given path, a + """Parses a DMARC aggregate or failure file at the given path, a file-like object. or bytes Args: @@ -1870,7 +2039,7 @@ dns_retries (int): Number of times to retry DNS queries on timeout or other transient errors strip_attachment_payloads (bool): Remove attachment payloads from - forensic report results + failure report results ip_db_path (str): Path to a MMDB file from IPinfo, MaxMind, or DBIP always_use_local_files (bool): Do not download files reverse_dns_map_path (str): Path to a reverse DNS map @@ -1969,7 +2138,7 @@ dns_retries (int): Number of times to retry DNS queries on timeout or other transient errors strip_attachment_payloads (bool): Remove attachment payloads from - forensic report results + failure report results always_use_local_files (bool): Do not download files reverse_dns_map_path (str): Path to a reverse DNS map file reverse_dns_map_url (str): URL to a reverse DNS map file @@ -1978,11 +2147,11 @@ normalize_timespan_threshold_hours (float): Normalize timespans beyond this Returns: - dict: Lists of ``aggregate_reports``, ``forensic_reports``, and ``smtp_tls_reports`` + dict: Lists of ``aggregate_reports``, ``failure_reports``, and ``smtp_tls_reports`` """ aggregate_reports: list[AggregateReport] = [] - forensic_reports: list[ForensicReport] = [] + failure_reports: list[FailureReport] = [] smtp_tls_reports: list[SMTPTLSReport] = [] try: mbox = mailbox.mbox(input_) @@ -2020,8 +2189,8 @@ "Skipping duplicate aggregate report " f"from {report_org} with ID: {report_id}" ) - elif parsed_email["report_type"] == "forensic": - forensic_reports.append(parsed_email["report"]) + elif parsed_email["report_type"] == "failure": + failure_reports.append(parsed_email["report"]) elif parsed_email["report_type"] == "smtp_tls": smtp_tls_reports.append(parsed_email["report"]) except InvalidDMARCReport as error: @@ -2030,12 +2199,60 @@ raise InvalidDMARCReport("Mailbox {0} does not exist".format(input_)) return { "aggregate_reports": aggregate_reports, - "forensic_reports": forensic_reports, + "failure_reports": failure_reports, "smtp_tls_reports": smtp_tls_reports, }
+def _migrate_forensic_archive_folder( + connection: MailboxConnection, archive_folder: str +) -> None: + """Consolidate a pre-rename ``<archive>/Forensic`` subfolder into + ``<archive>/Failure``. + + Before failure reports were renamed from "forensic" reports, they were + archived under ``<archive_folder>/Forensic``; they now go to + ``<archive_folder>/Failure``. This best-effort, run-on-startup migration + moves any pre-existing legacy archive into the new location so reports + filed before and after the rename live in the same folder. + + It is a no-op when there is no legacy ``Forensic`` folder (the common + case), and never raises: a mailbox that cannot be reorganized is logged + and skipped, consistent with the rest of parsedmarc's mailbox handling + (warn, don't crash). Uses the folder-management API added in mailsuite + 2.1.0 (``folder_exists`` / ``rename_folder`` / ``merge_folders``). + """ + old_folder = "{0}/Forensic".format(archive_folder) + new_folder = "{0}/Failure".format(archive_folder) + try: + if not connection.folder_exists(old_folder): + return + if connection.folder_exists(new_folder): + # Both exist (e.g. a partial earlier migration, or a manually + # created Failure folder): move the legacy folder's messages into + # the new one and drop the now-empty legacy folder. + connection.merge_folders(old_folder, new_folder) + logger.info( + "Merged legacy archive folder {0} into {1}".format( + old_folder, new_folder + ) + ) + else: + connection.rename_folder(old_folder, new_folder) + logger.info( + "Renamed legacy archive folder {0} to {1}".format( + old_folder, new_folder + ) + ) + except Exception as error: + logger.warning( + "Could not migrate legacy archive folder {0} to {1}: {2}".format( + old_folder, new_folder, error + ) + ) + +
[docs] def get_dmarc_reports_from_mailbox( @@ -2079,7 +2296,7 @@ dns_retries (int): Number of times to retry DNS queries on timeout or other transient errors strip_attachment_payloads (bool): Remove attachment payloads from - forensic report results + failure report results results (dict): Results from the previous run batch_size (int): Number of messages to read and process before saving (use 0 for no limit) @@ -2090,7 +2307,7 @@ normalize_timespan_threshold_hours (float): Normalize timespans beyond this Returns: - dict: Lists of ``aggregate_reports``, ``forensic_reports``, and ``smtp_tls_reports`` + dict: Lists of ``aggregate_reports``, ``failure_reports``, and ``smtp_tls_reports`` """ if delete and test: raise ValueError("delete and test options are mutually exclusive") @@ -2102,25 +2319,26 @@ current_time: Optional[Union[datetime, date, str]] = None aggregate_reports: list[AggregateReport] = [] - forensic_reports: list[ForensicReport] = [] + failure_reports: list[FailureReport] = [] smtp_tls_reports: list[SMTPTLSReport] = [] aggregate_report_msg_uids = [] - forensic_report_msg_uids = [] + failure_report_msg_uids = [] smtp_tls_msg_uids = [] aggregate_reports_folder = "{0}/Aggregate".format(archive_folder) - forensic_reports_folder = "{0}/Forensic".format(archive_folder) + failure_reports_folder = "{0}/Failure".format(archive_folder) smtp_tls_reports_folder = "{0}/SMTP-TLS".format(archive_folder) invalid_reports_folder = "{0}/Invalid".format(archive_folder) if results: aggregate_reports = results["aggregate_reports"].copy() - forensic_reports = results["forensic_reports"].copy() + failure_reports = results["failure_reports"].copy() smtp_tls_reports = results["smtp_tls_reports"].copy() if not test and create_folders: + _migrate_forensic_archive_folder(connection, archive_folder) connection.create_folder(archive_folder) connection.create_folder(aggregate_reports_folder) - connection.create_folder(forensic_reports_folder) + connection.create_folder(failure_reports_folder) connection.create_folder(smtp_tls_reports_folder) connection.create_folder(invalid_reports_folder) @@ -2226,9 +2444,9 @@ f"Skipping duplicate aggregate report with ID: {report_id}" ) aggregate_report_msg_uids.append(message_id) - elif parsed_email["report_type"] == "forensic": - forensic_reports.append(parsed_email["report"]) - forensic_report_msg_uids.append(message_id) + elif parsed_email["report_type"] == "failure": + failure_reports.append(parsed_email["report"]) + failure_report_msg_uids.append(message_id) elif parsed_email["report_type"] == "smtp_tls": smtp_tls_reports.append(parsed_email["report"]) smtp_tls_msg_uids.append(message_id) @@ -2255,7 +2473,7 @@ if not test: if delete: processed_messages = ( - aggregate_report_msg_uids + forensic_report_msg_uids + smtp_tls_msg_uids + aggregate_report_msg_uids + failure_report_msg_uids + smtp_tls_msg_uids ) number_of_processed_msgs = len(processed_messages) @@ -2295,24 +2513,24 @@ message = "Error moving message UID" e = "{0} {1}: {2}".format(message, msg_uid, e) logger.error("Mailbox error: {0}".format(e)) - if len(forensic_report_msg_uids) > 0: - message = "Moving forensic report messages from" + if len(failure_report_msg_uids) > 0: + message = "Moving failure report messages from" logger.debug( "{0} {1} to {2}".format( - message, reports_folder, forensic_reports_folder + message, reports_folder, failure_reports_folder ) ) - number_of_forensic_msgs = len(forensic_report_msg_uids) - for i in range(number_of_forensic_msgs): - msg_uid = forensic_report_msg_uids[i] + number_of_failure_msgs = len(failure_report_msg_uids) + for i in range(number_of_failure_msgs): + msg_uid = failure_report_msg_uids[i] message = "Moving message" logger.debug( "{0} {1} of {2}: UID {3}".format( - message, i + 1, number_of_forensic_msgs, msg_uid + message, i + 1, number_of_failure_msgs, msg_uid ) ) try: - connection.move_message(msg_uid, forensic_reports_folder) + connection.move_message(msg_uid, failure_reports_folder) except Exception as e: e = "Error moving message UID {0}: {1}".format(msg_uid, e) logger.error("Mailbox error: {0}".format(e)) @@ -2339,7 +2557,7 @@ logger.error("Mailbox error: {0}".format(e)) results = { "aggregate_reports": aggregate_reports, - "forensic_reports": forensic_reports, + "failure_reports": failure_reports, "smtp_tls_reports": smtp_tls_reports, } @@ -2428,7 +2646,7 @@ dns_retries (int): Number of times to retry DNS queries on timeout or other transient errors strip_attachment_payloads (bool): Replace attachment payloads in - forensic report samples with None + failure report samples with None batch_size (int): Number of messages to read and process before saving since: Search for messages since certain time normalize_timespan_threshold_hours (float): Normalize timespans beyond this @@ -2470,34 +2688,50 @@ +
+[docs] def append_json( filename: str, reports: Union[ Sequence[AggregateReport], - Sequence[ForensicReport], + Sequence[FailureReport], Sequence[SMTPTLSReport], ], ) -> None: - with open(filename, "a+", newline="\n", encoding="utf-8") as output: - output_json = json.dumps(reports, ensure_ascii=False, indent=2) - if output.seek(0, os.SEEK_END) != 0: - if len(reports) == 0: - # not appending anything, don't do any dance to append it - # correctly - return - output.seek(output.tell() - 1) - last_char = output.read(1) - if last_char == "]": - # remove the trailing "\n]", leading "[\n", and replace with - # ",\n" - output.seek(output.tell() - 2) - output.write(",\n") - output_json = output_json[2:] - else: - output.seek(0) - output.truncate() + """Append ``reports`` to a JSON array on disk, creating the file + if needed. + + Reads the existing array (if the file exists and parses cleanly), + merges the new reports onto the end, and rewrites the file as a + single valid JSON array. An earlier version of this used an + ``open(..., "a+")`` + ``seek()`` + overwrite pattern, but Python's + documentation is explicit that on POSIX, ``a`` / ``a+`` writes + *always* go to EOF regardless of seek position — so the second + call onto an existing file produced ``[...],\\n[...]``-style + corrupted output. Read-merge-write is the only way to get a valid + JSON array out of repeated appends. + """ + if len(reports) == 0: + # Don't create an empty-array file for an empty input; if a + # file already exists, leave it alone. + return + + existing: list = [] + if os.path.isfile(filename) and os.path.getsize(filename) > 0: + try: + with open(filename, "r", encoding="utf-8") as f: + loaded = json.loads(f.read()) + if isinstance(loaded, list): + existing = loaded + except (json.JSONDecodeError, OSError): + # Corrupted or unreadable: overwrite cleanly rather than + # silently fail to record. + existing = [] + + merged = existing + list(reports) + with open(filename, "w", newline="\n", encoding="utf-8") as output: + json.dump(merged, output, ensure_ascii=False, indent=2)
- output.write(output_json) def append_csv(filename: str, csv: str) -> None: @@ -2519,10 +2753,10 @@ *, output_directory: str = "output", aggregate_json_filename: str = "aggregate.json", - forensic_json_filename: str = "forensic.json", + failure_json_filename: str = "failure.json", smtp_tls_json_filename: str = "smtp_tls.json", aggregate_csv_filename: str = "aggregate.csv", - forensic_csv_filename: str = "forensic.csv", + failure_csv_filename: str = "failure.csv", smtp_tls_csv_filename: str = "smtp_tls.csv", ): """ @@ -2532,15 +2766,15 @@ results: Parsing results output_directory (str): The path to the directory to save in aggregate_json_filename (str): Filename for the aggregate JSON file - forensic_json_filename (str): Filename for the forensic JSON file + failure_json_filename (str): Filename for the failure JSON file smtp_tls_json_filename (str): Filename for the SMTP TLS JSON file aggregate_csv_filename (str): Filename for the aggregate CSV file - forensic_csv_filename (str): Filename for the forensic CSV file + failure_csv_filename (str): Filename for the failure CSV file smtp_tls_csv_filename (str): Filename for the SMTP TLS CSV file """ aggregate_reports = results["aggregate_reports"] - forensic_reports = results["forensic_reports"] + failure_reports = results["failure_reports"] smtp_tls_reports = results["smtp_tls_reports"] output_directory = os.path.expanduser(output_directory) @@ -2559,13 +2793,11 @@ parsed_aggregate_reports_to_csv(aggregate_reports), ) - append_json( - os.path.join(output_directory, forensic_json_filename), forensic_reports - ) + append_json(os.path.join(output_directory, failure_json_filename), failure_reports) append_csv( - os.path.join(output_directory, forensic_csv_filename), - parsed_forensic_reports_to_csv(forensic_reports), + os.path.join(output_directory, failure_csv_filename), + parsed_failure_reports_to_csv(failure_reports), ) append_json( @@ -2582,10 +2814,10 @@ os.makedirs(samples_directory) sample_filenames = [] - for forensic_report in forensic_reports: - sample = forensic_report["sample"] + for failure_report in failure_reports: + sample = failure_report["sample"] message_count = 0 - parsed_sample = forensic_report["parsed_sample"] + parsed_sample = failure_report["parsed_sample"] subject = ( parsed_sample.get("filename_safe_subject") or parsed_sample.get("subject") @@ -2726,6 +2958,12 @@ plain_message=message, )
+ + +# Backward-compatible aliases +parse_forensic_report = parse_failure_report +parsed_forensic_reports_to_csv_rows = parsed_failure_reports_to_csv_rows +parsed_forensic_reports_to_csv = parsed_failure_reports_to_csv
diff --git a/_modules/parsedmarc/elastic.html b/_modules/parsedmarc/elastic.html index e294320..68eac08 100644 --- a/_modules/parsedmarc/elastic.html +++ b/_modules/parsedmarc/elastic.html @@ -5,14 +5,14 @@ - parsedmarc.elastic — parsedmarc 9.11.2 documentation + parsedmarc.elastic — parsedmarc 10.0.0 documentation - + @@ -95,6 +95,7 @@ InnerDoc, Integer, Ip, + Keyword, Nested, Object, Search, @@ -103,7 +104,7 @@ ) from elasticsearch_dsl.search import Q -from parsedmarc import InvalidForensicReport +from parsedmarc import InvalidFailureReport from parsedmarc.log import logger from parsedmarc.utils import human_timestamp_to_datetime @@ -115,6 +116,18 @@ +# Mirror of the ``serverless`` flag passed to ``set_hosts``; consulted by +# ``create_indexes`` to strip settings Elastic Cloud Serverless rejects. +# Module-level state is consistent with the existing ``connections.create_connection`` +# global the rest of this module relies on — there is a single default ES +# connection per process. +_SERVERLESS = False + +# Index settings rejected by Elastic Cloud Serverless with HTTP 400. Other +# settings (e.g. ``refresh_interval``) are accepted and pass through. +_SERVERLESS_REJECTED_SETTINGS = frozenset({"number_of_shards", "number_of_replicas"}) + + class _PolicyOverride(InnerDoc): type = Text() comment = Text() @@ -128,18 +141,23 @@ sp = Text() pct = Integer() fo = Text() + np = Keyword() + testing = Keyword() + discovery_method = Keyword() class _DKIMResult(InnerDoc): domain = Text() selector = Text() result = Text() + human_result = Text() class _SPFResult(InnerDoc): domain = Text() scope = Text() results = Text() + human_result = Text() class _AggregateReportDoc(Document): @@ -147,6 +165,7 @@ name = "dmarc_aggregate" xml_schema = Text() + xml_namespace = Keyword() org_name = Text() org_email = Text() org_extra_contact_info = Text() @@ -178,17 +197,45 @@ envelope_to = Text() dkim_results = Nested(_DKIMResult) spf_results = Nested(_SPFResult) + np = Keyword() + testing = Keyword() + discovery_method = Keyword() + generator = Text() def add_policy_override(self, type_: str, comment: str): self.policy_overrides.append(_PolicyOverride(type=type_, comment=comment)) # pyright: ignore[reportCallIssue] - def add_dkim_result(self, domain: str, selector: str, result: _DKIMResult): + def add_dkim_result( + self, + domain: str, + selector: str, + result: _DKIMResult, + human_result: str = None, + ): self.dkim_results.append( - _DKIMResult(domain=domain, selector=selector, result=result) + _DKIMResult( + domain=domain, + selector=selector, + result=result, + human_result=human_result, + ) ) # pyright: ignore[reportCallIssue] - def add_spf_result(self, domain: str, scope: str, result: _SPFResult): - self.spf_results.append(_SPFResult(domain=domain, scope=scope, result=result)) # pyright: ignore[reportCallIssue] + def add_spf_result( + self, + domain: str, + scope: str, + result: _SPFResult, + human_result: str = None, + ): + self.spf_results.append( + _SPFResult( + domain=domain, + scope=scope, + result=result, + human_result=human_result, + ) + ) # pyright: ignore[reportCallIssue] def save(self, **kwargs): # pyright: ignore[reportIncompatibleMethodOverride] self.passed_dmarc = False @@ -208,7 +255,7 @@ sha256 = Text() -class _ForensicSampleDoc(InnerDoc): +class _FailureSampleDoc(InnerDoc): raw = Text() headers = Object() headers_only = Boolean() @@ -245,9 +292,9 @@ ) # pyright: ignore[reportCallIssue] -class _ForensicReportDoc(Document): +class _FailureReportDoc(Document): class Index: - name = "dmarc_forensic" + name = "dmarc_failure" feedback_type = Text() user_agent = Text() @@ -268,7 +315,7 @@ source_auth_failures = Text() dkim_domain = Text() original_rcpt_to = Text() - sample = Object(_ForensicSampleDoc) + sample = Object(_FailureSampleDoc) class _SMTPTLSFailureDetailsDoc(InnerDoc): @@ -369,6 +416,7 @@ password: Optional[str] = None, api_key: Optional[str] = None, timeout: float = 60.0, + serverless: bool = False, ): """ Sets the Elasticsearch hosts to use @@ -382,7 +430,14 @@ password (str): The password to use for authentication api_key (str): The Base64 encoded API key to use for authentication timeout (float): Timeout in seconds + serverless (bool): Target an Elastic Cloud Serverless project. When True, + ``create_indexes`` strips ``number_of_shards`` / ``number_of_replicas`` + from its settings (which Serverless rejects with HTTP 400) and passes + any other settings through unchanged. """ + # Module-global; see the _SERVERLESS comment at the top of the module. + global _SERVERLESS + _SERVERLESS = serverless if not isinstance(hosts, list): hosts = [hosts] conn_params = {"hosts": hosts, "timeout": timeout} @@ -410,18 +465,28 @@ Args: names (list): A list of index names - settings (dict): Index settings - + settings (dict): Index settings. In Serverless mode, keys in + ``_SERVERLESS_REJECTED_SETTINGS`` are filtered out and the + remaining keys are passed through; defaults are skipped entirely. """ + if settings is None: + effective_settings: dict[str, Any] = ( + {} if _SERVERLESS else {"number_of_shards": 1, "number_of_replicas": 0} + ) + elif _SERVERLESS: + effective_settings = { + k: v for k, v in settings.items() if k not in _SERVERLESS_REJECTED_SETTINGS + } + else: + effective_settings = dict(settings) + for name in names: index = Index(name) try: if not index.exists(): logger.debug("Creating Elasticsearch index: {0}".format(name)) - if settings is None: - index.settings(number_of_shards=1, number_of_replicas=0) - else: - index.settings(**settings) + if effective_settings: + index.settings(**effective_settings) index.create() except Exception as e: raise ElasticsearchError("Elasticsearch error: {0}".format(e.__str__()))
@@ -432,20 +497,20 @@ [docs] def migrate_indexes( aggregate_indexes: Optional[list[str]] = None, - forensic_indexes: Optional[list[str]] = None, + failure_indexes: Optional[list[str]] = None, ): """ Updates index mappings Args: aggregate_indexes (list): A list of aggregate index names - forensic_indexes (list): A list of forensic index names + failure_indexes (list): A list of failure index names """ version = 2 if aggregate_indexes is None: aggregate_indexes = [] - if forensic_indexes is None: - forensic_indexes = [] + if failure_indexes is None: + failure_indexes = [] for aggregate_index_name in aggregate_indexes: if not Index(aggregate_index_name).exists(): continue @@ -475,7 +540,7 @@ reindex(connections.get_connection(), aggregate_index_name, new_index_name) # pyright: ignore[reportArgumentType] Index(aggregate_index_name).delete() - for forensic_index in forensic_indexes: + for failure_index in failure_indexes: pass @@ -494,7 +559,7 @@ Saves a parsed DMARC aggregate report to Elasticsearch Args: - aggregate_report (dict): A parsed forensic report + aggregate_report (dict): A parsed aggregate report index_suffix (str): The suffix of the name of the index to save to index_prefix (str): The prefix of the name of the index to save to monthly_indexes (bool): Use monthly indexes instead of daily indexes @@ -562,6 +627,9 @@ sp=aggregate_report["policy_published"]["sp"], pct=aggregate_report["policy_published"]["pct"], fo=aggregate_report["policy_published"]["fo"], + np=aggregate_report["policy_published"].get("np"), + testing=aggregate_report["policy_published"].get("testing"), + discovery_method=aggregate_report["policy_published"].get("discovery_method"), ) for record in aggregate_report["records"]: @@ -578,6 +646,7 @@ date_range = [aggregate_report["begin_date"], aggregate_report["end_date"]] agg_doc = _AggregateReportDoc( xml_schema=aggregate_report["xml_schema"], + xml_namespace=aggregate_report.get("xml_namespace"), org_name=metadata["org_name"], org_email=metadata["org_email"], org_extra_contact_info=metadata["org_extra_contact_info"], @@ -606,6 +675,12 @@ header_from=record["identifiers"]["header_from"], envelope_from=record["identifiers"]["envelope_from"], envelope_to=record["identifiers"]["envelope_to"], + np=aggregate_report["policy_published"].get("np"), + testing=aggregate_report["policy_published"].get("testing"), + discovery_method=aggregate_report["policy_published"].get( + "discovery_method" + ), + generator=metadata.get("generator"), ) for override in record["policy_evaluated"]["policy_override_reasons"]: @@ -618,6 +693,7 @@ domain=dkim_result["domain"], selector=dkim_result["selector"], result=dkim_result["result"], + human_result=dkim_result.get("human_result"), ) for spf_result in record["auth_results"]["spf"]: @@ -625,6 +701,7 @@ domain=spf_result["domain"], scope=spf_result["scope"], result=spf_result["result"], + human_result=spf_result.get("human_result"), ) index = "dmarc_aggregate" @@ -647,10 +724,10 @@ -
-[docs] -def save_forensic_report_to_elasticsearch( - forensic_report: dict[str, Any], +
+[docs] +def save_failure_report_to_elasticsearch( + failure_report: dict[str, Any], index_suffix: Optional[Any] = None, index_prefix: Optional[str] = None, monthly_indexes: Optional[bool] = False, @@ -658,10 +735,10 @@ number_of_replicas: int = 0, ): """ - Saves a parsed DMARC forensic report to Elasticsearch + Saves a parsed DMARC failure report to Elasticsearch Args: - forensic_report (dict): A parsed forensic report + failure_report (dict): A parsed failure report index_suffix (str): The suffix of the name of the index to save to index_prefix (str): The prefix of the name of the index to save to monthly_indexes (bool): Use monthly indexes instead of daily @@ -674,26 +751,28 @@ AlreadySaved """ - logger.info("Saving forensic report to Elasticsearch") - forensic_report = forensic_report.copy() + logger.info("Saving failure report to Elasticsearch") + failure_report = failure_report.copy() sample_date = None - if forensic_report["parsed_sample"]["date"] is not None: - sample_date = forensic_report["parsed_sample"]["date"] + if failure_report["parsed_sample"]["date"] is not None: + sample_date = failure_report["parsed_sample"]["date"] sample_date = human_timestamp_to_datetime(sample_date) - original_headers = forensic_report["parsed_sample"]["headers"] + original_headers = failure_report["parsed_sample"]["headers"] headers: dict[str, Any] = {} for original_header in original_headers: headers[original_header.lower()] = original_headers[original_header] - arrival_date = human_timestamp_to_datetime(forensic_report["arrival_date_utc"]) + arrival_date = human_timestamp_to_datetime(failure_report["arrival_date_utc"]) arrival_date_epoch_milliseconds = int(arrival_date.timestamp() * 1000) if index_suffix is not None: - search_index = "dmarc_forensic_{0}*".format(index_suffix) + search_index = "dmarc_failure_{0}*,dmarc_forensic_{0}*".format(index_suffix) else: - search_index = "dmarc_forensic*" + search_index = "dmarc_failure*,dmarc_forensic*" if index_prefix is not None: - search_index = "{0}{1}".format(index_prefix, search_index) + search_index = ",".join( + "{0}{1}".format(index_prefix, part) for part in search_index.split(",") + ) search = Search(index=search_index) q = Q(dict(match=dict(arrival_date=arrival_date_epoch_milliseconds))) # pyright: ignore[reportArgumentType] @@ -734,67 +813,67 @@ if len(existing) > 0: raise AlreadySaved( - "A forensic sample to {0} from {1} " + "A failure sample to {0} from {1} " "with a subject of {2} and arrival date of {3} " "already exists in " "Elasticsearch".format( - to_, from_, subject, forensic_report["arrival_date_utc"] + to_, from_, subject, failure_report["arrival_date_utc"] ) ) - parsed_sample = forensic_report["parsed_sample"] - sample = _ForensicSampleDoc( - raw=forensic_report["sample"], + parsed_sample = failure_report["parsed_sample"] + sample = _FailureSampleDoc( + raw=failure_report["sample"], headers=headers, - headers_only=forensic_report["sample_headers_only"], + headers_only=failure_report["sample_headers_only"], date=sample_date, - subject=forensic_report["parsed_sample"]["subject"], + subject=failure_report["parsed_sample"]["subject"], filename_safe_subject=parsed_sample["filename_safe_subject"], - body=forensic_report["parsed_sample"]["body"], + body=failure_report["parsed_sample"]["body"], ) - for address in forensic_report["parsed_sample"]["to"]: + for address in failure_report["parsed_sample"]["to"]: sample.add_to(display_name=address["display_name"], address=address["address"]) - for address in forensic_report["parsed_sample"]["reply_to"]: + for address in failure_report["parsed_sample"]["reply_to"]: sample.add_reply_to( display_name=address["display_name"], address=address["address"] ) - for address in forensic_report["parsed_sample"]["cc"]: + for address in failure_report["parsed_sample"]["cc"]: sample.add_cc(display_name=address["display_name"], address=address["address"]) - for address in forensic_report["parsed_sample"]["bcc"]: + for address in failure_report["parsed_sample"]["bcc"]: sample.add_bcc(display_name=address["display_name"], address=address["address"]) - for attachment in forensic_report["parsed_sample"]["attachments"]: + for attachment in failure_report["parsed_sample"]["attachments"]: sample.add_attachment( filename=attachment["filename"], content_type=attachment["mail_content_type"], sha256=attachment["sha256"], ) try: - forensic_doc = _ForensicReportDoc( - feedback_type=forensic_report["feedback_type"], - user_agent=forensic_report["user_agent"], - version=forensic_report["version"], - original_mail_from=forensic_report["original_mail_from"], + failure_doc = _FailureReportDoc( + feedback_type=failure_report["feedback_type"], + user_agent=failure_report["user_agent"], + version=failure_report["version"], + original_mail_from=failure_report["original_mail_from"], arrival_date=arrival_date_epoch_milliseconds, - domain=forensic_report["reported_domain"], - original_envelope_id=forensic_report["original_envelope_id"], - authentication_results=forensic_report["authentication_results"], - delivery_results=forensic_report["delivery_result"], - source_ip_address=forensic_report["source"]["ip_address"], - source_country=forensic_report["source"]["country"], - source_reverse_dns=forensic_report["source"]["reverse_dns"], - source_base_domain=forensic_report["source"]["base_domain"], - source_asn=forensic_report["source"]["asn"], - source_as_name=forensic_report["source"]["as_name"], - source_as_domain=forensic_report["source"]["as_domain"], - authentication_mechanisms=forensic_report["authentication_mechanisms"], - auth_failure=forensic_report["auth_failure"], - dkim_domain=forensic_report["dkim_domain"], - original_rcpt_to=forensic_report["original_rcpt_to"], + domain=failure_report["reported_domain"], + original_envelope_id=failure_report["original_envelope_id"], + authentication_results=failure_report["authentication_results"], + delivery_results=failure_report["delivery_result"], + source_ip_address=failure_report["source"]["ip_address"], + source_country=failure_report["source"]["country"], + source_reverse_dns=failure_report["source"]["reverse_dns"], + source_base_domain=failure_report["source"]["base_domain"], + source_asn=failure_report["source"]["asn"], + source_as_name=failure_report["source"]["as_name"], + source_as_domain=failure_report["source"]["as_domain"], + authentication_mechanisms=failure_report["authentication_mechanisms"], + auth_failure=failure_report["auth_failure"], + dkim_domain=failure_report["dkim_domain"], + original_rcpt_to=failure_report["original_rcpt_to"], sample=sample, ) - index = "dmarc_forensic" + index = "dmarc_failure" if index_suffix: index = "{0}_{1}".format(index, index_suffix) if index_prefix: @@ -808,14 +887,14 @@ number_of_shards=number_of_shards, number_of_replicas=number_of_replicas ) create_indexes([index], index_settings) - forensic_doc.meta.index = index # pyright: ignore[reportAttributeAccessIssue, reportOptionalMemberAccess] + failure_doc.meta.index = index # pyright: ignore[reportAttributeAccessIssue, reportOptionalMemberAccess] try: - forensic_doc.save() + failure_doc.save() except Exception as e: raise ElasticsearchError("Elasticsearch error: {0}".format(e.__str__())) except KeyError as e: - raise InvalidForensicReport( - "Forensic report missing required field: {0}".format(e.__str__()) + raise InvalidFailureReport( + "Failure report missing required field: {0}".format(e.__str__()) )
@@ -973,6 +1052,12 @@ except Exception as e: raise ElasticsearchError("Elasticsearch error: {0}".format(e.__str__()))
+ + +# Backward-compatible aliases +_ForensicSampleDoc = _FailureSampleDoc +_ForensicReportDoc = _FailureReportDoc +save_forensic_report_to_elasticsearch = save_failure_report_to_elasticsearch diff --git a/_modules/parsedmarc/opensearch.html b/_modules/parsedmarc/opensearch.html index 0078c3e..320d5a4 100644 --- a/_modules/parsedmarc/opensearch.html +++ b/_modules/parsedmarc/opensearch.html @@ -5,14 +5,14 @@ - parsedmarc.opensearch — parsedmarc 9.11.2 documentation + parsedmarc.opensearch — parsedmarc 10.0.0 documentation - + @@ -96,6 +96,7 @@ InnerDoc, Integer, Ip, + Keyword, Nested, Object, Q, @@ -106,7 +107,7 @@ ) from opensearchpy.helpers import reindex -from parsedmarc import InvalidForensicReport +from parsedmarc import InvalidFailureReport from parsedmarc.log import logger from parsedmarc.utils import human_timestamp_to_datetime @@ -131,18 +132,23 @@ sp = Text() pct = Integer() fo = Text() + np = Keyword() + testing = Keyword() + discovery_method = Keyword() class _DKIMResult(InnerDoc): domain = Text() selector = Text() result = Text() + human_result = Text() class _SPFResult(InnerDoc): domain = Text() scope = Text() results = Text() + human_result = Text() class _AggregateReportDoc(Document): @@ -150,6 +156,7 @@ name = "dmarc_aggregate" xml_schema = Text() + xml_namespace = Keyword() org_name = Text() org_email = Text() org_extra_contact_info = Text() @@ -181,17 +188,45 @@ envelope_to = Text() dkim_results = Nested(_DKIMResult) spf_results = Nested(_SPFResult) + np = Keyword() + testing = Keyword() + discovery_method = Keyword() + generator = Text() def add_policy_override(self, type_: str, comment: str): self.policy_overrides.append(_PolicyOverride(type=type_, comment=comment)) - def add_dkim_result(self, domain: str, selector: str, result: _DKIMResult): + def add_dkim_result( + self, + domain: str, + selector: str, + result: _DKIMResult, + human_result: str = None, + ): self.dkim_results.append( - _DKIMResult(domain=domain, selector=selector, result=result) + _DKIMResult( + domain=domain, + selector=selector, + result=result, + human_result=human_result, + ) ) - def add_spf_result(self, domain: str, scope: str, result: _SPFResult): - self.spf_results.append(_SPFResult(domain=domain, scope=scope, result=result)) + def add_spf_result( + self, + domain: str, + scope: str, + result: _SPFResult, + human_result: str = None, + ): + self.spf_results.append( + _SPFResult( + domain=domain, + scope=scope, + result=result, + human_result=human_result, + ) + ) def save(self, **kwargs): # pyright: ignore[reportIncompatibleMethodOverride] self.passed_dmarc = False @@ -211,7 +246,7 @@ sha256 = Text() -class _ForensicSampleDoc(InnerDoc): +class _FailureSampleDoc(InnerDoc): raw = Text() headers = Object() headers_only = Boolean() @@ -248,9 +283,9 @@ ) -class _ForensicReportDoc(Document): +class _FailureReportDoc(Document): class Index: - name = "dmarc_forensic" + name = "dmarc_failure" feedback_type = Text() user_agent = Text() @@ -271,7 +306,7 @@ source_auth_failures = Text() dkim_domain = Text() original_rcpt_to = Text() - sample = Object(_ForensicSampleDoc) + sample = Object(_FailureSampleDoc) class _SMTPTLSFailureDetailsDoc(InnerDoc): @@ -462,20 +497,20 @@ [docs] def migrate_indexes( aggregate_indexes: Optional[list[str]] = None, - forensic_indexes: Optional[list[str]] = None, + failure_indexes: Optional[list[str]] = None, ): """ Updates index mappings Args: aggregate_indexes (list): A list of aggregate index names - forensic_indexes (list): A list of forensic index names + failure_indexes (list): A list of failure index names """ version = 2 if aggregate_indexes is None: aggregate_indexes = [] - if forensic_indexes is None: - forensic_indexes = [] + if failure_indexes is None: + failure_indexes = [] for aggregate_index_name in aggregate_indexes: if not Index(aggregate_index_name).exists(): continue @@ -505,7 +540,7 @@ reindex(connections.get_connection(), aggregate_index_name, new_index_name) Index(aggregate_index_name).delete() - for forensic_index in forensic_indexes: + for failure_index in failure_indexes: pass @@ -524,7 +559,7 @@ Saves a parsed DMARC aggregate report to OpenSearch Args: - aggregate_report (dict): A parsed forensic report + aggregate_report (dict): A parsed aggregate report index_suffix (str): The suffix of the name of the index to save to index_prefix (str): The prefix of the name of the index to save to monthly_indexes (bool): Use monthly indexes instead of daily indexes @@ -592,6 +627,9 @@ sp=aggregate_report["policy_published"]["sp"], pct=aggregate_report["policy_published"]["pct"], fo=aggregate_report["policy_published"]["fo"], + np=aggregate_report["policy_published"].get("np"), + testing=aggregate_report["policy_published"].get("testing"), + discovery_method=aggregate_report["policy_published"].get("discovery_method"), ) for record in aggregate_report["records"]: @@ -608,6 +646,7 @@ date_range = [aggregate_report["begin_date"], aggregate_report["end_date"]] agg_doc = _AggregateReportDoc( xml_schema=aggregate_report["xml_schema"], + xml_namespace=aggregate_report.get("xml_namespace"), org_name=metadata["org_name"], org_email=metadata["org_email"], org_extra_contact_info=metadata["org_extra_contact_info"], @@ -636,6 +675,12 @@ header_from=record["identifiers"]["header_from"], envelope_from=record["identifiers"]["envelope_from"], envelope_to=record["identifiers"]["envelope_to"], + np=aggregate_report["policy_published"].get("np"), + testing=aggregate_report["policy_published"].get("testing"), + discovery_method=aggregate_report["policy_published"].get( + "discovery_method" + ), + generator=metadata.get("generator"), ) for override in record["policy_evaluated"]["policy_override_reasons"]: @@ -648,6 +693,7 @@ domain=dkim_result["domain"], selector=dkim_result["selector"], result=dkim_result["result"], + human_result=dkim_result.get("human_result"), ) for spf_result in record["auth_results"]["spf"]: @@ -655,6 +701,7 @@ domain=spf_result["domain"], scope=spf_result["scope"], result=spf_result["result"], + human_result=spf_result.get("human_result"), ) index = "dmarc_aggregate" @@ -677,10 +724,10 @@ -
-[docs] -def save_forensic_report_to_opensearch( - forensic_report: dict[str, Any], +
+[docs] +def save_failure_report_to_opensearch( + failure_report: dict[str, Any], index_suffix: Optional[str] = None, index_prefix: Optional[str] = None, monthly_indexes: bool = False, @@ -688,10 +735,10 @@ number_of_replicas: int = 0, ): """ - Saves a parsed DMARC forensic report to OpenSearch + Saves a parsed DMARC failure report to OpenSearch Args: - forensic_report (dict): A parsed forensic report + failure_report (dict): A parsed failure report index_suffix (str): The suffix of the name of the index to save to index_prefix (str): The prefix of the name of the index to save to monthly_indexes (bool): Use monthly indexes instead of daily @@ -704,26 +751,28 @@ AlreadySaved """ - logger.info("Saving forensic report to OpenSearch") - forensic_report = forensic_report.copy() + logger.info("Saving failure report to OpenSearch") + failure_report = failure_report.copy() sample_date = None - if forensic_report["parsed_sample"]["date"] is not None: - sample_date = forensic_report["parsed_sample"]["date"] + if failure_report["parsed_sample"]["date"] is not None: + sample_date = failure_report["parsed_sample"]["date"] sample_date = human_timestamp_to_datetime(sample_date) - original_headers = forensic_report["parsed_sample"]["headers"] + original_headers = failure_report["parsed_sample"]["headers"] headers: dict[str, Any] = {} for original_header in original_headers: headers[original_header.lower()] = original_headers[original_header] - arrival_date = human_timestamp_to_datetime(forensic_report["arrival_date_utc"]) + arrival_date = human_timestamp_to_datetime(failure_report["arrival_date_utc"]) arrival_date_epoch_milliseconds = int(arrival_date.timestamp() * 1000) if index_suffix is not None: - search_index = "dmarc_forensic_{0}*".format(index_suffix) + search_index = "dmarc_failure_{0}*,dmarc_forensic_{0}*".format(index_suffix) else: - search_index = "dmarc_forensic*" + search_index = "dmarc_failure*,dmarc_forensic*" if index_prefix is not None: - search_index = "{0}{1}".format(index_prefix, search_index) + search_index = ",".join( + "{0}{1}".format(index_prefix, part) for part in search_index.split(",") + ) search = Search(index=search_index) q = Q(dict(match=dict(arrival_date=arrival_date_epoch_milliseconds))) @@ -764,67 +813,65 @@ if len(existing) > 0: raise AlreadySaved( - "A forensic sample to {0} from {1} " + "A failure sample to {0} from {1} " "with a subject of {2} and arrival date of {3} " "already exists in " - "OpenSearch".format( - to_, from_, subject, forensic_report["arrival_date_utc"] - ) + "OpenSearch".format(to_, from_, subject, failure_report["arrival_date_utc"]) ) - parsed_sample = forensic_report["parsed_sample"] - sample = _ForensicSampleDoc( - raw=forensic_report["sample"], + parsed_sample = failure_report["parsed_sample"] + sample = _FailureSampleDoc( + raw=failure_report["sample"], headers=headers, - headers_only=forensic_report["sample_headers_only"], + headers_only=failure_report["sample_headers_only"], date=sample_date, - subject=forensic_report["parsed_sample"]["subject"], + subject=failure_report["parsed_sample"]["subject"], filename_safe_subject=parsed_sample["filename_safe_subject"], - body=forensic_report["parsed_sample"]["body"], + body=failure_report["parsed_sample"]["body"], ) - for address in forensic_report["parsed_sample"]["to"]: + for address in failure_report["parsed_sample"]["to"]: sample.add_to(display_name=address["display_name"], address=address["address"]) - for address in forensic_report["parsed_sample"]["reply_to"]: + for address in failure_report["parsed_sample"]["reply_to"]: sample.add_reply_to( display_name=address["display_name"], address=address["address"] ) - for address in forensic_report["parsed_sample"]["cc"]: + for address in failure_report["parsed_sample"]["cc"]: sample.add_cc(display_name=address["display_name"], address=address["address"]) - for address in forensic_report["parsed_sample"]["bcc"]: + for address in failure_report["parsed_sample"]["bcc"]: sample.add_bcc(display_name=address["display_name"], address=address["address"]) - for attachment in forensic_report["parsed_sample"]["attachments"]: + for attachment in failure_report["parsed_sample"]["attachments"]: sample.add_attachment( filename=attachment["filename"], content_type=attachment["mail_content_type"], sha256=attachment["sha256"], ) try: - forensic_doc = _ForensicReportDoc( - feedback_type=forensic_report["feedback_type"], - user_agent=forensic_report["user_agent"], - version=forensic_report["version"], - original_mail_from=forensic_report["original_mail_from"], + failure_doc = _FailureReportDoc( + feedback_type=failure_report["feedback_type"], + user_agent=failure_report["user_agent"], + version=failure_report["version"], + original_mail_from=failure_report["original_mail_from"], arrival_date=arrival_date_epoch_milliseconds, - domain=forensic_report["reported_domain"], - original_envelope_id=forensic_report["original_envelope_id"], - authentication_results=forensic_report["authentication_results"], - delivery_results=forensic_report["delivery_result"], - source_ip_address=forensic_report["source"]["ip_address"], - source_country=forensic_report["source"]["country"], - source_reverse_dns=forensic_report["source"]["reverse_dns"], - source_base_domain=forensic_report["source"]["base_domain"], - source_asn=forensic_report["source"]["asn"], - source_as_name=forensic_report["source"]["as_name"], - source_as_domain=forensic_report["source"]["as_domain"], - authentication_mechanisms=forensic_report["authentication_mechanisms"], - auth_failure=forensic_report["auth_failure"], - dkim_domain=forensic_report["dkim_domain"], - original_rcpt_to=forensic_report["original_rcpt_to"], + domain=failure_report["reported_domain"], + original_envelope_id=failure_report["original_envelope_id"], + authentication_results=failure_report["authentication_results"], + delivery_results=failure_report["delivery_result"], + source_ip_address=failure_report["source"]["ip_address"], + source_country=failure_report["source"]["country"], + source_reverse_dns=failure_report["source"]["reverse_dns"], + source_base_domain=failure_report["source"]["base_domain"], + source_asn=failure_report["source"]["asn"], + source_as_name=failure_report["source"]["as_name"], + source_as_domain=failure_report["source"]["as_domain"], + authentication_mechanisms=failure_report["authentication_mechanisms"], + auth_failure=failure_report["auth_failure"], + dkim_domain=failure_report["dkim_domain"], + original_rcpt_to=failure_report["original_rcpt_to"], sample=sample, ) - index = "dmarc_forensic" + index = "dmarc_failure" if index_suffix: index = "{0}_{1}".format(index, index_suffix) if index_prefix: @@ -838,14 +885,14 @@ number_of_shards=number_of_shards, number_of_replicas=number_of_replicas ) create_indexes([index], index_settings) - forensic_doc.meta.index = index + failure_doc.meta.index = index try: - forensic_doc.save() + failure_doc.save() except Exception as e: raise OpenSearchError("OpenSearch error: {0}".format(e.__str__())) except KeyError as e: - raise InvalidForensicReport( - "Forensic report missing required field: {0}".format(e.__str__()) + raise InvalidFailureReport( + "Failure report missing required field: {0}".format(e.__str__()) )
@@ -1003,6 +1050,12 @@ except Exception as e: raise OpenSearchError("OpenSearch error: {0}".format(e.__str__()))
+ + +# Backward-compatible aliases +_ForensicSampleDoc = _FailureSampleDoc +_ForensicReportDoc = _FailureReportDoc +save_forensic_report_to_opensearch = save_failure_report_to_opensearch diff --git a/_modules/parsedmarc/splunk.html b/_modules/parsedmarc/splunk.html index 5fdd7ad..9646389 100644 --- a/_modules/parsedmarc/splunk.html +++ b/_modules/parsedmarc/splunk.html @@ -5,14 +5,14 @@ - parsedmarc.splunk — parsedmarc 9.11.2 documentation + parsedmarc.splunk — parsedmarc 10.0.0 documentation - + @@ -229,30 +229,30 @@ raise SplunkError(response["text"]) -
-[docs] - def save_forensic_reports_to_splunk( +
+[docs] + def save_failure_reports_to_splunk( self, - forensic_reports: Union[list[dict[str, Any]], dict[str, Any]], + failure_reports: Union[list[dict[str, Any]], dict[str, Any]], ): """ - Saves forensic DMARC reports to Splunk + Saves failure DMARC reports to Splunk Args: - forensic_reports (list): A list of forensic report dictionaries + failure_reports (list): A list of failure report dictionaries to save in Splunk """ - logger.debug("Saving forensic reports to Splunk") - if isinstance(forensic_reports, dict): - forensic_reports = [forensic_reports] + logger.debug("Saving failure reports to Splunk") + if isinstance(failure_reports, dict): + failure_reports = [failure_reports] - if len(forensic_reports) < 1: + if len(failure_reports) < 1: return json_str = "" - for report in forensic_reports: + for report in failure_reports: data = self._common_data.copy() - data["sourcetype"] = "dmarc:forensic" + data["sourcetype"] = "dmarc:failure" timestamp = human_timestamp_to_unix_timestamp(report["arrival_date_utc"]) data["time"] = timestamp data["event"] = report.copy() @@ -320,6 +320,10 @@ self.session.close()
+ + +# Backward-compatible aliases +HECClient.save_forensic_reports_to_splunk = HECClient.save_failure_reports_to_splunk diff --git a/_modules/parsedmarc/types.html b/_modules/parsedmarc/types.html index 4706101..da80e7f 100644 --- a/_modules/parsedmarc/types.html +++ b/_modules/parsedmarc/types.html @@ -5,14 +5,14 @@ - parsedmarc.types — parsedmarc 9.11.2 documentation + parsedmarc.types — parsedmarc 10.0.0 documentation - + @@ -90,7 +90,7 @@ # For optional keys, use total=False TypedDicts. -ReportType = Literal["aggregate", "forensic", "smtp_tls"] +ReportType = Literal["aggregate", "failure", "smtp_tls"]
@@ -104,7 +104,8 @@ end_date: str timespan_requires_normalization: bool original_timespan_seconds: int - errors: List[str]
+ errors: List[str] + generator: Optional[str] @@ -116,8 +117,11 @@ aspf: str p: str sp: str - pct: str - fo: str + pct: Optional[str] + fo: Optional[str] + np: Optional[str] + testing: Optional[str] + discovery_method: Optional[str] @@ -167,7 +171,8 @@ class AggregateAuthResultDKIM(TypedDict): domain: str result: str - selector: str + selector: str + human_result: Optional[str] @@ -176,7 +181,8 @@ class AggregateAuthResultSPF(TypedDict): domain: str result: str - scope: str + scope: str + human_result: Optional[str] @@ -217,6 +223,7 @@ [docs] class AggregateReport(TypedDict): xml_schema: str + xml_namespace: Optional[str] report_metadata: AggregateReportMetadata policy_published: AggregatePolicyPublished records: List[AggregateRecord] @@ -246,7 +253,7 @@ "ParsedEmail", { # This is a lightly-specified version of mailsuite/mailparser JSON. - # It focuses on the fields parsedmarc uses in forensic handling. + # It focuses on the fields parsedmarc uses in failure report handling. "headers": Dict[str, Any], "subject": Optional[str], "filename_safe_subject": Optional[str], @@ -265,9 +272,9 @@ ) -
-[docs] -class ForensicReport(TypedDict): +
+[docs] +class FailureReport(TypedDict): feedback_type: Optional[str] user_agent: Optional[str] version: Optional[str] @@ -289,6 +296,10 @@ +# Backward-compatible alias +ForensicReport = FailureReport + +
[docs] class SMTPTLSFailureDetails(TypedDict): @@ -349,14 +360,18 @@ -
-[docs] -class ForensicParsedReport(TypedDict): - report_type: Literal["forensic"] - report: ForensicReport
+
+[docs] +class FailureParsedReport(TypedDict): + report_type: Literal["failure"] + report: FailureReport
+# Backward-compatible alias +ForensicParsedReport = FailureParsedReport + +
[docs] class SMTPTLSParsedReport(TypedDict): @@ -365,14 +380,14 @@ -ParsedReport = Union[AggregateParsedReport, ForensicParsedReport, SMTPTLSParsedReport] +ParsedReport = Union[AggregateParsedReport, FailureParsedReport, SMTPTLSParsedReport]
[docs] class ParsingResults(TypedDict): aggregate_reports: List[AggregateReport] - forensic_reports: List[ForensicReport] + failure_reports: List[FailureReport] smtp_tls_reports: List[SMTPTLSReport]
diff --git a/_modules/parsedmarc/utils.html b/_modules/parsedmarc/utils.html index a260161..4b66bcb 100644 --- a/_modules/parsedmarc/utils.html +++ b/_modules/parsedmarc/utils.html @@ -5,14 +5,14 @@ - parsedmarc.utils — parsedmarc 9.11.2 documentation + parsedmarc.utils — parsedmarc 10.0.0 documentation - + @@ -139,7 +139,7 @@ parenthesis_regex = re.compile(r"\s*\(.*\)\s*") -null_file = open(os.devnull, "w") +null_file = subprocess.DEVNULL mailparser_logger = logging.getLogger("mailparser") mailparser_logger.setLevel(logging.CRITICAL) psl = publicsuffixlist.PublicSuffixList() diff --git a/_sources/elasticsearch.md.txt b/_sources/elasticsearch.md.txt index a47a248..c437527 100644 --- a/_sources/elasticsearch.md.txt +++ b/_sources/elasticsearch.md.txt @@ -125,7 +125,7 @@ server.ssl.key: /etc/kibana/kibana.key ``` :::{note} -For more security, you can configure Kibana to use a local network connexion +For more security, you can configure Kibana to use a local network connection to elasticsearch : ```text elasticsearch.hosts: ['https://SERVER_IP:9200'] @@ -214,7 +214,7 @@ Kibana index patterns with versions that match the upgraded indexes: 1. Login in to Kibana, and click on Management 2. Under Kibana, click on Saved Objects -3. Check the checkboxes for the `dmarc_aggregate` and `dmarc_forensic` +3. Check the checkboxes for the `dmarc_aggregate` and `dmarc_failure` index patterns 4. Click Delete 5. Click Delete on the conformation message diff --git a/_sources/index.md.txt b/_sources/index.md.txt index 3d86fef..c95b740 100644 --- a/_sources/index.md.txt +++ b/_sources/index.md.txt @@ -29,35 +29,40 @@ and Valimail. ## Features -- Parses draft and 1.0 standard aggregate/rua DMARC reports -- Parses forensic/failure/ruf DMARC reports -- Parses reports from SMTP TLS Reporting +- Parses aggregate/rua DMARC reports: the legacy draft and 1.0 schemas + (RFC 7489) and the new RFC 9990 schema for the final DMARC standard + (RFC 9989) +- Parses failure/ruf DMARC reports (RFC 6591 and RFC 9991; formerly called + forensic reports) +- Parses reports from SMTP TLS Reporting (TLS-RPT, RFC 8460) - Can parse reports from an inbox over IMAP, Microsoft Graph, or Gmail API - Transparently handles gzip or zip compressed reports - Consistent data structures - Simple JSON and/or CSV output - Optionally email the results -- Optionally send the results to Elasticsearch, Opensearch, and/or Splunk, for use - with premade dashboards -- Optionally send reports to Apache Kafka +- Optionally send the results to Elasticsearch, OpenSearch, Splunk, or + PostgreSQL, for use with premade dashboards +- Optionally send the results to Apache Kafka, Amazon S3, Azure Log + Analytics (Microsoft Sentinel), a Graylog (GELF) endpoint, a syslog server, + or an HTTP webhook ## Python Compatibility This project supports the following Python versions, which are either actively maintained or are the default versions for RHEL or Debian. -| Version | Supported | Reason | -|---------|-----------|------------------------------------------------------------| -| < 3.6 | ❌ | End of Life (EOL) | -| 3.6 | ❌ | Used in RHEL 8, but not supported by project dependencies | -| 3.7 | ❌ | End of Life (EOL) | -| 3.8 | ❌ | End of Life (EOL) | -| 3.9 | ❌ | Used in Debian 11 and RHEL 9, but not supported by project dependencies | -| 3.10 | ✅ | Actively maintained | -| 3.11 | ✅ | Actively maintained; supported until June 2028 (Debian 12) | -| 3.12 | ✅ | Actively maintained; supported until May 2035 (RHEL 10) | -| 3.13 | ✅ | Actively maintained; supported until June 2030 (Debian 13) | -| 3.14 | ✅ | Supported (requires `imapclient>=3.1.0`) | +| Version | Supported | Reason | +| --- | --- | --- | +| < 3.6 | ❌ | End of Life (EOL) | +| 3.6 | ❌ | Used in RHEL 8, but not supported by project dependencies | +| 3.7 | ❌ | End of Life (EOL) | +| 3.8 | ❌ | End of Life (EOL) | +| 3.9 | ❌ | Used in Debian 11 and RHEL 9, but not supported by project dependencies | +| 3.10 | ✅ | Actively maintained | +| 3.11 | ✅ | Actively maintained; supported until June 2028 (Debian 12) | +| 3.12 | ✅ | Actively maintained; supported until May 2035 (RHEL 10) | +| 3.13 | ✅ | Actively maintained; supported until June 2030 (Debian 13) | +| 3.14 | ✅ | Supported (requires `imapclient>=3.1.0`) | ```{toctree} :caption: 'Contents' diff --git a/_sources/kibana.md.txt b/_sources/kibana.md.txt index 50f781b..04d929d 100644 --- a/_sources/kibana.md.txt +++ b/_sources/kibana.md.txt @@ -89,7 +89,7 @@ information on DMARC failure reports (also known as forensic or ruf reports). These reports contain samples of emails that have failed to pass DMARC. :::{note} -Most recipients do not send forensic/failure/ruf reports at all to avoid +Most recipients do not send failure/ruf reports at all to avoid privacy leaks. Some recipients (notably Chinese webmail services) will only supply the headers of sample emails. Very few provide the entire email. ::: diff --git a/_sources/output.md.txt b/_sources/output.md.txt index 095193b..6a6b2bb 100644 --- a/_sources/output.md.txt +++ b/_sources/output.md.txt @@ -99,12 +99,12 @@ draft,acme.com,noreply-dmarc-support@acme.com,http://acme.com/dmarc/support,9391 ``` -## Sample forensic report output +## Sample failure report output Thanks to GitHub user [xennn](https://github.com/xennn) for the anonymized -[forensic report email sample](). +[failure report email sample](). -### JSON forensic report +### JSON failure report ```json { @@ -198,7 +198,7 @@ Thanks to GitHub user [xennn](https://github.com/xennn) for the anonymized } ``` -### CSV forensic report +### CSV failure report ```text feedback_type,user_agent,version,original_envelope_id,original_mail_from,original_rcpt_to,arrival_date,arrival_date_utc,subject,message_id,authentication_results,dkim_domain,source_ip_address,source_country,source_reverse_dns,source_base_domain,source_name,source_type,source_asn,source_as_name,source_as_domain,delivery_result,auth_failure,reported_domain,authentication_mechanisms,sample_headers_only diff --git a/_sources/splunk.md.txt b/_sources/splunk.md.txt index f21d24b..c884ef8 100644 --- a/_sources/splunk.md.txt +++ b/_sources/splunk.md.txt @@ -1,10 +1,10 @@ # Splunk Starting in version 4.3.0 `parsedmarc` supports sending aggregate and/or -forensic DMARC data to a Splunk [HTTP Event collector (HEC)]. +failure DMARC data to a Splunk [HTTP Event collector (HEC)]. The project repository contains [XML files] for premade Splunk -dashboards for aggregate and forensic DMARC reports. +dashboards for aggregate and failure DMARC reports. Copy and paste the contents of each file into a separate Splunk dashboard XML editor. diff --git a/_sources/usage.md.txt b/_sources/usage.md.txt index b6a2bed..eb4ffd4 100644 --- a/_sources/usage.md.txt +++ b/_sources/usage.md.txt @@ -4,9 +4,9 @@ ```text usage: parsedmarc [-h] [-c CONFIG_FILE] [--strip-attachment-payloads] [-o OUTPUT] - [--aggregate-json-filename AGGREGATE_JSON_FILENAME] [--forensic-json-filename FORENSIC_JSON_FILENAME] + [--aggregate-json-filename AGGREGATE_JSON_FILENAME] [--failure-json-filename FAILURE_JSON_FILENAME] [--smtp-tls-json-filename SMTP_TLS_JSON_FILENAME] [--aggregate-csv-filename AGGREGATE_CSV_FILENAME] - [--forensic-csv-filename FORENSIC_CSV_FILENAME] [--smtp-tls-csv-filename SMTP_TLS_CSV_FILENAME] + [--failure-csv-filename FAILURE_CSV_FILENAME] [--smtp-tls-csv-filename SMTP_TLS_CSV_FILENAME] [-n NAMESERVERS [NAMESERVERS ...]] [-t DNS_TIMEOUT] [--offline] [-s] [-w] [--verbose] [--debug] [--log-file LOG_FILE] [--no-prettify-json] [-v] [file_path ...] @@ -14,26 +14,26 @@ usage: parsedmarc [-h] [-c CONFIG_FILE] [--strip-attachment-payloads] [-o OUTPUT Parses DMARC reports positional arguments: - file_path one or more paths to aggregate or forensic report files, emails, or mbox files' + file_path one or more paths to aggregate or failure report files, emails, or mbox files' options: -h, --help show this help message and exit -c CONFIG_FILE, --config-file CONFIG_FILE a path to a configuration file (--silent implied) --strip-attachment-payloads - remove attachment payloads from forensic report output + remove attachment payloads from failure report output -o OUTPUT, --output OUTPUT write output files to the given directory --aggregate-json-filename AGGREGATE_JSON_FILENAME filename for the aggregate JSON output file - --forensic-json-filename FORENSIC_JSON_FILENAME - filename for the forensic JSON output file + --failure-json-filename FAILURE_JSON_FILENAME + filename for the failure JSON output file --smtp-tls-json-filename SMTP_TLS_JSON_FILENAME filename for the SMTP TLS JSON output file --aggregate-csv-filename AGGREGATE_CSV_FILENAME filename for the aggregate CSV output file - --forensic-csv-filename FORENSIC_CSV_FILENAME - filename for the forensic CSV output file + --failure-csv-filename FAILURE_CSV_FILENAME + filename for the failure CSV output file --smtp-tls-csv-filename SMTP_TLS_CSV_FILENAME filename for the SMTP TLS CSV output file -n NAMESERVERS [NAMESERVERS ...], --nameservers NAMESERVERS [NAMESERVERS ...] @@ -70,7 +70,7 @@ For example [general] save_aggregate = True -save_forensic = True +save_failure = True [imap] host = imap.example.com @@ -109,7 +109,7 @@ mode = tcp [webhook] aggregate_url = https://aggregate_url.example.com -forensic_url = https://forensic_url.example.com +failure_url = https://failure_url.example.com smtp_tls_url = https://smtp_tls_url.example.com timeout = 60 ``` @@ -119,7 +119,7 @@ The full set of configuration options are: - `general` - `save_aggregate` - bool: Save aggregate report data to Elasticsearch, Splunk and/or S3 - - `save_forensic` - bool: Save forensic report data to + - `save_failure` - bool: Save failure report data to Elasticsearch, Splunk and/or S3 - `save_smtp_tls` - bool: Save SMTP-STS report data to Elasticsearch, Splunk and/or S3 @@ -130,7 +130,7 @@ The full set of configuration options are: - `output` - str: Directory to place JSON and CSV files in. This is required if you set either of the JSON output file options. - `aggregate_json_filename` - str: filename for the aggregate JSON output file - - `forensic_json_filename` - str: filename for the forensic + - `failure_json_filename` - str: filename for the failure JSON output file - `ip_db_path` - str: An optional custom path to a MMDB file from IPinfo, MaxMind, or DBIP @@ -297,6 +297,12 @@ The full set of configuration options are: creating the index (Default: `1`) - `number_of_replicas` - int: The number of replicas to use when creating the index (Default: `0`) + - `serverless` - bool: Set to `True` when targeting an Elastic Cloud + Serverless project. Serverless manages sharding and replication itself + and rejects the `number_of_shards` / `number_of_replicas` index settings + with HTTP 400. With this flag set, parsedmarc strips those keys from the + settings sent at index creation; any other settings (e.g. + `refresh_interval`) are passed through unchanged (Default: `False`) - `opensearch` - `hosts` - str: A comma separated list of hostnames and ports or URLs (e.g. `127.0.0.1:9200` or @@ -340,7 +346,7 @@ The full set of configuration options are: - `skip_certificate_verification` - bool: Skip certificate verification (not recommended) - `aggregate_topic` - str: The Kafka topic for aggregate reports - - `forensic_topic` - str: The Kafka topic for forensic reports + - `failure_topic` - str: The Kafka topic for failure reports - `smtp` - `host` - str: The SMTP hostname - `port` - int: The SMTP port (Default: `25`) @@ -361,6 +367,52 @@ The full set of configuration options are: `%` characters must be escaped with another `%` character, so use `%%` wherever a `%` character is used. ::: +- `postgresql` + - `host` - str: The PostgreSQL server hostname or IP address. + Required unless `connection_string` is provided. + - `port` - int: The PostgreSQL server port (Default: `5432`) + - `user` - str: The database user name (Optional) + - `password` - str: The database user password (Optional) + - `database` - str: The database name (Optional) + - `connection_string` - str: A full libpq connection string or URI + (e.g. `postgresql://user:pass@host/dbname`). When provided, + all individual parameters above are ignored. + + The PostgreSQL backend is an optional extra. Install it with + `pip install parsedmarc[postgresql]` (it pulls in `psycopg`); the + prebuilt binary wheels are not available for every platform, which is + why it is not a mandatory dependency. + + Tables are created automatically on first run using + `CREATE TABLE IF NOT EXISTS`, so no manual schema migration is needed + for fresh installations. + + **Example configuration:** + + ```ini + [postgresql] + host = localhost + port = 5432 + user = parsedmarc + password = secret + database = parsedmarc + ``` + + Or using a DSN/URI: + + ```ini + [postgresql] + connection_string = postgresql://parsedmarc:secret@localhost/parsedmarc + ``` + + Saving parsed data to PostgreSQL is controlled by the `[general]` + options `save_aggregate`, `save_failure`, and `save_smtp_tls` + (`save_forensic` is still accepted as a deprecated alias for + `save_failure`). These flags must be set to `True` for the + corresponding report types (aggregate DMARC, failure DMARC, and + SMTP TLS reports) or no data will be written to PostgreSQL, even if + this section is configured. + - `s3` - `bucket` - str: The S3 bucket name - `path` - str: The path to upload reports to (Default: `/`) @@ -458,7 +510,7 @@ The full set of configuration options are: - `dce` - str: The Data Collection Endpoint (DCE). Example: `https://{DCE-NAME}.{REGION}.ingest.monitor.azure.com`. - `dcr_immutable_id` - str: The immutable ID of the Data Collection Rule (DCR) - `dcr_aggregate_stream` - str: The stream name for aggregate reports in the DCR - - `dcr_forensic_stream` - str: The stream name for the forensic reports in the DCR + - `dcr_failure_stream` - str: The stream name for the failure reports in the DCR - `dcr_smtp_tls_stream` - str: The stream name for the SMTP TLS reports in the DCR :::{note} @@ -470,12 +522,12 @@ The full set of configuration options are: - `mode` - str: The GELF transport type to use. Valid modes: `tcp`, `udp`, `tls` - `maildir` - - `maildir_path` - str: Full path for mailbox maidir location (Default: `INBOX`) + - `maildir_path` - str: Full path for mailbox maildir location (Default: `INBOX`) - `maildir_create` - bool: Create maildir if not present (Default: False) - `webhook` - Post the individual reports to a webhook url with the report as the JSON body - `aggregate_url` - str: URL of the webhook which should receive the aggregate reports - - `forensic_url` - str: URL of the webhook which should receive the forensic reports + - `failure_url` - str: URL of the webhook which should receive the failure reports - `smtp_tls_url` - str: URL of the webhook which should receive the smtp_tls reports - `timeout` - int: Interval in which the webhook call should timeout @@ -490,26 +542,26 @@ blocks DNS requests to outside resolvers. ::: :::{note} -`save_aggregate` and `save_forensic` are separate options -because you may not want to save forensic reports -(also known as failure reports) to your Elasticsearch instance, +`save_aggregate` and `save_failure` are separate options +because you may not want to save failure reports +(formerly known as forensic reports) to your Elasticsearch instance, particularly if you are in a highly-regulated industry that handles sensitive data, such as healthcare or finance. If your legitimate outgoing email fails DMARC, it is possible -that email may appear later in a forensic report. +that email may appear later in a failure report. -Forensic reports contain the original headers of an email that +Failure reports contain the original headers of an email that failed a DMARC check, and sometimes may also include the full message body, depending on the policy of the reporting organization. -Most reporting organizations do not send forensic reports of any +Most reporting organizations do not send failure reports of any kind for privacy reasons. While aggregate DMARC reports are sent -at least daily, it is normal to receive very few forensic reports. +at least daily, it is normal to receive very few failure reports. -An alternative approach is to still collect forensic/failure/ruf +An alternative approach is to still collect failure/ruf reports in your DMARC inbox, but run `parsedmarc` with -```save_forensic = True``` manually on a separate IMAP folder (using +```save_failure = True``` manually on a separate IMAP folder (using the ```reports_folder``` option), after you have manually moved known samples you want to save to that folder (e.g. malicious samples and non-sensitive legitimate samples). @@ -610,31 +662,76 @@ services: PARSEDMARC_MAILBOX_WATCH: "true" PARSEDMARC_ELASTICSEARCH_HOSTS: http://elasticsearch:9200 PARSEDMARC_GENERAL_SAVE_AGGREGATE: "true" - PARSEDMARC_GENERAL_SAVE_FORENSIC: "true" + PARSEDMARC_GENERAL_SAVE_FAILURE: "true" ``` +### Docker secrets (`_FILE` suffix) + +Any `PARSEDMARC_{SECTION}_{KEY}` environment variable can also be supplied +via a file by appending `_FILE` to its name. The file's contents (with any +trailing CR/LF characters stripped) are used as the value. This is the +same convention used by the official Postgres, MariaDB, and Redis container +images, and is designed to plug straight into Docker / Docker Compose / +Kubernetes secrets so credentials never appear in plain `environment:` +blocks (where they would be readable via `docker inspect`, container logs, +and `/proc//environ`). + +The bare `DEBUG` / `PARSEDMARC_DEBUG` aliases and `PARSEDMARC_CONFIG_FILE` +do not have a `_FILE` form; only `PARSEDMARC_{SECTION}_{KEY}` vars resolved +to a known config section are eligible. + +If both the direct env var and the `_FILE` variant are set, the `_FILE` +variant wins. If the file does not exist or is unreadable, parsedmarc +exits with a configuration error rather than silently falling back to an +empty value. + +```yaml +secrets: + imap_password: + file: ./secrets/imap_password.txt + +services: + parsedmarc: + image: parsedmarc:latest + secrets: + - imap_password + environment: + PARSEDMARC_IMAP_HOST: imap.example.com + PARSEDMARC_IMAP_USER: dmarc@example.com + PARSEDMARC_IMAP_PASSWORD_FILE: /run/secrets/imap_password +``` + +Note that a small set of config keys whose own names already end in +`_file` (`[general] log_file`, `[msgraph] token_file`, +`[gmail_api] credentials_file`, `[gmail_api] token_file`) keep their +pre-existing meaning when set via `PARSEDMARC_..._FILE` — that env var is +the path itself, not a wrapper around a file containing the path. To pass +*those* paths via a Docker secret, double up the suffix +(`PARSEDMARC_GMAIL_API_CREDENTIALS_FILE_FILE`); the inner contents are +then read and stored as the `credentials_file` value. + ### Section name mapping For sections with underscores in the name, the full section name is used: -| Section | Env var prefix | -|------------------|-------------------------------| -| `general` | `PARSEDMARC_GENERAL_` | -| `mailbox` | `PARSEDMARC_MAILBOX_` | -| `imap` | `PARSEDMARC_IMAP_` | -| `msgraph` | `PARSEDMARC_MSGRAPH_` | -| `elasticsearch` | `PARSEDMARC_ELASTICSEARCH_` | -| `opensearch` | `PARSEDMARC_OPENSEARCH_` | -| `splunk_hec` | `PARSEDMARC_SPLUNK_HEC_` | -| `kafka` | `PARSEDMARC_KAFKA_` | -| `smtp` | `PARSEDMARC_SMTP_` | -| `s3` | `PARSEDMARC_S3_` | -| `syslog` | `PARSEDMARC_SYSLOG_` | -| `gmail_api` | `PARSEDMARC_GMAIL_API_` | -| `maildir` | `PARSEDMARC_MAILDIR_` | -| `log_analytics` | `PARSEDMARC_LOG_ANALYTICS_` | -| `gelf` | `PARSEDMARC_GELF_` | -| `webhook` | `PARSEDMARC_WEBHOOK_` | +| Section | Env var prefix | +| --- | --- | +| `general` | `PARSEDMARC_GENERAL_` | +| `mailbox` | `PARSEDMARC_MAILBOX_` | +| `imap` | `PARSEDMARC_IMAP_` | +| `msgraph` | `PARSEDMARC_MSGRAPH_` | +| `elasticsearch` | `PARSEDMARC_ELASTICSEARCH_` | +| `opensearch` | `PARSEDMARC_OPENSEARCH_` | +| `splunk_hec` | `PARSEDMARC_SPLUNK_HEC_` | +| `kafka` | `PARSEDMARC_KAFKA_` | +| `smtp` | `PARSEDMARC_SMTP_` | +| `s3` | `PARSEDMARC_S3_` | +| `syslog` | `PARSEDMARC_SYSLOG_` | +| `gmail_api` | `PARSEDMARC_GMAIL_API_` | +| `maildir` | `PARSEDMARC_MAILDIR_` | +| `log_analytics` | `PARSEDMARC_LOG_ANALYTICS_` | +| `gelf` | `PARSEDMARC_GELF_` | +| `webhook` | `PARSEDMARC_WEBHOOK_` | ## Performance tuning @@ -651,7 +748,7 @@ imports more predictable: - Use `mailbox.since` to process reports in smaller time windows such as `1d`, `7d`, or another interval that fits the backlog. This makes it easier to catch up incrementally instead of loading an entire mailbox history in one run. -- Set `strip_attachment_payloads = True` when forensic reports contain large +- Set `strip_attachment_payloads = True` when failure reports contain large attachments and you do not need to retain the raw payloads in the parsed output. - Prefer running parsedmarc separately from Elasticsearch or OpenSearch, or diff --git a/_static/documentation_options.js b/_static/documentation_options.js index 1bc3c11..6acc351 100644 --- a/_static/documentation_options.js +++ b/_static/documentation_options.js @@ -1,5 +1,5 @@ const DOCUMENTATION_OPTIONS = { - VERSION: '9.11.2', + VERSION: '10.0.0', LANGUAGE: 'en', COLLAPSE_INDEX: false, BUILDER: 'html', diff --git a/api.html b/api.html index 31c9edb..e61e6b4 100644 --- a/api.html +++ b/api.html @@ -6,14 +6,14 @@ - API reference — parsedmarc 9.11.2 documentation + API reference — parsedmarc 10.0.0 documentation - + @@ -57,9 +57,11 @@
  • parsedmarc
    Returns:
    -

    Lists of aggregate_reports, forensic_reports, and smtp_tls_reports

    +

    Lists of aggregate_reports, failure_reports, and smtp_tls_reports

    Return type:

    dict

    @@ -346,7 +378,7 @@ DMARC reports

  • dns_retries (int) – Number of times to retry DNS queries on timeout or other transient errors

  • strip_attachment_payloads (bool) – Remove attachment payloads from -forensic report results

    • +failure report results

  • always_use_local_files (bool) – Do not download files

  • reverse_dns_map_path (str) – Path to a reverse DNS map file

  • reverse_dns_map_url (str) – URL to a reverse DNS map file

    • @@ -356,7 +388,7 @@ forensic report results

    Returns:
    -

    Lists of aggregate_reports, forensic_reports, and smtp_tls_reports

    +

    Lists of aggregate_reports, failure_reports, and smtp_tls_reports

    Return type:

    dict

    @@ -445,9 +477,9 @@ other transient errors

    -
    -parsedmarc.parse_forensic_report(feedback_report: str, sample: str, msg_date: datetime, *, always_use_local_files: bool = False, reverse_dns_map_path: str | None = None, reverse_dns_map_url: str | None = None, offline: bool = False, ip_db_path: str | None = None, nameservers: list[str] | None = None, dns_timeout: float = 2.0, dns_retries: int = 0, strip_attachment_payloads: bool = False) → ForensicReport[source]
    -

    Converts a DMARC forensic report and sample to a dict

    +
    +parsedmarc.parse_failure_report(feedback_report: str, sample: str, msg_date: datetime, *, always_use_local_files: bool = False, reverse_dns_map_path: str | None = None, reverse_dns_map_url: str | None = None, offline: bool = False, ip_db_path: str | None = None, nameservers: list[str] | None = None, dns_timeout: float = 2.0, dns_retries: int = 0, strip_attachment_payloads: bool = False) → FailureReport[source]
    +

    Converts a DMARC failure report and sample to a dict

    Parameters:
      @@ -465,7 +497,40 @@ other transient errors

    • dns_retries (int) – Number of times to retry DNS queries on timeout or other transient errors

    • strip_attachment_payloads (bool) – Remove attachment payloads from -forensic report results

      • +failure report results

      +
    +
    +
    Returns:
    +

    A parsed report and sample

    +
    +
    Return type:
    +

    dict

    +
    +
    +
    + +
    +
    +parsedmarc.parse_forensic_report(feedback_report: str, sample: str, msg_date: datetime, *, always_use_local_files: bool = False, reverse_dns_map_path: str | None = None, reverse_dns_map_url: str | None = None, offline: bool = False, ip_db_path: str | None = None, nameservers: list[str] | None = None, dns_timeout: float = 2.0, dns_retries: int = 0, strip_attachment_payloads: bool = False) → FailureReport
    +

    Converts a DMARC failure report and sample to a dict

    +
    +
    Parameters:
    +
      +

    • feedback_report (str) – A message’s feedback report as a string

      • +

    • sample (str) – The RFC 822 headers or RFC 822 message sample

      • +

    • ip_db_path (str) – Path to a MMDB file from IPinfo, MaxMind, or DBIP

      • +

    • always_use_local_files (bool) – Do not download files

      • +

    • reverse_dns_map_path (str) – Path to a reverse DNS map file

      • +

    • reverse_dns_map_url (str) – URL to a reverse DNS map file

      • +

    • offline (bool) – Do not query online for geolocation or DNS

      • +

    • msg_date (str) – The message’s date header

      • +

    • nameservers (list) – A list of one or more nameservers to use +(Cloudflare’s public DNS resolvers by default)

      • +

    • dns_timeout (float) – Sets the DNS timeout in seconds

      • +

    • dns_retries (int) – Number of times to retry DNS queries on timeout +or other transient errors

      • +

    • strip_attachment_payloads (bool) – Remove attachment payloads from +failure report results

    Returns:
    @@ -479,7 +544,7 @@ forensic report results

    -parsedmarc.parse_report_email(input_: bytes | str, *, offline: bool = False, ip_db_path: str | None = None, always_use_local_files: bool = False, reverse_dns_map_path: str | None = None, reverse_dns_map_url: str | None = None, nameservers: list[str] | None = None, dns_timeout: float = 2.0, dns_retries: int = 0, strip_attachment_payloads: bool = False, keep_alive: Callable | None = None, normalize_timespan_threshold_hours: float = 24.0) → AggregateParsedReport | ForensicParsedReport | SMTPTLSParsedReport[source]
    +parsedmarc.parse_report_email(input_: bytes | str, *, offline: bool = False, ip_db_path: str | None = None, always_use_local_files: bool = False, reverse_dns_map_path: str | None = None, reverse_dns_map_url: str | None = None, nameservers: list[str] | None = None, dns_timeout: float = 2.0, dns_retries: int = 0, strip_attachment_payloads: bool = False, keep_alive: Callable | None = None, normalize_timespan_threshold_hours: float = 24.0) → AggregateParsedReport | FailureParsedReport | SMTPTLSParsedReport[source]

    Parses a DMARC report from an email

    Parameters:
    @@ -495,14 +560,14 @@ forensic report results

  • dns_retries (int) – Number of times to retry DNS queries on timeout or other transient errors

  • strip_attachment_payloads (bool) – Remove attachment payloads from -forensic report results

    • +failure report results

  • keep_alive (callable) – keep alive function

  • normalize_timespan_threshold_hours (float) – Normalize timespans beyond this

    • Returns:

      -

    • report_type: aggregate or forensic

      • +

    • report_type: aggregate or failure

    • report: The parsed report

    @@ -515,8 +580,8 @@ forensic report results

    -parsedmarc.parse_report_file(input_: bytes | str | PathLike[str] | PathLike[bytes] | BinaryIO, *, nameservers: list[str] | None = None, dns_timeout: float = 2.0, dns_retries: int = 0, strip_attachment_payloads: bool = False, ip_db_path: str | None = None, always_use_local_files: bool = False, reverse_dns_map_path: str | None = None, reverse_dns_map_url: str | None = None, offline: bool = False, keep_alive: Callable | None = None, normalize_timespan_threshold_hours: float = 24) → AggregateParsedReport | ForensicParsedReport | SMTPTLSParsedReport[source]
    -

    Parses a DMARC aggregate or forensic file at the given path, a +parsedmarc.parse_report_file(input_: bytes | str | PathLike[str] | PathLike[bytes] | BinaryIO, *, nameservers: list[str] | None = None, dns_timeout: float = 2.0, dns_retries: int = 0, strip_attachment_payloads: bool = False, ip_db_path: str | None = None, always_use_local_files: bool = False, reverse_dns_map_path: str | None = None, reverse_dns_map_url: str | None = None, offline: bool = False, keep_alive: Callable | None = None, normalize_timespan_threshold_hours: float = 24) → AggregateParsedReport | FailureParsedReport | SMTPTLSParsedReport[source] +

    Parses a DMARC aggregate or failure file at the given path, a file-like object. or bytes

    Parameters:
    @@ -529,7 +594,7 @@ a file-like object, or bytes

  • dns_retries (int) – Number of times to retry DNS queries on timeout or other transient errors

  • strip_attachment_payloads (bool) – Remove attachment payloads from -forensic report results

    • +failure report results

  • ip_db_path (str) – Path to a MMDB file from IPinfo, MaxMind, or DBIP

  • always_use_local_files (bool) – Do not download files

  • reverse_dns_map_path (str) – Path to a reverse DNS map

    • @@ -591,16 +656,52 @@ format

    -
    -parsedmarc.parsed_forensic_reports_to_csv(reports: ForensicReport | list[ForensicReport]) → str[source]
    -

    Converts one or more parsed forensic reports to flat CSV format, including +

    +parsedmarc.parsed_failure_reports_to_csv(reports: FailureReport | list[FailureReport]) → str[source]
    +

    Converts one or more parsed failure reports to flat CSV format, including headers

    Parameters:
    -

    reports – A parsed forensic report or list of parsed forensic reports

    +

    reports – A parsed failure report or list of parsed failure reports

    Returns:
    -

    Parsed forensic report data in flat CSV format, including headers

    +

    Parsed failure report data in flat CSV format, including headers

    +
    +
    Return type:
    +

    str

    +
    +
    +
    + +
    +
    +parsedmarc.parsed_failure_reports_to_csv_rows(reports: FailureReport | list[FailureReport]) → list[dict[str, Any]][source]
    +

    Converts one or more parsed failure reports to a list of dicts in flat CSV +format

    +
    +
    Parameters:
    +

    reports – A parsed failure report or list of parsed failure reports

    +
    +
    Returns:
    +

    Parsed failure report data as a list of dicts in flat CSV format

    +
    +
    Return type:
    +

    list

    +
    +
    +
    + +
    +
    +parsedmarc.parsed_forensic_reports_to_csv(reports: FailureReport | list[FailureReport]) → str
    +

    Converts one or more parsed failure reports to flat CSV format, including +headers

    +
    +
    Parameters:
    +

    reports – A parsed failure report or list of parsed failure reports

    +
    +
    Returns:
    +

    Parsed failure report data in flat CSV format, including headers

    Return type:

    str

    @@ -610,15 +711,15 @@ headers

    -parsedmarc.parsed_forensic_reports_to_csv_rows(reports: ForensicReport | list[ForensicReport]) → list[dict[str, Any]][source]
    -

    Converts one or more parsed forensic reports to a list of dicts in flat CSV +parsedmarc.parsed_forensic_reports_to_csv_rows(reports: FailureReport | list[FailureReport]) → list[dict[str, Any]] +

    Converts one or more parsed failure reports to a list of dicts in flat CSV format

    Parameters:
    -

    reports – A parsed forensic report or list of parsed forensic reports

    +

    reports – A parsed failure report or list of parsed failure reports

    Returns:
    -

    Parsed forensic report data as a list of dicts in flat CSV format

    +

    Parsed failure report data as a list of dicts in flat CSV format

    Return type:

    list

    @@ -653,7 +754,7 @@ layer dict objects suitable for use in a CSV

    -parsedmarc.save_output(results: ParsingResults, *, output_directory: str = 'output', aggregate_json_filename: str = 'aggregate.json', forensic_json_filename: str = 'forensic.json', smtp_tls_json_filename: str = 'smtp_tls.json', aggregate_csv_filename: str = 'aggregate.csv', forensic_csv_filename: str = 'forensic.csv', smtp_tls_csv_filename: str = 'smtp_tls.csv')[source]
    +parsedmarc.save_output(results: ParsingResults, *, output_directory: str = 'output', aggregate_json_filename: str = 'aggregate.json', failure_json_filename: str = 'failure.json', smtp_tls_json_filename: str = 'smtp_tls.json', aggregate_csv_filename: str = 'aggregate.csv', failure_csv_filename: str = 'failure.csv', smtp_tls_csv_filename: str = 'smtp_tls.csv')[source]

    Save report data in the given directory

    Parameters:
    @@ -661,10 +762,10 @@ layer dict objects suitable for use in a CSV

  • results – Parsing results

  • output_directory (str) – The path to the directory to save in

  • aggregate_json_filename (str) – Filename for the aggregate JSON file

    • -

  • forensic_json_filename (str) – Filename for the forensic JSON file

    • +

  • failure_json_filename (str) – Filename for the failure JSON file

  • smtp_tls_json_filename (str) – Filename for the SMTP TLS JSON file

  • aggregate_csv_filename (str) – Filename for the aggregate CSV file

    • -

  • forensic_csv_filename (str) – Filename for the forensic CSV file

    • +

  • failure_csv_filename (str) – Filename for the failure CSV file

  • smtp_tls_csv_filename (str) – Filename for the SMTP TLS CSV file

    • @@ -700,7 +801,7 @@ or the number of seconds until the next mail check

  • dns_retries (int) – Number of times to retry DNS queries on timeout or other transient errors

  • strip_attachment_payloads (bool) – Replace attachment payloads in -forensic report samples with None

    • +failure report samples with None

  • batch_size (int) – Number of messages to read and process before saving

  • since – Search for messages since certain time

  • normalize_timespan_threshold_hours (float) – Normalize timespans beyond this

    • @@ -734,7 +835,9 @@ reload has been requested (e.g. via SIGHUP)

    Parameters:

    • names (list) – A list of index names

      • -

    • settings (dict) – Index settings

      • +

    • settings (dict) – Index settings. In Serverless mode, keys in +_SERVERLESS_REJECTED_SETTINGS are filtered out and the +remaining keys are passed through; defaults are skipped entirely.

    @@ -742,13 +845,13 @@ reload has been requested (e.g. via SIGHUP)

    -parsedmarc.elastic.migrate_indexes(aggregate_indexes: list[str] | None = None, forensic_indexes: list[str] | None = None)[source]
    +parsedmarc.elastic.migrate_indexes(aggregate_indexes: list[str] | None = None, failure_indexes: list[str] | None = None)[source]

    Updates index mappings

    Parameters:

    • aggregate_indexes (list) – A list of aggregate index names

      • -

    • forensic_indexes (list) – A list of forensic index names

      • +

    • failure_indexes (list) – A list of failure index names

    @@ -761,7 +864,7 @@ reload has been requested (e.g. via SIGHUP)

    Parameters:
      -

    • aggregate_report (dict) – A parsed forensic report

      • +

    • aggregate_report (dict) – A parsed aggregate report

    • index_suffix (str) – The suffix of the name of the index to save to

    • index_prefix (str) – The prefix of the name of the index to save to

    • monthly_indexes (bool) – Use monthly indexes instead of daily indexes

      • @@ -776,13 +879,36 @@ reload has been requested (e.g. via SIGHUP)

    -
    -parsedmarc.elastic.save_forensic_report_to_elasticsearch(forensic_report: dict[str, Any], index_suffix: Any | None = None, index_prefix: str | None = None, monthly_indexes: bool | None = False, number_of_shards: int = 1, number_of_replicas: int = 0)[source]
    -

    Saves a parsed DMARC forensic report to Elasticsearch

    +
    +parsedmarc.elastic.save_failure_report_to_elasticsearch(failure_report: dict[str, Any], index_suffix: Any | None = None, index_prefix: str | None = None, monthly_indexes: bool | None = False, number_of_shards: int = 1, number_of_replicas: int = 0)[source]
    +

    Saves a parsed DMARC failure report to Elasticsearch

    Parameters:
      -

    • forensic_report (dict) – A parsed forensic report

      • +

    • failure_report (dict) – A parsed failure report

      • +

    • index_suffix (str) – The suffix of the name of the index to save to

      • +

    • index_prefix (str) – The prefix of the name of the index to save to

      • +

    • monthly_indexes (bool) – Use monthly indexes instead of daily +indexes

      • +

    • number_of_shards (int) – The number of shards to use in the index

      • +

    • number_of_replicas (int) – The number of replicas to use in the +index

      • +
    +
    +
    Raises:
    +

    AlreadySaved –

    +
    +
    +
    + +
    +
    +parsedmarc.elastic.save_forensic_report_to_elasticsearch(failure_report: dict[str, Any], index_suffix: Any | None = None, index_prefix: str | None = None, monthly_indexes: bool | None = False, number_of_shards: int = 1, number_of_replicas: int = 0)
    +

    Saves a parsed DMARC failure report to Elasticsearch

    +
    +
    Parameters:
    +
      +

    • failure_report (dict) – A parsed failure report

    • index_suffix (str) – The suffix of the name of the index to save to

    • index_prefix (str) – The prefix of the name of the index to save to

    • monthly_indexes (bool) – Use monthly indexes instead of daily @@ -821,7 +947,7 @@ index

      • -parsedmarc.elastic.set_hosts(hosts: str | list[str], *, use_ssl: bool = False, ssl_cert_path: str | None = None, skip_certificate_verification: bool = False, username: str | None = None, password: str | None = None, api_key: str | None = None, timeout: float = 60.0)[source]
      +parsedmarc.elastic.set_hosts(hosts: str | list[str], *, use_ssl: bool = False, ssl_cert_path: str | None = None, skip_certificate_verification: bool = False, username: str | None = None, password: str | None = None, api_key: str | None = None, timeout: float = 60.0, serverless: bool = False)[source]

      Sets the Elasticsearch hosts to use

      Parameters:
      @@ -834,6 +960,10 @@ index

    • password (str) – The password to use for authentication

    • api_key (str) – The Base64 encoded API key to use for authentication

    • timeout (float) – Timeout in seconds

      • +

    • serverless (bool) – Target an Elastic Cloud Serverless project. When True, +create_indexes strips number_of_shards / number_of_replicas +from its settings (which Serverless rejects with HTTP 400) and passes +any other settings through unchanged.

    @@ -870,13 +1000,13 @@ index

    -parsedmarc.opensearch.migrate_indexes(aggregate_indexes: list[str] | None = None, forensic_indexes: list[str] | None = None)[source]
    +parsedmarc.opensearch.migrate_indexes(aggregate_indexes: list[str] | None = None, failure_indexes: list[str] | None = None)[source]

    Updates index mappings

    Parameters:

    • aggregate_indexes (list) – A list of aggregate index names

      • -

    • forensic_indexes (list) – A list of forensic index names

      • +

    • failure_indexes (list) – A list of failure index names

    @@ -889,7 +1019,7 @@ index

    Parameters:
      -

    • aggregate_report (dict) – A parsed forensic report

      • +

    • aggregate_report (dict) – A parsed aggregate report

    • index_suffix (str) – The suffix of the name of the index to save to

    • index_prefix (str) – The prefix of the name of the index to save to

    • monthly_indexes (bool) – Use monthly indexes instead of daily indexes

      • @@ -904,13 +1034,36 @@ index

    -
    -parsedmarc.opensearch.save_forensic_report_to_opensearch(forensic_report: dict[str, Any], index_suffix: str | None = None, index_prefix: str | None = None, monthly_indexes: bool = False, number_of_shards: int = 1, number_of_replicas: int = 0)[source]
    -

    Saves a parsed DMARC forensic report to OpenSearch

    +
    +parsedmarc.opensearch.save_failure_report_to_opensearch(failure_report: dict[str, Any], index_suffix: str | None = None, index_prefix: str | None = None, monthly_indexes: bool = False, number_of_shards: int = 1, number_of_replicas: int = 0)[source]
    +

    Saves a parsed DMARC failure report to OpenSearch

    Parameters:
      -

    • forensic_report (dict) – A parsed forensic report

      • +

    • failure_report (dict) – A parsed failure report

      • +

    • index_suffix (str) – The suffix of the name of the index to save to

      • +

    • index_prefix (str) – The prefix of the name of the index to save to

      • +

    • monthly_indexes (bool) – Use monthly indexes instead of daily +indexes

      • +

    • number_of_shards (int) – The number of shards to use in the index

      • +

    • number_of_replicas (int) – The number of replicas to use in the +index

      • +
    +
    +
    Raises:
    +

    AlreadySaved –

    +
    +
    +
    + +
    +
    +parsedmarc.opensearch.save_forensic_report_to_opensearch(failure_report: dict[str, Any], index_suffix: str | None = None, index_prefix: str | None = None, monthly_indexes: bool = False, number_of_shards: int = 1, number_of_replicas: int = 0)
    +

    Saves a parsed DMARC failure report to OpenSearch

    +
    +
    Parameters:
    +
      +

    • failure_report (dict) – A parsed failure report

    • index_suffix (str) – The suffix of the name of the index to save to

    • index_prefix (str) – The prefix of the name of the index to save to

    • monthly_indexes (bool) – Use monthly indexes instead of daily @@ -1009,12 +1162,24 @@ to save in Splunk

    -
    -save_forensic_reports_to_splunk(forensic_reports: list[dict[str, Any]] | dict[str, Any])[source]
    -

    Saves forensic DMARC reports to Splunk

    +
    +save_failure_reports_to_splunk(failure_reports: list[dict[str, Any]] | dict[str, Any])[source]
    +

    Saves failure DMARC reports to Splunk

    Parameters:
    -

    forensic_reports (list) – A list of forensic report dictionaries +

    failure_reports (list) – A list of failure report dictionaries +to save in Splunk

    +
    +
    +
    + +
    +
    +save_forensic_reports_to_splunk(failure_reports: list[dict[str, Any]] | dict[str, Any])
    +

    Saves failure DMARC reports to Splunk

    +
    +
    Parameters:
    +

    failure_reports (list) – A list of failure report dictionaries to save in Splunk

    @@ -1114,15 +1279,27 @@ to save in Splunk

    -
    -class parsedmarc.types.ForensicParsedReport[source]
    +
    +class parsedmarc.types.FailureParsedReport[source]
    -
    -class parsedmarc.types.ForensicReport[source]
    +
    +class parsedmarc.types.FailureReport[source]
    +
    +
    +parsedmarc.types.ForensicParsedReport
    +

    alias of FailureParsedReport

    +
    + +
    +
    +parsedmarc.types.ForensicReport
    +

    alias of FailureReport

    +
    +
    class parsedmarc.types.IPSourceInfo[source]
    diff --git a/contributing.html b/contributing.html index f35288a..ab72354 100644 --- a/contributing.html +++ b/contributing.html @@ -6,14 +6,14 @@ - Contributing to parsedmarc — parsedmarc 9.11.2 documentation + Contributing to parsedmarc — parsedmarc 10.0.0 documentation - + diff --git a/davmail.html b/davmail.html index 5b18b17..cc649de 100644 --- a/davmail.html +++ b/davmail.html @@ -6,14 +6,14 @@ - Accessing an inbox using OWA/EWS — parsedmarc 9.11.2 documentation + Accessing an inbox using OWA/EWS — parsedmarc 10.0.0 documentation - + diff --git a/dmarc.html b/dmarc.html index f1426b2..542ab7d 100644 --- a/dmarc.html +++ b/dmarc.html @@ -6,14 +6,14 @@ - Understanding DMARC — parsedmarc 9.11.2 documentation + Understanding DMARC — parsedmarc 10.0.0 documentation - + diff --git a/elasticsearch.html b/elasticsearch.html index 640d2b0..b7da3a3 100644 --- a/elasticsearch.html +++ b/elasticsearch.html @@ -6,14 +6,14 @@ - Elasticsearch and Kibana — parsedmarc 9.11.2 documentation + Elasticsearch and Kibana — parsedmarc 10.0.0 documentation - + @@ -186,7 +186,7 @@ server.ssl.key: /etc/kibana/kibana.key

    Note

    -

    For more security, you can configure Kibana to use a local network connexion +

    For more security, you can configure Kibana to use a local network connection to elasticsearch :

    elasticsearch.hosts: ['https://SERVER_IP:9200']
    @@ -251,7 +251,7 @@ Kibana index patterns with versions that match the upgraded indexes:

    1. Login in to Kibana, and click on Management

    2. Under Kibana, click on Saved Objects

      3. -

    3. Check the checkboxes for the dmarc_aggregate and dmarc_forensic +

    4. Check the checkboxes for the dmarc_aggregate and dmarc_failure index patterns

    5. Click Delete

    6. Click Delete on the conformation message

      7. diff --git a/genindex.html b/genindex.html index eb46209..8a5a6fc 100644 --- a/genindex.html +++ b/genindex.html @@ -5,14 +5,14 @@ - Index — parsedmarc 9.11.2 documentation + Index — parsedmarc 10.0.0 documentation - + @@ -115,10 +115,10 @@
    7. AggregateParsedReport (class in parsedmarc.types)
      8. - - + @@ -191,11 +193,15 @@

      F

      @@ -249,7 +255,9 @@
    8. InvalidDMARCReport
      9. -
    9. InvalidForensicReport +
    10. InvalidFailureReport +
      11. +
    11. InvalidForensicReport (in module parsedmarc)
    12. InvalidIPinfoAPIKey
      13. @@ -327,6 +335,8 @@
    13. parse_aggregate_report_xml() (in module parsedmarc)
    14. parse_email() (in module parsedmarc.utils) +
      15. +
    15. parse_failure_report() (in module parsedmarc)
    16. parse_forensic_report() (in module parsedmarc)
      17. @@ -339,6 +349,10 @@
    17. parsed_aggregate_reports_to_csv() (in module parsedmarc)
    18. parsed_aggregate_reports_to_csv_rows() (in module parsedmarc) +
      19. +
    19. parsed_failure_reports_to_csv() (in module parsedmarc) +
      20. +
    20. parsed_failure_reports_to_csv_rows() (in module parsedmarc)
    21. parsed_forensic_reports_to_csv() (in module parsedmarc)
      22. @@ -346,12 +360,12 @@
    22. parsed_smtp_tls_reports_to_csv() (in module parsedmarc)
      23. + + -
        +
      • save_smtp_tls_report_to_opensearch() (in module parsedmarc.opensearch) +
      • save_smtp_tls_reports_to_splunk() (parsedmarc.splunk.HECClient method)
      • set_hosts() (in module parsedmarc.elastic) diff --git a/index.html b/index.html index c9b9871..ff0cd95 100644 --- a/index.html +++ b/index.html @@ -6,14 +6,14 @@ - parsedmarc documentation - Open source DMARC report analyzer and visualizer — parsedmarc 9.11.2 documentation + parsedmarc documentation - Open source DMARC report analyzer and visualizer — parsedmarc 10.0.0 documentation - + @@ -101,17 +101,22 @@ and Valimail.

        Features

          -

        • Parses draft and 1.0 standard aggregate/rua DMARC reports

          • -

        • Parses forensic/failure/ruf DMARC reports

          • -

        • Parses reports from SMTP TLS Reporting

          • +

        • Parses aggregate/rua DMARC reports: the legacy draft and 1.0 schemas +(RFC 7489) and the new RFC 9990 schema for the final DMARC standard +(RFC 9989)

          • +

        • Parses failure/ruf DMARC reports (RFC 6591 and RFC 9991; formerly called +forensic reports)

          • +

        • Parses reports from SMTP TLS Reporting (TLS-RPT, RFC 8460)

        • Can parse reports from an inbox over IMAP, Microsoft Graph, or Gmail API

        • Transparently handles gzip or zip compressed reports

        • Consistent data structures

        • Simple JSON and/or CSV output

        • Optionally email the results

          • -

        • Optionally send the results to Elasticsearch, Opensearch, and/or Splunk, for use -with premade dashboards

          • -

        • Optionally send reports to Apache Kafka

          • +

        • Optionally send the results to Elasticsearch, OpenSearch, Splunk, or +PostgreSQL, for use with premade dashboards

          • +

        • Optionally send the results to Apache Kafka, Amazon S3, Azure Log +Analytics (Microsoft Sentinel), a Graylog (GELF) endpoint, a syslog server, +or an HTTP webhook

        @@ -189,7 +194,7 @@ for RHEL or Debian.

      • Sample outputs
      • Elasticsearch and Kibana
          diff --git a/installation.html b/installation.html index 0c50c26..823030a 100644 --- a/installation.html +++ b/installation.html @@ -6,14 +6,14 @@ - Installation — parsedmarc 9.11.2 documentation + Installation — parsedmarc 10.0.0 documentation - + diff --git a/kibana.html b/kibana.html index 7575ac5..0ed42aa 100644 --- a/kibana.html +++ b/kibana.html @@ -6,14 +6,14 @@ - Using the Kibana dashboards — parsedmarc 9.11.2 documentation + Using the Kibana dashboards — parsedmarc 10.0.0 documentation - + @@ -166,7 +166,7 @@ information on DMARC failure reports (also known as forensic or ruf reports). These reports contain samples of emails that have failed to pass DMARC.

          Note

          -

          Most recipients do not send forensic/failure/ruf reports at all to avoid +

          Most recipients do not send failure/ruf reports at all to avoid privacy leaks. Some recipients (notably Chinese webmail services) will only supply the headers of sample emails. Very few provide the entire email.

          diff --git a/mailing-lists.html b/mailing-lists.html index 8ababfa..c5d172d 100644 --- a/mailing-lists.html +++ b/mailing-lists.html @@ -6,14 +6,14 @@ - What about mailing lists? — parsedmarc 9.11.2 documentation + What about mailing lists? — parsedmarc 10.0.0 documentation - + diff --git a/objects.inv b/objects.inv index e0d4ebe..b7c73c5 100644 Binary files a/objects.inv and b/objects.inv differ diff --git a/opensearch.html b/opensearch.html index 17d8d94..5b00461 100644 --- a/opensearch.html +++ b/opensearch.html @@ -6,14 +6,14 @@ - OpenSearch and Grafana — parsedmarc 9.11.2 documentation + OpenSearch and Grafana — parsedmarc 10.0.0 documentation - + diff --git a/output.html b/output.html index 9c9e90b..da827a8 100644 --- a/output.html +++ b/output.html @@ -6,14 +6,14 @@ - Sample outputs — parsedmarc 9.11.2 documentation + Sample outputs — parsedmarc 10.0.0 documentation - + @@ -52,9 +52,9 @@
        • CSV aggregate report
        • -
      • Sample forensic report output
    -
    -

    Sample forensic report output

    +
    +

    Sample failure report output

    Thanks to GitHub user xennn for the anonymized -forensic report email sample.

    -
    -

    JSON forensic report

    +failure report email sample.

    +
    +

    JSON failure report

    {
      "feedback_type": "auth-failure",
      "user_agent": "Lua/1.0",
@@ -291,8 +291,8 @@ draft,acme.com,noreply-dmarc-support@acme.com,http://acme.com/dmarc/support,9391
    -
    -

    CSV forensic report

    +
    +

    CSV failure report

    feedback_type,user_agent,version,original_envelope_id,original_mail_from,original_rcpt_to,arrival_date,arrival_date_utc,subject,message_id,authentication_results,dkim_domain,source_ip_address,source_country,source_reverse_dns,source_base_domain,source_name,source_type,source_asn,source_as_name,source_as_domain,delivery_result,auth_failure,reported_domain,authentication_mechanisms,sample_headers_only
 auth-failure,Lua/1.0,1.0,,sharepoint@domain.de,peter.pan@domain.de,"Mon, 01 Oct 2018 11:20:27 +0200",2018-10-01 09:20:27,Subject,<38.E7.30937.BD6E1BB5@ mailrelay.de>,"dmarc=fail (p=none, dis=none) header.from=domain.de",,10.10.10.10,,,,policy,dmarc,domain.de,,False
    diff --git a/py-modindex.html b/py-modindex.html index 3dc2da4..69ec0ab 100644 --- a/py-modindex.html +++ b/py-modindex.html @@ -5,14 +5,14 @@ - Python Module Index — parsedmarc 9.11.2 documentation + Python Module Index — parsedmarc 10.0.0 documentation - + diff --git a/search.html b/search.html index b14d748..3165429 100644 --- a/search.html +++ b/search.html @@ -5,7 +5,7 @@ - Search — parsedmarc 9.11.2 documentation + Search — parsedmarc 10.0.0 documentation @@ -13,7 +13,7 @@ - + diff --git a/searchindex.js b/searchindex.js index 0d355a4..75f3b1b 100644 --- a/searchindex.js +++ b/searchindex.js @@ -1 +1 @@ -Search.setIndex({"alltitles":{"API reference":[[0,null]],"Accessing an inbox using OWA/EWS":[[2,null]],"Bug reports":[[1,"bug-reports"]],"CLI help":[[12,"cli-help"]],"CSV aggregate report":[[10,"csv-aggregate-report"]],"CSV forensic report":[[10,"csv-forensic-report"]],"Configuration file":[[12,"configuration-file"]],"Configuring parsedmarc for DavMail":[[2,"configuring-parsedmarc-for-davmail"]],"Contents":[[5,null]],"Contributing to parsedmarc":[[1,null]],"DMARC Alignment Guide":[[3,"dmarc-alignment-guide"]],"DMARC aggregate reports":[[7,"dmarc-aggregate-reports"]],"DMARC failure reports":[[7,"dmarc-failure-reports"]],"DMARC guides":[[3,"dmarc-guides"]],"Do":[[3,"do"],[8,"do"]],"Do not":[[3,"do-not"],[8,"do-not"]],"Docker Compose example":[[12,"docker-compose-example"]],"Elasticsearch and Kibana":[[4,null]],"Environment variable configuration":[[12,"environment-variable-configuration"]],"Examples":[[12,"examples"]],"Features":[[5,"features"]],"IP-to-country database":[[6,"ip-to-country-database"]],"Indices and tables":[[0,"indices-and-tables"]],"Installation":[[4,"installation"],[6,null],[9,"installation"]],"Installing parsedmarc":[[6,"installing-parsedmarc"]],"JSON SMTP TLS report":[[10,"json-smtp-tls-report"]],"JSON aggregate report":[[10,"json-aggregate-report"]],"JSON forensic report":[[10,"json-forensic-report"]],"LISTSERV":[[3,"listserv"],[8,"listserv"]],"Lookalike domains":[[3,"lookalike-domains"]],"Mailing list best practices":[[3,"mailing-list-best-practices"],[8,"mailing-list-best-practices"]],"Mailman 2":[[3,"mailman-2"],[3,"id1"],[8,"mailman-2"],[8,"id1"]],"Mailman 3":[[3,"mailman-3"],[3,"id2"],[8,"mailman-3"],[8,"id2"]],"Multi-tenant support":[[12,"multi-tenant-support"]],"OpenSearch and Grafana":[[9,null]],"Optional dependencies":[[6,"optional-dependencies"]],"Performance tuning":[[12,"performance-tuning"]],"Prerequisites":[[6,"prerequisites"]],"Python Compatibility":[[5,"python-compatibility"]],"Records retention":[[4,"records-retention"],[9,"records-retention"]],"Reloading configuration without restarting":[[12,"reloading-configuration-without-restarting"]],"Resources":[[3,"resources"]],"Running DavMail as a systemd service":[[2,"running-davmail-as-a-systemd-service"]],"Running parsedmarc as a systemd service":[[12,"running-parsedmarc-as-a-systemd-service"]],"Running without a config file (env-only mode)":[[12,"running-without-a-config-file-env-only-mode"]],"SMTP TLS reporting":[[7,"smtp-tls-reporting"]],"SPF and DMARC record validation":[[3,"spf-and-dmarc-record-validation"]],"Sample aggregate report output":[[10,"sample-aggregate-report-output"]],"Sample forensic report output":[[10,"sample-forensic-report-output"]],"Sample outputs":[[10,null]],"Section name mapping":[[12,"section-name-mapping"]],"Specifying the config file via environment variable":[[12,"specifying-the-config-file-via-environment-variable"]],"Splunk":[[11,null]],"Testing multiple report analyzers":[[6,"testing-multiple-report-analyzers"]],"Understanding DMARC":[[3,null]],"Upgrading Kibana index patterns":[[4,"upgrading-kibana-index-patterns"]],"Using MaxMind GeoLite2 (optional)":[[6,"using-maxmind-geolite2-optional"]],"Using Microsoft Exchange":[[6,"using-microsoft-exchange"]],"Using a web proxy":[[6,"using-a-web-proxy"]],"Using parsedmarc":[[12,null]],"Using the Kibana dashboards":[[7,null]],"What about mailing lists?":[[3,"what-about-mailing-lists"],[8,null]],"What if a sender won\u2019t support DKIM/DMARC?":[[3,"what-if-a-sender-wont-support-dkim-dmarc"]],"Workarounds":[[3,"workarounds"],[8,"workarounds"]],"parsedmarc":[[0,"module-parsedmarc"]],"parsedmarc documentation - Open source DMARC report analyzer and visualizer":[[5,null]],"parsedmarc.elastic":[[0,"module-parsedmarc.elastic"]],"parsedmarc.opensearch":[[0,"module-parsedmarc.opensearch"]],"parsedmarc.splunk":[[0,"module-parsedmarc.splunk"]],"parsedmarc.types":[[0,"module-parsedmarc.types"]],"parsedmarc.utils":[[0,"module-parsedmarc.utils"]]},"docnames":["api","contributing","davmail","dmarc","elasticsearch","index","installation","kibana","mailing-lists","opensearch","output","splunk","usage"],"envversion":{"sphinx":65,"sphinx.domains.c":3,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":9,"sphinx.domains.index":1,"sphinx.domains.javascript":3,"sphinx.domains.math":2,"sphinx.domains.python":4,"sphinx.domains.rst":2,"sphinx.domains.std":2,"sphinx.ext.todo":2,"sphinx.ext.viewcode":1},"filenames":["api.md","contributing.md","davmail.md","dmarc.md","elasticsearch.md","index.md","installation.md","kibana.md","mailing-lists.md","opensearch.md","output.md","splunk.md","usage.md"],"indexentries":{"aggregatealignment (class in parsedmarc.types)":[[0,"parsedmarc.types.AggregateAlignment",false]],"aggregateauthresultdkim (class in parsedmarc.types)":[[0,"parsedmarc.types.AggregateAuthResultDKIM",false]],"aggregateauthresults (class in parsedmarc.types)":[[0,"parsedmarc.types.AggregateAuthResults",false]],"aggregateauthresultspf (class in parsedmarc.types)":[[0,"parsedmarc.types.AggregateAuthResultSPF",false]],"aggregateidentifiers (class in parsedmarc.types)":[[0,"parsedmarc.types.AggregateIdentifiers",false]],"aggregateparsedreport (class in parsedmarc.types)":[[0,"parsedmarc.types.AggregateParsedReport",false]],"aggregatepolicyevaluated (class in parsedmarc.types)":[[0,"parsedmarc.types.AggregatePolicyEvaluated",false]],"aggregatepolicyoverridereason (class in parsedmarc.types)":[[0,"parsedmarc.types.AggregatePolicyOverrideReason",false]],"aggregatepolicypublished (class in parsedmarc.types)":[[0,"parsedmarc.types.AggregatePolicyPublished",false]],"aggregaterecord (class in parsedmarc.types)":[[0,"parsedmarc.types.AggregateRecord",false]],"aggregatereport (class in parsedmarc.types)":[[0,"parsedmarc.types.AggregateReport",false]],"aggregatereportmetadata (class in parsedmarc.types)":[[0,"parsedmarc.types.AggregateReportMetadata",false]],"alreadysaved":[[0,"parsedmarc.elastic.AlreadySaved",false],[0,"parsedmarc.opensearch.AlreadySaved",false]],"close() (parsedmarc.splunk.hecclient method)":[[0,"parsedmarc.splunk.HECClient.close",false]],"configure_ipinfo_api() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.configure_ipinfo_api",false]],"convert_outlook_msg() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.convert_outlook_msg",false]],"create_indexes() (in module parsedmarc.elastic)":[[0,"parsedmarc.elastic.create_indexes",false]],"create_indexes() (in module parsedmarc.opensearch)":[[0,"parsedmarc.opensearch.create_indexes",false]],"decode_base64() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.decode_base64",false]],"downloaderror":[[0,"parsedmarc.utils.DownloadError",false]],"elasticsearcherror":[[0,"parsedmarc.elastic.ElasticsearchError",false]],"email_results() (in module parsedmarc)":[[0,"parsedmarc.email_results",false]],"emailaddress (class in parsedmarc.types)":[[0,"parsedmarc.types.EmailAddress",false]],"emailattachment (class in parsedmarc.types)":[[0,"parsedmarc.types.EmailAttachment",false]],"emailparsererror":[[0,"parsedmarc.utils.EmailParserError",false]],"extract_report() (in module parsedmarc)":[[0,"parsedmarc.extract_report",false]],"extract_report_from_file_path() (in module parsedmarc)":[[0,"parsedmarc.extract_report_from_file_path",false]],"forensicparsedreport (class in parsedmarc.types)":[[0,"parsedmarc.types.ForensicParsedReport",false]],"forensicreport (class in parsedmarc.types)":[[0,"parsedmarc.types.ForensicReport",false]],"get_base_domain() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.get_base_domain",false]],"get_dmarc_reports_from_mailbox() (in module parsedmarc)":[[0,"parsedmarc.get_dmarc_reports_from_mailbox",false]],"get_dmarc_reports_from_mbox() (in module parsedmarc)":[[0,"parsedmarc.get_dmarc_reports_from_mbox",false]],"get_filename_safe_string() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.get_filename_safe_string",false]],"get_ip_address_country() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.get_ip_address_country",false]],"get_ip_address_db_record() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.get_ip_address_db_record",false]],"get_ip_address_info() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.get_ip_address_info",false]],"get_report_zip() (in module parsedmarc)":[[0,"parsedmarc.get_report_zip",false]],"get_reverse_dns() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.get_reverse_dns",false]],"get_service_from_reverse_dns_base_domain() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.get_service_from_reverse_dns_base_domain",false]],"hecclient (class in parsedmarc.splunk)":[[0,"parsedmarc.splunk.HECClient",false]],"human_timestamp_to_datetime() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.human_timestamp_to_datetime",false]],"human_timestamp_to_unix_timestamp() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.human_timestamp_to_unix_timestamp",false]],"invalidaggregatereport":[[0,"parsedmarc.InvalidAggregateReport",false]],"invaliddmarcreport":[[0,"parsedmarc.InvalidDMARCReport",false]],"invalidforensicreport":[[0,"parsedmarc.InvalidForensicReport",false]],"invalidipinfoapikey":[[0,"parsedmarc.utils.InvalidIPinfoAPIKey",false]],"invalidsmtptlsreport":[[0,"parsedmarc.InvalidSMTPTLSReport",false]],"ipaddressinfo (class in parsedmarc.utils)":[[0,"parsedmarc.utils.IPAddressInfo",false]],"ipsourceinfo (class in parsedmarc.types)":[[0,"parsedmarc.types.IPSourceInfo",false]],"is_mbox() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.is_mbox",false]],"is_outlook_msg() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.is_outlook_msg",false]],"load_ip_db() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.load_ip_db",false]],"load_psl_overrides() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.load_psl_overrides",false]],"load_reverse_dns_map() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.load_reverse_dns_map",false]],"migrate_indexes() (in module parsedmarc.elastic)":[[0,"parsedmarc.elastic.migrate_indexes",false]],"migrate_indexes() (in module parsedmarc.opensearch)":[[0,"parsedmarc.opensearch.migrate_indexes",false]],"module":[[0,"module-parsedmarc",false],[0,"module-parsedmarc.elastic",false],[0,"module-parsedmarc.opensearch",false],[0,"module-parsedmarc.splunk",false],[0,"module-parsedmarc.types",false],[0,"module-parsedmarc.utils",false]],"opensearcherror":[[0,"parsedmarc.opensearch.OpenSearchError",false]],"parse_aggregate_report_file() (in module parsedmarc)":[[0,"parsedmarc.parse_aggregate_report_file",false]],"parse_aggregate_report_xml() (in module parsedmarc)":[[0,"parsedmarc.parse_aggregate_report_xml",false]],"parse_email() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.parse_email",false]],"parse_forensic_report() (in module parsedmarc)":[[0,"parsedmarc.parse_forensic_report",false]],"parse_report_email() (in module parsedmarc)":[[0,"parsedmarc.parse_report_email",false]],"parse_report_file() (in module parsedmarc)":[[0,"parsedmarc.parse_report_file",false]],"parse_smtp_tls_report_json() (in module parsedmarc)":[[0,"parsedmarc.parse_smtp_tls_report_json",false]],"parsed_aggregate_reports_to_csv() (in module parsedmarc)":[[0,"parsedmarc.parsed_aggregate_reports_to_csv",false]],"parsed_aggregate_reports_to_csv_rows() (in module parsedmarc)":[[0,"parsedmarc.parsed_aggregate_reports_to_csv_rows",false]],"parsed_forensic_reports_to_csv() (in module parsedmarc)":[[0,"parsedmarc.parsed_forensic_reports_to_csv",false]],"parsed_forensic_reports_to_csv_rows() (in module parsedmarc)":[[0,"parsedmarc.parsed_forensic_reports_to_csv_rows",false]],"parsed_smtp_tls_reports_to_csv() (in module parsedmarc)":[[0,"parsedmarc.parsed_smtp_tls_reports_to_csv",false]],"parsed_smtp_tls_reports_to_csv_rows() (in module parsedmarc)":[[0,"parsedmarc.parsed_smtp_tls_reports_to_csv_rows",false]],"parsedemail (class in parsedmarc.types)":[[0,"parsedmarc.types.ParsedEmail",false]],"parsedmarc":[[0,"module-parsedmarc",false]],"parsedmarc.elastic":[[0,"module-parsedmarc.elastic",false]],"parsedmarc.opensearch":[[0,"module-parsedmarc.opensearch",false]],"parsedmarc.splunk":[[0,"module-parsedmarc.splunk",false]],"parsedmarc.types":[[0,"module-parsedmarc.types",false]],"parsedmarc.utils":[[0,"module-parsedmarc.utils",false]],"parsererror":[[0,"parsedmarc.ParserError",false]],"parsingresults (class in parsedmarc.types)":[[0,"parsedmarc.types.ParsingResults",false]],"query_dns() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.query_dns",false]],"reversednsservice (class in parsedmarc.utils)":[[0,"parsedmarc.utils.ReverseDNSService",false]],"save_aggregate_report_to_elasticsearch() (in module parsedmarc.elastic)":[[0,"parsedmarc.elastic.save_aggregate_report_to_elasticsearch",false]],"save_aggregate_report_to_opensearch() (in module parsedmarc.opensearch)":[[0,"parsedmarc.opensearch.save_aggregate_report_to_opensearch",false]],"save_aggregate_reports_to_splunk() (parsedmarc.splunk.hecclient method)":[[0,"parsedmarc.splunk.HECClient.save_aggregate_reports_to_splunk",false]],"save_forensic_report_to_elasticsearch() (in module parsedmarc.elastic)":[[0,"parsedmarc.elastic.save_forensic_report_to_elasticsearch",false]],"save_forensic_report_to_opensearch() (in module parsedmarc.opensearch)":[[0,"parsedmarc.opensearch.save_forensic_report_to_opensearch",false]],"save_forensic_reports_to_splunk() (parsedmarc.splunk.hecclient method)":[[0,"parsedmarc.splunk.HECClient.save_forensic_reports_to_splunk",false]],"save_output() (in module parsedmarc)":[[0,"parsedmarc.save_output",false]],"save_smtp_tls_report_to_elasticsearch() (in module parsedmarc.elastic)":[[0,"parsedmarc.elastic.save_smtp_tls_report_to_elasticsearch",false]],"save_smtp_tls_report_to_opensearch() (in module parsedmarc.opensearch)":[[0,"parsedmarc.opensearch.save_smtp_tls_report_to_opensearch",false]],"save_smtp_tls_reports_to_splunk() (parsedmarc.splunk.hecclient method)":[[0,"parsedmarc.splunk.HECClient.save_smtp_tls_reports_to_splunk",false]],"set_hosts() (in module parsedmarc.elastic)":[[0,"parsedmarc.elastic.set_hosts",false]],"set_hosts() (in module parsedmarc.opensearch)":[[0,"parsedmarc.opensearch.set_hosts",false]],"smtptlsfailuredetails (class in parsedmarc.types)":[[0,"parsedmarc.types.SMTPTLSFailureDetails",false]],"smtptlsfailuredetailsoptional (class in parsedmarc.types)":[[0,"parsedmarc.types.SMTPTLSFailureDetailsOptional",false]],"smtptlsparsedreport (class in parsedmarc.types)":[[0,"parsedmarc.types.SMTPTLSParsedReport",false]],"smtptlspolicy (class in parsedmarc.types)":[[0,"parsedmarc.types.SMTPTLSPolicy",false]],"smtptlspolicysummary (class in parsedmarc.types)":[[0,"parsedmarc.types.SMTPTLSPolicySummary",false]],"smtptlsreport (class in parsedmarc.types)":[[0,"parsedmarc.types.SMTPTLSReport",false]],"splunkerror":[[0,"parsedmarc.splunk.SplunkError",false]],"timestamp_to_datetime() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.timestamp_to_datetime",false]],"timestamp_to_human() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.timestamp_to_human",false]],"watch_inbox() (in module parsedmarc)":[[0,"parsedmarc.watch_inbox",false]]},"objects":{"":[[0,0,0,"-","parsedmarc"]],"parsedmarc":[[0,1,1,"","InvalidAggregateReport"],[0,1,1,"","InvalidDMARCReport"],[0,1,1,"","InvalidForensicReport"],[0,1,1,"","InvalidSMTPTLSReport"],[0,1,1,"","ParserError"],[0,0,0,"-","elastic"],[0,2,1,"","email_results"],[0,2,1,"","extract_report"],[0,2,1,"","extract_report_from_file_path"],[0,2,1,"","get_dmarc_reports_from_mailbox"],[0,2,1,"","get_dmarc_reports_from_mbox"],[0,2,1,"","get_report_zip"],[0,0,0,"-","opensearch"],[0,2,1,"","parse_aggregate_report_file"],[0,2,1,"","parse_aggregate_report_xml"],[0,2,1,"","parse_forensic_report"],[0,2,1,"","parse_report_email"],[0,2,1,"","parse_report_file"],[0,2,1,"","parse_smtp_tls_report_json"],[0,2,1,"","parsed_aggregate_reports_to_csv"],[0,2,1,"","parsed_aggregate_reports_to_csv_rows"],[0,2,1,"","parsed_forensic_reports_to_csv"],[0,2,1,"","parsed_forensic_reports_to_csv_rows"],[0,2,1,"","parsed_smtp_tls_reports_to_csv"],[0,2,1,"","parsed_smtp_tls_reports_to_csv_rows"],[0,2,1,"","save_output"],[0,0,0,"-","splunk"],[0,0,0,"-","types"],[0,0,0,"-","utils"],[0,2,1,"","watch_inbox"]],"parsedmarc.elastic":[[0,1,1,"","AlreadySaved"],[0,1,1,"","ElasticsearchError"],[0,2,1,"","create_indexes"],[0,2,1,"","migrate_indexes"],[0,2,1,"","save_aggregate_report_to_elasticsearch"],[0,2,1,"","save_forensic_report_to_elasticsearch"],[0,2,1,"","save_smtp_tls_report_to_elasticsearch"],[0,2,1,"","set_hosts"]],"parsedmarc.opensearch":[[0,1,1,"","AlreadySaved"],[0,1,1,"","OpenSearchError"],[0,2,1,"","create_indexes"],[0,2,1,"","migrate_indexes"],[0,2,1,"","save_aggregate_report_to_opensearch"],[0,2,1,"","save_forensic_report_to_opensearch"],[0,2,1,"","save_smtp_tls_report_to_opensearch"],[0,2,1,"","set_hosts"]],"parsedmarc.splunk":[[0,3,1,"","HECClient"],[0,1,1,"","SplunkError"]],"parsedmarc.splunk.HECClient":[[0,4,1,"","close"],[0,4,1,"","save_aggregate_reports_to_splunk"],[0,4,1,"","save_forensic_reports_to_splunk"],[0,4,1,"","save_smtp_tls_reports_to_splunk"]],"parsedmarc.types":[[0,3,1,"","AggregateAlignment"],[0,3,1,"","AggregateAuthResultDKIM"],[0,3,1,"","AggregateAuthResultSPF"],[0,3,1,"","AggregateAuthResults"],[0,3,1,"","AggregateIdentifiers"],[0,3,1,"","AggregateParsedReport"],[0,3,1,"","AggregatePolicyEvaluated"],[0,3,1,"","AggregatePolicyOverrideReason"],[0,3,1,"","AggregatePolicyPublished"],[0,3,1,"","AggregateRecord"],[0,3,1,"","AggregateReport"],[0,3,1,"","AggregateReportMetadata"],[0,3,1,"","EmailAddress"],[0,3,1,"","EmailAttachment"],[0,3,1,"","ForensicParsedReport"],[0,3,1,"","ForensicReport"],[0,3,1,"","IPSourceInfo"],[0,3,1,"","ParsedEmail"],[0,3,1,"","ParsingResults"],[0,3,1,"","SMTPTLSFailureDetails"],[0,3,1,"","SMTPTLSFailureDetailsOptional"],[0,3,1,"","SMTPTLSParsedReport"],[0,3,1,"","SMTPTLSPolicy"],[0,3,1,"","SMTPTLSPolicySummary"],[0,3,1,"","SMTPTLSReport"]],"parsedmarc.utils":[[0,1,1,"","DownloadError"],[0,1,1,"","EmailParserError"],[0,3,1,"","IPAddressInfo"],[0,1,1,"","InvalidIPinfoAPIKey"],[0,3,1,"","ReverseDNSService"],[0,2,1,"","configure_ipinfo_api"],[0,2,1,"","convert_outlook_msg"],[0,2,1,"","decode_base64"],[0,2,1,"","get_base_domain"],[0,2,1,"","get_filename_safe_string"],[0,2,1,"","get_ip_address_country"],[0,2,1,"","get_ip_address_db_record"],[0,2,1,"","get_ip_address_info"],[0,2,1,"","get_reverse_dns"],[0,2,1,"","get_service_from_reverse_dns_base_domain"],[0,2,1,"","human_timestamp_to_datetime"],[0,2,1,"","human_timestamp_to_unix_timestamp"],[0,2,1,"","is_mbox"],[0,2,1,"","is_outlook_msg"],[0,2,1,"","load_ip_db"],[0,2,1,"","load_psl_overrides"],[0,2,1,"","load_reverse_dns_map"],[0,2,1,"","parse_email"],[0,2,1,"","query_dns"],[0,2,1,"","timestamp_to_datetime"],[0,2,1,"","timestamp_to_human"]]},"objnames":{"0":["py","module","Python module"],"1":["py","exception","Python exception"],"2":["py","function","Python function"],"3":["py","class","Python class"],"4":["py","method","Python method"]},"objtypes":{"0":"py:module","1":"py:exception","2":"py:function","3":"py:class","4":"py:method"},"terms":{"":[0,2,3,4,6,7,8,10,12],"0":[0,2,3,4,5,6,8,9,10,11,12],"00":10,"003":10,"00z":10,"00z_exampl":10,"01":10,"0200":10,"0240":10,"04":10,"08":10,"09":10,"09t00":10,"09t23":10,"1":[0,2,4,5,10,12],"10":[0,5,6,10,12],"100":[10,12],"1000":12,"11":[5,6,10],"1143":2,"12":[5,6],"12201":12,"127":[2,4,12],"13":5,"14":5,"150":10,"16":[3,8],"173":10,"176":10,"19":[10,12],"1d":12,"1g":4,"1w":12,"2":[0,4,10,12],"20":10,"2000":12,"201":10,"2010":[6,10],"2012":10,"2013":6,"2016":6,"2017a":[3,8],"2018":10,"2019":6,"2024":10,"2028":5,"2030":5,"2035":5,"208":10,"209":10,"21":6,"212":10,"22":6,"222":10,"23":10,"2369":[3,8],"24":0,"241":10,"25":12,"27":10,"28":10,"2919":[3,8],"2d":12,"2k":12,"3":[0,5,6,10,11,12],"30":[0,12],"300":2,"30937":10,"3128":6,"365":[2,4],"38":10,"3d":10,"3h":12,"4":[4,6,11],"401":0,"403":0,"4096":4,"41":10,"5":[2,4,9,12],"500":12,"514":12,"5601":4,"59":10,"59z":10,"5m":[2,12],"6":[0,4,5,12],"60":[0,12],"6514":12,"660":4,"7":[4,5],"7018":10,"72":10,"7480":10,"7d":12,"8":[2,4,5,7,10,12],"8080":12,"822":0,"85":10,"86399":10,"86400":10,"9":[5,12],"9200":[4,12],"932":12,"9391651994964116463":10,"94":10,"993":12,"A":[0,3,7,12],"AT":10,"And":0,"As":[4,7],"By":[7,12],"For":[4,12],"If":[0,3,4,6,7,8,12],"In":[2,3,7,8,12],"It":[2,4,7,10,12],"No":[3,6,8],"On":[3,4,6,7,8,12],"Or":[4,6],"That":7,"The":[0,3,6,7,11,12],"Then":[2,3,4,6,8,12],"There":7,"These":[7,12],"To":[2,4,6,7,9,10,12],"With":7,"_":12,"_attempt":0,"_cluster":12,"_input":0,"_ipdatabaserecord":0,"abl":6,"abort":12,"about":[0,5],"abov":[2,6,12],"accept":[0,3,4,7,8,12],"access":[0,4,5,12],"access_key_id":12,"access_token":0,"accessright":12,"accident":[3,8],"account":[6,7,12],"acm":10,"acquir":12,"across":7,"action":[3,8],"activ":[4,5,12],"active_primary_shard":12,"active_shard":12,"actual":[3,10],"ad":[3,8,12],"add":[2,3,4,6,7,8,12],"addit":[3,8,12],"address":[0,2,3,4,7,8,10,12],"addresse":7,"adkim":10,"admin":[3,8,12],"administr":[3,8],"after":[0,2,4,12],"against":[3,8],"agari":5,"agent":4,"aggreg":[0,5,11,12],"aggregate_csv_filenam":[0,12],"aggregate_index":0,"aggregate_json_filenam":[0,12],"aggregate_report":0,"aggregate_top":12,"aggregate_url":12,"aggregatealign":0,"aggregateauthresult":0,"aggregateauthresultdkim":0,"aggregateauthresultspf":0,"aggregateidentifi":0,"aggregateparsedreport":0,"aggregatepolicyevalu":0,"aggregatepolicyoverridereason":0,"aggregatepolicypublish":0,"aggregaterecord":0,"aggregatereport":0,"aggregatereportmetadata":0,"aggress":12,"alia":12,"align":[5,7,10],"aliv":0,"all":[3,7,8,11,12],"allow":[2,3,8,12],"allow_unencrypted_storag":12,"allowremot":2,"alreadi":12,"alreadysav":0,"also":[0,2,3,6,7,8,12],"alter":[3,8],"altern":[5,12],"although":11,"alwai":[0,2,4,12],"always_use_local_fil":[0,12],"amount":12,"an":[0,3,5,7,8,10,12],"analyt":12,"analyz":12,"ani":[0,3,6,7,8,12],"anonym":10,"anoth":[6,12],"answer":[0,12],"apach":5,"api":[2,4,5,12],"api_kei":[0,12],"app":12,"appear":12,"appendix":10,"appid":12,"appli":12,"applic":12,"applicationaccesspolici":12,"approach":12,"approxim":2,"apt":[2,4,6],"ar":[0,2,3,4,5,6,7,8,10,12],"archiv":[0,12],"archive_fold":[0,12],"argument":12,"arriv":12,"arrival_d":10,"arrival_date_utc":10,"artifact":4,"as_domain":[0,10],"as_nam":[0,10],"ask":3,"asmx":2,"asn":[0,10,12],"aspf":10,"assign":4,"associ":0,"assum":12,"att":10,"attach":[0,3,8,10,12],"attachment_filenam":0,"attempt":[0,12],"attribut":6,"auth":[0,2,10,12],"auth_failur":10,"auth_method":12,"auth_mod":12,"auth_result":10,"auth_typ":[0,12],"authent":[0,2,3,4,7,12],"authentication_mechan":10,"authentication_result":10,"authentication_typ":12,"auto":2,"automat":[6,12],"avail":6,"avoid":[7,12],"aw":[0,12],"aws_region":[0,12],"aws_servic":[0,12],"awssigv4":[0,12],"azur":12,"b":10,"b2c":7,"back":[0,12],"backfil":12,"backlog":12,"backward":12,"base":[0,2,3,4,6,7,8,10],"base64":0,"base_domain":[0,10],"basic":[0,2,12],"batch":12,"batch_siz":[0,12],"bcc":[0,10],"bd6e1bb5":10,"becaus":[2,3,7,8,12],"becom":12,"been":[0,12],"befor":[0,12],"begin_d":10,"behavior":0,"behind":6,"being":0,"below":[3,6,8,12],"benefit":5,"best":7,"between":[0,4,7,12],"beyond":0,"bin":[2,4,6,12],"binari":0,"binaryio":0,"bind":2,"bindaddress":2,"blank":[3,8],"block":[2,12],"bodi":[0,3,8,10,12],"bool":[0,12],"both":12,"brand":[5,7],"break":[3,4,8],"broken":0,"browser":4,"bucket":12,"budget":0,"bug":5,"build":6,"built":0,"bundl":[0,6,7,12],"busi":7,"button":[3,8],"byte":0,"c":[10,12],"ca":[4,12],"cach":[0,12],"cafile_path":12,"call":12,"callabl":0,"callback":0,"came":[3,8],"can":[0,2,3,4,5,6,7,8,12],"cannot":12,"cap":[0,12],"carri":0,"case":[2,3,8],"catch":[0,12],"caught":0,"caus":[3,4,7,8],"cc":[0,10],"center":7,"cento":[4,6],"cert":[4,12],"cert_path":12,"certain":[0,12],"certfile_path":12,"certif":[0,4,7,12],"certificate_password":12,"certificate_path":12,"cest":10,"chain":0,"chang":[4,7,11,12],"charact":[2,12],"charset":10,"chart":7,"check":[0,2,3,4,7,12],"check_timeout":[0,12],"checkbox":4,"checkdmarc":3,"chines":7,"chmod":[2,4,12],"choos":[3,8],"chown":[2,12],"ci":7,"cisco":12,"class":0,"clear":0,"cli":[0,5],"click":[4,7],"client":[2,3,4,8,12],"client_id":12,"client_secret":12,"clientsecret":12,"clientsotimeout":2,"close":[0,12],"cloud":12,"cloudflar":[0,12],"cluster":[4,12],"co":4,"code":[0,4,12],"collect":[7,12],"collector":[11,12],"com":[1,2,3,8,9,10,12],"come":[0,7],"comma":[6,12],"command":[2,3,6,8,12],"comment":12,"commerci":[4,5],"common":[3,4,6,8],"commun":[3,8],"compat":[7,12],"complet":[3,4,12],"compli":[3,4,6,8,9],"compliant":[3,8],"compon":6,"compress":5,"conf":6,"config":[0,2,6],"config_fil":12,"config_reload":0,"configur":[0,3,4,5,6,7,8,9],"configure_ipinfo_api":0,"conform":4,"connect":[0,2,4,12],"connexion":4,"consid":[5,7],"consist":[0,5,10],"consol":[4,12],"constant":0,"consum":[7,12],"contact":7,"contain":[0,7,11,12],"content":[0,3,8,10,11],"contrib":6,"contribut":5,"control":4,"convent":12,"convert":[0,3,8],"convert_outlook_msg":0,"copi":[0,6,11],"core":[3,8],"correct":6,"correctli":[0,7,12],"could":[3,4,8,12],"count":[2,7,10],"countri":[0,7,10,12],"country_cod":0,"cpan":6,"crash":[2,4,12],"creat":[0,2,3,4,6,8,12],"create_fold":0,"create_index":0,"creativ":6,"credenti":[6,12],"credentials_fil":12,"cron":6,"cross":0,"crt":4,"csr":4,"csv":[0,5,12],"cumul":6,"current":[2,4,12],"custom":[7,12],"d":[0,4,12],"daemon":[2,4,12],"dai":[0,4,9,12],"daili":[0,12],"dashboard":[4,5,9,11],"dat":0,"data":[0,4,5,6,7,9,11,12],"databas":[0,12],"date":[0,3,8,10],"date_utc":10,"datetim":0,"davmail":5,"db_path":0,"dbip":[0,12],"dce":12,"dcr":12,"dcr_aggregate_stream":12,"dcr_forensic_stream":12,"dcr_immutable_id":12,"dcr_smtp_tls_stream":12,"dd":0,"de":10,"dearmor":4,"deb":4,"debian":[4,5,6],"debug":12,"decemb":6,"decod":0,"decode_base64":0,"dedic":6,"default":[0,2,4,5,6,7,12],"defens":5,"delai":[2,10,12],"deleg":12,"delegated_us":12,"delet":[0,2,4,12],"delivery_result":10,"demystifi":3,"depend":[0,4,5,12],"deploi":[3,8],"deploy":12,"deprec":12,"describ":12,"descript":[2,6,12],"destin":[0,12],"detail":[6,7,12],"dev":[6,12],"devel":6,"develop":5,"devicecod":12,"di":10,"dict":0,"dictionari":0,"differ":[7,12],"difficult":12,"dig":0,"digest":[3,8],"dir":6,"directli":7,"directori":[0,6,12],"disabl":[0,2,12],"disclaim":[3,8],"disk":12,"displai":[3,7,11],"display_nam":10,"disposit":[7,10],"distribut":6,"dkim":[5,7,8,10],"dkim_align":10,"dkim_domain":10,"dkim_result":10,"dkim_selector":10,"dkm":3,"dmarc":[0,4,6,8,9,10,11,12],"dmarc_aggreg":4,"dmarc_align":10,"dmarc_forens":4,"dmarc_moderation_act":[3,8],"dmarc_none_moderation_act":[3,8],"dmarc_quarantine_moderation_act":[3,8],"dmarcian":5,"dmarcresport":12,"dn":[0,3,7,12],"dnf":6,"dns_retri":0,"dns_test_address":12,"dns_timeout":[0,12],"dnspython":0,"do":[0,2,6,7,12],"doc":[9,12],"doctyp":10,"document":[2,12],"doe":[3,8],"domain":[0,4,7,8,10,12],"domainawar":[1,3,12],"don":3,"down":[7,12],"download":[0,2,4,6,12],"downloaderror":0,"draft":[5,10],"dtd":10,"dummi":12,"dure":[2,12],"e":[0,2,3,4,6,8,12],"e7":10,"each":[0,4,6,9,11,12],"earlier":7,"easi":[4,9],"easier":[11,12],"echo":4,"edit":[2,6,12],"editor":11,"effect":12,"effici":4,"either":[5,12],"elast":[4,5],"elasticsearch":[0,5,12],"elasticsearcherror":0,"elk":12,"els":4,"email":[0,3,5,6,7,8,10,11,12],"email_result":0,"emailaddress":0,"emailattach":0,"emailparsererror":0,"empti":[0,3,8],"en":[3,4,8,10],"enabl":[2,4,12],"enableew":2,"enablekeepal":2,"enableproxi":2,"encod":[0,10,12],"encount":0,"encrypt":[4,12],"encryptedsavedobject":4,"encryptionkei":4,"end":[3,4,5],"end_dat":10,"endpoint":12,"endpoint_url":12,"enforc":[3,8],"enough":12,"enrol":4,"ensur":[3,6,8],"entir":[3,7,8,12],"entri":0,"envelop":3,"envelope_from":10,"envelope_to":10,"environ":[5,6],"eol":5,"epel":6,"equival":6,"error":[0,7,10,12],"escap":12,"especi":[7,12],"etc":[2,3,4,6,8,12],"even":[2,3,8,12],"event":[2,11,12],"everi":[0,2,6,7,12],"ew":5,"ex":12,"exactli":[3,8],"exampl":[3,4,6,8,10],"except":[0,12],"exchang":[2,10,12],"exclud":2,"execreload":12,"execstart":[2,12],"exhaust":12,"exist":[0,3,4,8,12],"exit":[0,12],"expiri":7,"expiringdict":0,"explain":[3,8],"explicit":[3,8],"export":[4,7,12],"extract":[0,2],"extract_report":0,"extract_report_from_file_path":0,"ey":[2,12],"f":4,"factor":2,"fail":[0,3,7,8,10,12],"fail_on_output_error":12,"failed_session_count":10,"failov":0,"failur":[0,5,10,12],"failure_detail":10,"fall":[0,12],"fallback":0,"fals":[0,2,10,12],"fantast":[3,8],"faster":12,"fatal":[0,12],"featur":[4,12],"feedback":0,"feedback_report":0,"feedback_typ":10,"fetch":[0,7,12],"few":[7,12],"field":[0,4],"file":[0,2,5,6,7,11],"file_path":[0,12],"filenam":[0,12],"filename_safe_subject":10,"filepath":12,"fill":[4,6],"filter":[3,7,8,11],"financ":12,"find":[3,7,8,12],"fine":[3,8],"finish":12,"first":[0,3,6,8,12],"first_strip_reply_to":[3,8],"fit":[3,8,12],"fix":4,"flag":[0,2,6,12],"flat":0,"flexibl":11,"flight":12,"float":[0,12],"fo":10,"fold":0,"folder":[0,2,12],"foldersizelimit":2,"follow":[2,4,5,12],"footer":[3,8],"forens":[0,5,7,11,12],"forensic_csv_filenam":[0,12],"forensic_index":0,"forensic_json_filenam":[0,12],"forensic_report":0,"forensic_top":12,"forensic_url":12,"forensicparsedreport":0,"forensicreport":0,"format":[0,7,12],"formerli":7,"forward":[3,7,8],"found":[0,6,12],"foundat":10,"fqdn":4,"fraud":5,"free":[6,12],"freshest":12,"friendli":7,"from":[0,2,3,4,5,6,7,8,10,12],"from_is_list":[3,8],"ftp_proxi":6,"full":12,"fulli":[3,8,12],"function":0,"further":7,"g":[0,2,3,4,6,8,12],"gatewai":2,"gb":4,"gcc":6,"gdpr":[4,9],"gelf":12,"gener":[3,4,8,10,12],"geoip":[6,12],"geoipupd":6,"geolite2":5,"geoloc":[0,12],"get":[0,2,4,6,12],"get_base_domain":0,"get_dmarc_reports_from_mailbox":0,"get_dmarc_reports_from_mbox":0,"get_filename_safe_str":0,"get_ip_address_countri":0,"get_ip_address_db_record":0,"get_ip_address_info":0,"get_report_zip":0,"get_reverse_dn":0,"get_service_from_reverse_dns_base_domain":0,"github":[1,6,10,12],"give":[0,4],"given":[0,12],"glass":7,"gmail":[5,7,12],"gmail_api":12,"go":[3,8],"goe":[3,8],"googl":[7,12],"googleapi":12,"got":12,"gov":12,"gpg":4,"grafana":5,"grant":12,"graph":[2,5,7,12],"graph_url":12,"group":[2,7,12],"guid":[4,5],"guidanc":12,"gzip":[0,5],"h":[0,12],"ha":[0,4,6,12],"hamburg":4,"hand":[3,8],"handl":[5,12],"handler":7,"happen":0,"hard":12,"has_defect":10,"have":[3,4,6,7,8,11,12],"head":10,"header":[0,3,7,8,10,12],"header_from":10,"headless":2,"health":12,"healthcar":12,"heap":4,"heavi":[4,12],"hec":[0,11,12],"hecclient":0,"hectokengoesher":12,"help":5,"here":[0,3,8,10],"hh":0,"hi":[3,8],"high":[7,12],"higher":[3,8],"highli":12,"histori":12,"hit":[0,12],"home":6,"hop":10,"host":[0,2,3,4,5,8,12],"hostnam":[0,12],"hour":[0,12],"hover":7,"href":10,"html":[3,4,8,10],"http":[0,1,2,3,4,6,8,9,10,11,12],"http_proxi":6,"https_proxi":6,"human":[0,7],"human_timestamp":0,"human_timestamp_to_datetim":0,"human_timestamp_to_unix_timestamp":0,"hup":12,"i":[0,2,3,4,5,6,7,8,10,12],"icon":7,"id":[3,8,10,12],"ideal":[3,8],"ident":[3,8,12],"identifi":10,"idl":[0,2,12],"ignor":12,"imag":12,"imap":[0,2,5,12],"imap_password":12,"imapalwaysapproxmsgs":2,"imapautoexpung":2,"imapcli":5,"imapidledelai":2,"imapport":2,"immedi":2,"immut":12,"impli":12,"import":[4,7,12],"improv":12,"inbox":[0,3,5,8,12],"inc":10,"includ":[0,3,6,8,12],"include_list_post_head":[3,8],"include_rfc2369_head":[3,8],"include_sender_head":[3,8],"include_spam_trash":12,"incom":[7,12],"incorrect":12,"increas":[4,12],"increment":12,"indent":12,"index":[0,5,9,11,12],"index_prefix":[0,12],"index_prefix_domain_map":12,"index_suffix":[0,12],"indic":[3,5],"individu":12,"industri":12,"inform":[0,4,7,12],"infrequ":12,"ingest":12,"ini":[2,12],"initi":0,"input":0,"input_":0,"instal":[2,5,12],"installed_app":12,"instanc":12,"instead":[0,3,6,8,12],"int":[0,12],"intend":[3,8],"interact":[2,4],"interakt":10,"interfer":[3,8],"interpret":6,"interrupt":12,"interv":12,"interval_begin":10,"interval_end":10,"invalid":[0,12],"invalidaggregatereport":0,"invaliddmarcreport":0,"invalidforensicreport":0,"invalidipinfoapikei":0,"invalidsmtptlsreport":0,"involv":7,"io":[0,12],"ip":[0,3,4,7,12],"ip_address":[0,10],"ip_db_path":[0,12],"ip_db_url":12,"ipaddressinfo":0,"ipinfo":[0,6,12],"ipinfo_api_token":12,"ipinfo_url":12,"ipsourceinfo":0,"ipv4":0,"ipv6":0,"is_mbox":0,"is_outlook_msg":0,"iso":0,"issu":1,"its":[6,12],"java":2,"job":[3,6,8],"joe":[3,8],"journalctl":[2,12],"jre":2,"json":[0,5,12],"june":5,"just":7,"jvm":4,"kafka":[5,12],"kb4099855":6,"kb4134118":6,"kb4295699":6,"keep":[0,12],"keep_al":0,"keepal":2,"kei":[0,3,4,6,12],"keyfile_path":12,"keyout":4,"keyr":4,"keystor":4,"kibana":[5,11],"kill":12,"kind":12,"know":3,"known":[3,7,8,12],"kwarg":0,"label":12,"languag":[3,8],"larg":[2,12],"larger":12,"last":6,"later":[4,6,12],"latest":[2,4,9,12],"layer":0,"layout":11,"leak":7,"least":[4,6,12],"leav":3,"left":7,"legal":[3,8],"legitim":[7,12],"less":12,"level":[0,3,4,12],"lib":6,"libemail":6,"libxml2":6,"libxslt":6,"licens":6,"life":5,"lifetim":0,"lifetimetimeout":0,"like":[0,3,6,8,12],"limit":[0,2,12],"line":[3,8,12],"link":[3,4,7,8],"linux":[3,6,8],"list":[0,2,4,5,7,12],"listen":[2,12],"lite":[0,6,12],"live":7,"ll":[3,8],"load":[0,4,12],"load_ip_db":0,"load_psl_overrid":0,"load_reverse_dns_map":0,"local":[0,2,4,6,10,12],"local_file_path":0,"local_psl_overrides_path":12,"local_reverse_dns_map_path":12,"localhost":12,"locat":[6,7,12],"log":[0,2,12],"log_analyt":12,"log_fil":12,"logger":12,"login":4,"logstash":4,"long":[3,12],"longer":[3,8],"look":[0,3,7],"lookup":[0,12],"loopback":2,"loss":0,"lot":7,"low":12,"lower":12,"lua":10,"m":[0,6,10,12],"m365":12,"maco":6,"magnifi":7,"mai":[5,7,12],"maidir":12,"mail":[0,5,6,10,12],"mail_bcc":0,"mail_cc":0,"mail_from":0,"mail_to":0,"mailbox":[0,7,12],"mailbox_connect":0,"mailboxconnect":0,"maildir":12,"maildir_cr":12,"maildir_path":12,"mailer":10,"mailrelai":10,"mailto":6,"main":4,"mainpid":12,"maintain":5,"make":[0,3,4,6,8,9,12],"malici":[7,12],"manag":[4,7,12],"manual":12,"map":0,"market":7,"massiv":12,"match":[0,4,11,12],"max_ag":10,"max_shards_per_nod":12,"maximum":4,"maxmind":[0,5,12],"mbox":[0,12],"mean":12,"mechan":3,"member":[3,8],"memori":12,"mention":7,"menu":[4,7],"messag":[0,2,3,4,6,7,8,10,12],"message_id":10,"meta":10,"method":12,"mfrom":10,"microsoft":[2,5,10,12],"might":[0,3,7,8],"migrat":7,"migrate_index":0,"mime":10,"min":0,"minimum":4,"minut":[0,2,12],"mirror":0,"miss":12,"mitig":[3,8],"mix":0,"mm":0,"mmdb":[0,6,12],"mobil":[3,8],"mode":[0,2,4,6,10],"modern":[2,3,8],"modifi":[0,3,8,12],"modul":[0,5,6,12],"mon":10,"monitor":[3,12],"monthli":[0,12],"monthly_index":[0,12],"more":[0,4,6,11,12],"most":[3,4,6,7,8,12],"mous":7,"move":[0,4,12],"msg":[0,6],"msg_byte":0,"msg_date":0,"msg_footer":[3,8],"msg_header":[3,8],"msgconvert":[0,6],"msgraph":12,"mta":7,"much":12,"multi":[2,5],"multipl":[0,12],"mung":[3,8],"must":[2,3,8,12],"mutual":[4,12],"mv":4,"mx":10,"my":[5,12],"n":[10,12],"n_proc":12,"name":[0,3,4,7,10,11],"nameserv":[0,12],"nano":[2,12],"nation":12,"navig":[3,8],"ncontent":10,"ndate":10,"ndjson":[4,7],"need":[2,3,4,6,7,8,12],"neither":12,"nelson":[3,8],"net":[2,12],"network":[0,2,4,12],"new":[0,2,3,6,7,12],"newer":6,"newest":[2,12],"newkei":4,"next":[0,12],"nfrom":10,"nmessag":10,"nmime":10,"node":4,"nologin":6,"non":[0,3,8,12],"nonameserv":0,"none":[0,3,10,12],"noproxyfor":2,"nor":12,"norepli":[3,10],"normal":[0,10,12],"normalize_timespan_threshold_hour":0,"normalized_timespan":10,"nosecureimap":2,"notabl":7,"note":12,"notic":12,"now":[4,7],"nsubject":10,"nto":10,"null":[6,10],"number":[0,12],"number_of_replica":[0,12],"number_of_shard":[0,12],"nwettbewerb":10,"nx":10,"o":[0,2,4,12],"oR":6,"oauth2":12,"oauth2_port":12,"object":[0,4,7],"observ":7,"occur":[0,7],"occurr":11,"oct":10,"offic":2,"office365":2,"offlin":[0,6,12],"often":[7,12],"ol":[0,6],"old":7,"older":[6,10],"oldest":[2,12],"onc":12,"ondmarc":5,"one":[0,3,5,6,8,12],"ones":12,"onli":[0,2,3,6,7,8],"onlin":[0,2,12],"oor":0,"open":3,"opendn":12,"opensearch":[5,7,12],"opensearch_dashboard":7,"opensearcherror":0,"openssl":4,"oper":12,"opt":[2,6,12],"option":[0,2,3,4,5,8,11,12],"order":12,"org":[0,6,9,10,12],"org_email":10,"org_extra_contact_info":10,"org_nam":10,"organ":[2,5,7,12],"organization_nam":10,"origin":[3,8,12],"original_envelope_id":10,"original_mail_from":10,"original_rcpt_to":10,"original_timespan_second":10,"oserror":0,"other":[0,3,4,7,8],"otherwis":12,"our":7,"out":[3,4,7],"outdat":7,"outgo":[3,8,12],"outlook":[0,2,6],"output":[0,5,12],"output_directori":0,"outsid":12,"over":[0,2,5,6,7,12],"overal":0,"overrid":[0,12],"overwrit":4,"owa":5,"own":[6,7,11],"ownership":6,"p":[3,10],"p12":4,"pack":4,"packag":[0,4,6],"packet":0,"pad":0,"page":[3,4,6,7,8],"paginate_messag":12,"pan":10,"parallel":12,"paramet":0,"parent":7,"pars":[0,3,5,6,10,12],"parse_aggregate_report_fil":0,"parse_aggregate_report_xml":0,"parse_email":0,"parse_forensic_report":0,"parse_report_email":0,"parse_report_fil":0,"parse_smtp_tls_report_json":0,"parsed_aggregate_reports_to_csv":0,"parsed_aggregate_reports_to_csv_row":0,"parsed_forensic_reports_to_csv":0,"parsed_forensic_reports_to_csv_row":0,"parsed_sampl":10,"parsed_smtp_tls_reports_to_csv":0,"parsed_smtp_tls_reports_to_csv_row":0,"parsedemail":0,"parsedmarc":[4,9,10,11],"parsedmarc_":12,"parsedmarc_config_fil":12,"parsedmarc_elasticsearch_":12,"parsedmarc_elasticsearch_host":12,"parsedmarc_elasticsearch_ssl":12,"parsedmarc_gelf_":12,"parsedmarc_general_":12,"parsedmarc_general_debug":12,"parsedmarc_general_ipinfo_api_token":12,"parsedmarc_general_ipinfo_url":12,"parsedmarc_general_offlin":12,"parsedmarc_general_save_aggreg":12,"parsedmarc_general_save_forens":12,"parsedmarc_gmail_api_":12,"parsedmarc_imap_":12,"parsedmarc_imap_host":12,"parsedmarc_imap_password":12,"parsedmarc_imap_us":12,"parsedmarc_kafka_":12,"parsedmarc_log_analytics_":12,"parsedmarc_mailbox_":12,"parsedmarc_mailbox_watch":12,"parsedmarc_maildir_":12,"parsedmarc_msgraph_":12,"parsedmarc_opensearch_":12,"parsedmarc_s3_":12,"parsedmarc_smtp_":12,"parsedmarc_splunk_hec_":12,"parsedmarc_splunk_hec_index":12,"parsedmarc_splunk_hec_token":12,"parsedmarc_splunk_hec_url":12,"parsedmarc_syslog_":12,"parsedmarc_webhook_":12,"parser":0,"parsererror":0,"parsingresult":0,"part":[3,4,7,8],"particular":7,"particularli":12,"pass":[0,3,7,10],"passag":7,"passsword":12,"password":[0,4,6,12],"past":[4,11],"patch":6,"path":[0,4,6,12],"pathlik":0,"pattern":[5,7],"payload":[0,12],"pct":10,"peak":12,"pem":12,"per":[0,7,12],"percentag":7,"perform":[2,5],"period":12,"perl":[0,6],"permiss":[4,12],"persist":12,"peter":10,"pick":[6,12],"pie":7,"pip":6,"pkcs12":12,"place":[0,4,7,12],"plain":0,"plaintext":[3,8],"platform":[3,6,8],"pleas":[1,5,12],"plu":7,"point":12,"polici":[3,7,8,10,12],"policy_domain":10,"policy_evalu":10,"policy_override_com":10,"policy_override_reason":10,"policy_publish":10,"policy_str":10,"policy_typ":10,"policyscopegroupid":12,"poll":[2,12],"popul":0,"port":[0,2,12],"posit":12,"possibl":12,"post":[3,8,12],"poster":[3,8],"postoriu":[3,8],"powershel":12,"ppa":6,"practic":12,"pre":[6,12],"predict":12,"prefer":[2,6,12],"prefix":[0,3,8,12],"premad":[5,11],"prerequisit":5,"present":12,"pressur":12,"pretti":12,"prettifi":12,"previou":[0,2,4,12],"previous":4,"primari":0,"print":12,"printabl":10,"prioriti":12,"privaci":[3,6,7,8,12],"privat":12,"probe":0,"process":[0,2,5,6,12],"produc":10,"program":12,"programdata":6,"progress":12,"project":[0,2,3,5,11],"prompt":4,"proofpoint":5,"properti":2,"protect":[2,3,5,8,12],"protocol":12,"provid":[0,4,7,12],"prox":6,"proxi":2,"proxyhost":2,"proxypassword":2,"proxyport":2,"proxyus":2,"pry":[2,12],"psl":[0,12],"psl_overrid":0,"psl_overrides_path":0,"psl_overrides_url":[0,12],"public":[0,3,10,12],"public_suffix_list":0,"publicbaseurl":4,"publicsuffix":0,"publish":[3,12],"put":[4,12],"python":[0,6],"python3":6,"qo":4,"quarantin":[3,8],"queri":[0,12],"query_dn":0,"quickli":0,"quickstart":12,"quot":10,"quota":[0,12],"r":[2,10,12],"rais":0,"ram":[4,12],"rate":[0,12],"rather":[3,8,12],"raw":12,"re":[6,12],"read":[0,12],"readabl":0,"readwrit":12,"realli":3,"reason":[0,2,4,5,12],"receiv":[0,7,10,12],"receiving_ip":10,"receiving_mx_hostnam":10,"recent":0,"recipi":7,"recogn":7,"recommend":12,"recommended_dns_nameserv":0,"record":[0,5,6,10],"record_typ":0,"reduc":12,"refer":[4,5],"referenc":12,"refresh":6,"regard":12,"regardless":10,"region":[0,12],"region_nam":12,"regist":6,"registr":12,"regul":[4,6,9,12],"regular":[3,8],"reject":[0,3,8],"relai":[3,8],"relat":[3,12],"releas":[4,6],"reli":7,"reliabl":12,"reload":[0,2,4],"remain":[7,12],"remot":2,"remov":[0,3,4,8,12],"repeat":[3,8],"replac":[0,3,4,8,12],"repli":[2,3,8],"replica":[0,12],"reply_goes_to_list":[3,8],"reply_to":10,"replyto":[3,8],"repopul":0,"report":[0,4,11,12],"report_id":10,"report_metadata":10,"report_typ":0,"reported_domain":10,"reports_fold":[0,12],"repositori":[6,11],"req":4,"request":[0,2,4,12],"requir":[0,2,3,4,5,6,8,12],"require_encrypt":0,"reserv":12,"resid":12,"resolv":[0,12],"resourc":[0,4,5,12],"respons":[0,12],"rest":[0,12],"restart":[2,3,4,6,8],"restartsec":[2,12],"restor":4,"restrict":12,"restrictaccess":12,"result":[0,5,7,10,12],"result_typ":10,"resum":12,"retain":[3,8,12],"retent":5,"retri":[0,12],"retriev":2,"retry_attempt":12,"retry_delai":12,"return":0,"revers":[0,7,12],"reverse_dn":[0,10],"reverse_dns_base_domain":0,"reverse_dns_map":0,"reverse_dns_map_path":0,"reverse_dns_map_url":[0,12],"reversednsservic":0,"review":7,"rewrit":[3,8],"rfc":[0,3,8,10],"rfc2369":[3,8],"rfc822":2,"rhel":[4,5,6],"right":[4,7],"rm":4,"ro":0,"rocki":6,"rollup":6,"root":[2,12],"rpm":4,"rpt":7,"rsa":4,"rua":[5,6],"ruf":[5,6,7,12],"rule":[7,12],"run":[0,4,5,6],"rw":[2,12],"s3":12,"safe":0,"safer":12,"same":[0,3,4,6,7,11,12],"sampl":[0,5,7,12],"sample_headers_onli":10,"save":[0,4,6,7,12],"save_aggreg":12,"save_aggregate_report_to_elasticsearch":0,"save_aggregate_report_to_opensearch":0,"save_aggregate_reports_to_splunk":0,"save_forens":12,"save_forensic_report_to_elasticsearch":0,"save_forensic_report_to_opensearch":0,"save_forensic_reports_to_splunk":0,"save_output":0,"save_smtp_tl":12,"save_smtp_tls_report_to_elasticsearch":0,"save_smtp_tls_report_to_opensearch":0,"save_smtp_tls_reports_to_splunk":0,"sbin":6,"schedul":[6,12],"schema":10,"scope":[10,12],"script":6,"scrub_nondigest":[3,8],"search":[0,3,8,12],"second":[0,2,12],"secret":12,"secret_access_kei":12,"secur":[0,4,12],"see":[2,3,4,6,7,12],"segment":7,"select":0,"selector":10,"self":[4,5],"send":[0,2,3,4,5,7,8,11,12],"sender":[5,7,8],"sending_mta_ip":10,"sensit":12,"sent":[3,8,12],"separ":[0,3,4,6,7,9,11,12],"server":[0,2,3,4,6,7,10,12],"server_ip":4,"servernameon":10,"servic":[0,3,4,5,6,7,8,10],"service_account":12,"service_account_us":12,"session":[0,7],"set":[0,2,3,4,6,7,8,9,12],"set_host":0,"setup":[4,6,9,12],"shard":[0,12],"share":[4,6,12],"sharealik":6,"sharepoint":10,"shell":6,"ship":6,"should":[3,7,8,12],"shouldn":[3,8],"show":[2,7,12],"shown":12,"side":7,"sighup":[0,6,12],"sign":[0,3,4,6,12],"signal":12,"signatur":[3,7,8],"sigv4":[0,12],"silent":12,"similar":7,"simpl":5,"simplifi":0,"sinc":[0,6,12],"singl":[0,12],"sink":12,"sister":3,"size":[2,4],"skel":6,"skip":[0,12],"skip_certificate_verif":[0,12],"slightli":11,"slow":0,"small":4,"smaller":12,"smtp":[0,3,5,12],"smtp_tl":[0,12],"smtp_tls_csv_filenam":[0,12],"smtp_tls_json_filenam":[0,12],"smtp_tls_report":0,"smtp_tls_url":12,"smtptlsfailuredetail":0,"smtptlsfailuredetailsopt":0,"smtptlsparsedreport":0,"smtptlspolici":0,"smtptlspolicysummari":0,"smtptlsreport":0,"so":[0,3,6,7,8,12],"socket":2,"solut":6,"some":[0,2,3,4,7,8],"someon":4,"sometim":12,"sort":[7,12],"sourc":[0,3,4,6,7,10],"source_as_domain":10,"source_as_nam":10,"source_asn":10,"source_base_domain":10,"source_countri":10,"source_ip_address":10,"source_nam":10,"source_reverse_dn":10,"source_typ":10,"sourceforg":2,"sp":[3,10],"spam":12,"special":12,"specif":[3,6,7,12],"specifi":[2,3],"spf":[7,10],"spf_align":10,"spf_domain":10,"spf_result":10,"spf_scope":10,"splunk":[5,12],"splunk_hec":12,"splunkerror":0,"splunkhec":12,"sponsor":5,"spoof":[3,8],"ss":0,"ssl":[0,2,4,12],"ssl_cert_path":0,"st":[7,10,12],"stabl":4,"stack":[4,7,12],"stai":7,"standard":[0,5,6,10],"start":[0,2,4,7,9,11,12],"starttl":[7,12],"startup":6,"statu":[2,12],"stdout":12,"step":[3,4,6,8],"still":[0,3,8,10,12],"storag":[0,12],"store":[2,4,9],"str":[0,12],"stream":12,"string":0,"strip":[3,8,12],"strip_attachment_payload":[0,12],"strongli":12,"structur":5,"stsv1":10,"subdomain":[0,3,12],"subject":[0,3,8,10,12],"subject_prefix":[3,8],"subsidiari":7,"substitut":6,"success":12,"successful_session_count":10,"sudo":[2,4,6,12],"suffix":[0,12],"suggest":7,"suitabl":0,"summari":[3,8],"supervis":12,"suppli":[0,7,12],"support":[2,5,7,10,11],"sure":4,"surfac":7,"sw50zxjha3rpdmugv2v0dgjld2vyymvylcocymvyc2ljahq":10,"switch":7,"syslog":[2,12],"system":[2,3,4,6,8,12],"systemctl":[2,4,12],"systemd":5,"systemdr":6,"t":[5,8,10,12],"tab":[3,4,8],"tabl":[5,7],"tag":6,"take":[0,12],"target":[2,12],"task":6,"tby":10,"tcp":12,"tee":4,"tell":[3,7,8],"templat":[3,8],"temporari":7,"tenant":5,"tenant_id":12,"term":6,"test":[0,10,12],"text":[0,10],"than":[3,4,8,12],"thank":10,"thei":[3,7,8,12],"theirs":3,"them":[0,4,7,12],"therebi":[3,8],"thi":[0,2,3,4,5,6,7,8,10,12],"those":[0,12],"thousand":12,"three":7,"through":[0,3],"throughput":12,"tier":12,"time":[0,2,4,6,7,12],"timeout":[0,2,12],"timespan":0,"timespan_requires_norm":10,"timestamp":0,"timestamp_to_datetim":0,"timestamp_to_human":0,"timezon":10,"tl":[0,5,12],"tld":3,"to_domain":10,"to_utc":0,"togeth":7,"token":[0,4,12],"token_fil":12,"tool":12,"top":[3,7],"topic":12,"touch":[3,8],"tracker":1,"trade":12,"tradit":[3,8],"trail":12,"transfer":10,"transient":0,"transpar":5,"transport":[4,12],"trash":12,"tri":0,"true":[0,2,4,10,12],"trust":12,"truststor":4,"try":12,"tuesdai":6,"tune":5,"two":6,"txt":0,"type":[5,7,10,12],"typo":12,"u":[2,6,10,12],"ubuntu":[4,6],"udp":[0,12],"ui":[3,8],"uncondition":[3,8],"under":[4,6,7,12],"underli":0,"underneath":7,"underscor":12,"understand":[5,7],"unencrypt":12,"unfortun":[3,8],"unit":[0,2,12],"unix":0,"unknown":0,"unless":6,"unreach":12,"unsubscrib":[3,8],"until":[0,5,12],"unzip":2,"up":[0,2,4,6,7,9,12],"updat":[0,4,6,12],"upersecur":12,"upgrad":[2,5,6,12],"upload":12,"upper":7,"uppercas":12,"uri":6,"url":[0,2,12],"us":[0,3,4,5,8,10],"usag":12,"use_ssl":0,"user":[0,2,3,4,6,7,8,10,12],"user_ag":10,"useradd":[2,6],"usernam":[0,12],"usernamepassword":12,"usesystemproxi":2,"usr":[4,6],"utc":0,"utf":10,"util":5,"v":12,"valid":[0,7,10,12],"valimail":5,"valu":[0,3,4,7,8,12],"var":[3,6,8,12],"variabl":5,"variou":6,"vendor":3,"venv":[6,12],"verbos":12,"veri":[4,7,12],"verif":[0,4,12],"verifi":0,"verification_mod":4,"version":[2,4,5,9,10,11,12],"vew":2,"via":[0,2],"view":[7,12],"vim":4,"virtualenv":6,"visual":[4,9],"volum":[7,12],"vulner":3,"w":[0,12],"w3c":10,"wa":[3,4,8],"wai":[4,7],"wait":[0,12],"want":[2,12],"wantedbi":[2,12],"warn":12,"watch":[0,2,4,6,12],"watch_inbox":0,"watcher":12,"web":[2,4],"webdav":2,"webhook":12,"webmail":[3,7,8],"week":[0,12],"weekli":6,"well":[2,12],"were":12,"wettbewerb":10,"wget":4,"whalensolut":12,"what":5,"when":[0,3,5,7,8,12],"whenev":[0,2,12],"where":[0,2,3,8,12],"wherea":7,"wherev":12,"whether":0,"which":[2,4,5,6,7,12],"while":[7,12],"who":7,"whole":0,"why":[3,7],"wide":[6,10,12],"wiki":10,"window":[6,12],"within":0,"without":[3,4,7,8],"won":5,"work":[2,3,5,6,7,8,12],"worker":12,"workstat":2,"worst":3,"would":[3,6,8],"wrap":[3,8],"write":12,"www":[4,6,12],"x":[4,7,10],"x509":4,"xennn":10,"xml":[0,11],"xml_schema":10,"xms4g":4,"xmx4g":4,"xpack":4,"xxxx":4,"y":[4,6],"yahoo":7,"yaml":12,"ye":[3,8],"year":12,"yet":3,"yml":4,"you":[2,3,4,5,6,7,8,12],"your":[3,4,5,6,7,8,11,12],"yyyi":0,"zero":12,"zip":[0,2,5,12],"\u00fcbersicht":10},"titles":["API reference","Contributing to parsedmarc","Accessing an inbox using OWA/EWS","Understanding DMARC","Elasticsearch and Kibana","parsedmarc documentation - Open source DMARC report analyzer and visualizer","Installation","Using the Kibana dashboards","What about mailing lists?","OpenSearch and Grafana","Sample outputs","Splunk","Using parsedmarc"],"titleterms":{"2":[3,8],"3":[3,8],"about":[3,8],"access":2,"aggreg":[7,10],"align":3,"an":2,"analyz":[5,6],"api":0,"best":[3,8],"bug":1,"cli":12,"compat":5,"compos":12,"config":12,"configur":[2,12],"content":5,"contribut":1,"countri":6,"csv":10,"dashboard":7,"databas":6,"davmail":2,"depend":6,"dkim":3,"dmarc":[3,5,7],"do":[3,8],"docker":12,"document":5,"domain":3,"elast":0,"elasticsearch":4,"env":12,"environ":12,"ew":2,"exampl":12,"exchang":6,"failur":7,"featur":5,"file":12,"forens":10,"geolite2":6,"grafana":9,"guid":3,"help":12,"inbox":2,"index":4,"indic":0,"instal":[4,6,9],"ip":6,"json":10,"kibana":[4,7],"list":[3,8],"listserv":[3,8],"lookalik":3,"mail":[3,8],"mailman":[3,8],"map":12,"maxmind":6,"microsoft":6,"mode":12,"multi":12,"multipl":6,"name":12,"onli":12,"open":5,"opensearch":[0,9],"option":6,"output":10,"owa":2,"parsedmarc":[0,1,2,5,6,12],"pattern":4,"perform":12,"practic":[3,8],"prerequisit":6,"proxi":6,"python":5,"record":[3,4,9],"refer":0,"reload":12,"report":[1,5,6,7,10],"resourc":3,"restart":12,"retent":[4,9],"run":[2,12],"sampl":10,"section":12,"sender":3,"servic":[2,12],"smtp":[7,10],"sourc":5,"specifi":12,"spf":3,"splunk":[0,11],"support":[3,12],"systemd":[2,12],"t":3,"tabl":0,"tenant":12,"test":6,"tl":[7,10],"tune":12,"type":0,"understand":3,"upgrad":4,"us":[2,6,7,12],"util":0,"valid":3,"variabl":12,"via":12,"visual":5,"web":6,"what":[3,8],"without":12,"won":3,"workaround":[3,8]}}) \ No newline at end of file +Search.setIndex({"alltitles":{"API reference":[[0,null]],"Accessing an inbox using OWA/EWS":[[2,null]],"Bug reports":[[1,"bug-reports"]],"CLI help":[[12,"cli-help"]],"CSV aggregate report":[[10,"csv-aggregate-report"]],"CSV failure report":[[10,"csv-failure-report"]],"Configuration file":[[12,"configuration-file"]],"Configuring parsedmarc for DavMail":[[2,"configuring-parsedmarc-for-davmail"]],"Contents":[[5,null]],"Contributing to parsedmarc":[[1,null]],"DMARC Alignment Guide":[[3,"dmarc-alignment-guide"]],"DMARC aggregate reports":[[7,"dmarc-aggregate-reports"]],"DMARC failure reports":[[7,"dmarc-failure-reports"]],"DMARC guides":[[3,"dmarc-guides"]],"Do":[[3,"do"],[8,"do"]],"Do not":[[3,"do-not"],[8,"do-not"]],"Docker Compose example":[[12,"docker-compose-example"]],"Docker secrets (_FILE suffix)":[[12,"docker-secrets-file-suffix"]],"Elasticsearch and Kibana":[[4,null]],"Environment variable configuration":[[12,"environment-variable-configuration"]],"Examples":[[12,"examples"]],"Features":[[5,"features"]],"IP-to-country database":[[6,"ip-to-country-database"]],"Indices and tables":[[0,"indices-and-tables"]],"Installation":[[4,"installation"],[6,null],[9,"installation"]],"Installing parsedmarc":[[6,"installing-parsedmarc"]],"JSON SMTP TLS report":[[10,"json-smtp-tls-report"]],"JSON aggregate report":[[10,"json-aggregate-report"]],"JSON failure report":[[10,"json-failure-report"]],"LISTSERV":[[3,"listserv"],[8,"listserv"]],"Lookalike domains":[[3,"lookalike-domains"]],"Mailing list best practices":[[3,"mailing-list-best-practices"],[8,"mailing-list-best-practices"]],"Mailman 2":[[3,"mailman-2"],[3,"id1"],[8,"mailman-2"],[8,"id1"]],"Mailman 3":[[3,"mailman-3"],[3,"id2"],[8,"mailman-3"],[8,"id2"]],"Multi-tenant support":[[12,"multi-tenant-support"]],"OpenSearch and Grafana":[[9,null]],"Optional dependencies":[[6,"optional-dependencies"]],"Performance tuning":[[12,"performance-tuning"]],"Prerequisites":[[6,"prerequisites"]],"Python Compatibility":[[5,"python-compatibility"]],"Records retention":[[4,"records-retention"],[9,"records-retention"]],"Reloading configuration without restarting":[[12,"reloading-configuration-without-restarting"]],"Resources":[[3,"resources"]],"Running DavMail as a systemd service":[[2,"running-davmail-as-a-systemd-service"]],"Running parsedmarc as a systemd service":[[12,"running-parsedmarc-as-a-systemd-service"]],"Running without a config file (env-only mode)":[[12,"running-without-a-config-file-env-only-mode"]],"SMTP TLS reporting":[[7,"smtp-tls-reporting"]],"SPF and DMARC record validation":[[3,"spf-and-dmarc-record-validation"]],"Sample aggregate report output":[[10,"sample-aggregate-report-output"]],"Sample failure report output":[[10,"sample-failure-report-output"]],"Sample outputs":[[10,null]],"Section name mapping":[[12,"section-name-mapping"]],"Specifying the config file via environment variable":[[12,"specifying-the-config-file-via-environment-variable"]],"Splunk":[[11,null]],"Testing multiple report analyzers":[[6,"testing-multiple-report-analyzers"]],"Understanding DMARC":[[3,null]],"Upgrading Kibana index patterns":[[4,"upgrading-kibana-index-patterns"]],"Using MaxMind GeoLite2 (optional)":[[6,"using-maxmind-geolite2-optional"]],"Using Microsoft Exchange":[[6,"using-microsoft-exchange"]],"Using a web proxy":[[6,"using-a-web-proxy"]],"Using parsedmarc":[[12,null]],"Using the Kibana dashboards":[[7,null]],"What about mailing lists?":[[3,"what-about-mailing-lists"],[8,null]],"What if a sender won\u2019t support DKIM/DMARC?":[[3,"what-if-a-sender-wont-support-dkim-dmarc"]],"Workarounds":[[3,"workarounds"],[8,"workarounds"]],"parsedmarc":[[0,"module-parsedmarc"]],"parsedmarc documentation - Open source DMARC report analyzer and visualizer":[[5,null]],"parsedmarc.elastic":[[0,"module-parsedmarc.elastic"]],"parsedmarc.opensearch":[[0,"module-parsedmarc.opensearch"]],"parsedmarc.splunk":[[0,"module-parsedmarc.splunk"]],"parsedmarc.types":[[0,"module-parsedmarc.types"]],"parsedmarc.utils":[[0,"module-parsedmarc.utils"]]},"docnames":["api","contributing","davmail","dmarc","elasticsearch","index","installation","kibana","mailing-lists","opensearch","output","splunk","usage"],"envversion":{"sphinx":65,"sphinx.domains.c":3,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":9,"sphinx.domains.index":1,"sphinx.domains.javascript":3,"sphinx.domains.math":2,"sphinx.domains.python":4,"sphinx.domains.rst":2,"sphinx.domains.std":2,"sphinx.ext.todo":2,"sphinx.ext.viewcode":1},"filenames":["api.md","contributing.md","davmail.md","dmarc.md","elasticsearch.md","index.md","installation.md","kibana.md","mailing-lists.md","opensearch.md","output.md","splunk.md","usage.md"],"indexentries":{"aggregatealignment (class in parsedmarc.types)":[[0,"parsedmarc.types.AggregateAlignment",false]],"aggregateauthresultdkim (class in parsedmarc.types)":[[0,"parsedmarc.types.AggregateAuthResultDKIM",false]],"aggregateauthresults (class in parsedmarc.types)":[[0,"parsedmarc.types.AggregateAuthResults",false]],"aggregateauthresultspf (class in parsedmarc.types)":[[0,"parsedmarc.types.AggregateAuthResultSPF",false]],"aggregateidentifiers (class in parsedmarc.types)":[[0,"parsedmarc.types.AggregateIdentifiers",false]],"aggregateparsedreport (class in parsedmarc.types)":[[0,"parsedmarc.types.AggregateParsedReport",false]],"aggregatepolicyevaluated (class in parsedmarc.types)":[[0,"parsedmarc.types.AggregatePolicyEvaluated",false]],"aggregatepolicyoverridereason (class in parsedmarc.types)":[[0,"parsedmarc.types.AggregatePolicyOverrideReason",false]],"aggregatepolicypublished (class in parsedmarc.types)":[[0,"parsedmarc.types.AggregatePolicyPublished",false]],"aggregaterecord (class in parsedmarc.types)":[[0,"parsedmarc.types.AggregateRecord",false]],"aggregatereport (class in parsedmarc.types)":[[0,"parsedmarc.types.AggregateReport",false]],"aggregatereportmetadata (class in parsedmarc.types)":[[0,"parsedmarc.types.AggregateReportMetadata",false]],"alreadysaved":[[0,"parsedmarc.elastic.AlreadySaved",false],[0,"parsedmarc.opensearch.AlreadySaved",false]],"append_json() (in module parsedmarc)":[[0,"parsedmarc.append_json",false]],"close() (parsedmarc.splunk.hecclient method)":[[0,"parsedmarc.splunk.HECClient.close",false]],"configure_ipinfo_api() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.configure_ipinfo_api",false]],"convert_outlook_msg() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.convert_outlook_msg",false]],"create_indexes() (in module parsedmarc.elastic)":[[0,"parsedmarc.elastic.create_indexes",false]],"create_indexes() (in module parsedmarc.opensearch)":[[0,"parsedmarc.opensearch.create_indexes",false]],"decode_base64() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.decode_base64",false]],"downloaderror":[[0,"parsedmarc.utils.DownloadError",false]],"elasticsearcherror":[[0,"parsedmarc.elastic.ElasticsearchError",false]],"email_results() (in module parsedmarc)":[[0,"parsedmarc.email_results",false]],"emailaddress (class in parsedmarc.types)":[[0,"parsedmarc.types.EmailAddress",false]],"emailattachment (class in parsedmarc.types)":[[0,"parsedmarc.types.EmailAttachment",false]],"emailparsererror":[[0,"parsedmarc.utils.EmailParserError",false]],"extract_report() (in module parsedmarc)":[[0,"parsedmarc.extract_report",false]],"extract_report_from_file_path() (in module parsedmarc)":[[0,"parsedmarc.extract_report_from_file_path",false]],"failureparsedreport (class in parsedmarc.types)":[[0,"parsedmarc.types.FailureParsedReport",false]],"failurereport (class in parsedmarc.types)":[[0,"parsedmarc.types.FailureReport",false]],"forensicparsedreport (in module parsedmarc.types)":[[0,"parsedmarc.types.ForensicParsedReport",false]],"forensicreport (in module parsedmarc.types)":[[0,"parsedmarc.types.ForensicReport",false]],"get_base_domain() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.get_base_domain",false]],"get_dmarc_reports_from_mailbox() (in module parsedmarc)":[[0,"parsedmarc.get_dmarc_reports_from_mailbox",false]],"get_dmarc_reports_from_mbox() (in module parsedmarc)":[[0,"parsedmarc.get_dmarc_reports_from_mbox",false]],"get_filename_safe_string() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.get_filename_safe_string",false]],"get_ip_address_country() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.get_ip_address_country",false]],"get_ip_address_db_record() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.get_ip_address_db_record",false]],"get_ip_address_info() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.get_ip_address_info",false]],"get_report_zip() (in module parsedmarc)":[[0,"parsedmarc.get_report_zip",false]],"get_reverse_dns() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.get_reverse_dns",false]],"get_service_from_reverse_dns_base_domain() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.get_service_from_reverse_dns_base_domain",false]],"hecclient (class in parsedmarc.splunk)":[[0,"parsedmarc.splunk.HECClient",false]],"human_timestamp_to_datetime() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.human_timestamp_to_datetime",false]],"human_timestamp_to_unix_timestamp() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.human_timestamp_to_unix_timestamp",false]],"invalidaggregatereport":[[0,"parsedmarc.InvalidAggregateReport",false]],"invaliddmarcreport":[[0,"parsedmarc.InvalidDMARCReport",false]],"invalidfailurereport":[[0,"parsedmarc.InvalidFailureReport",false]],"invalidforensicreport (in module parsedmarc)":[[0,"parsedmarc.InvalidForensicReport",false]],"invalidipinfoapikey":[[0,"parsedmarc.utils.InvalidIPinfoAPIKey",false]],"invalidsmtptlsreport":[[0,"parsedmarc.InvalidSMTPTLSReport",false]],"ipaddressinfo (class in parsedmarc.utils)":[[0,"parsedmarc.utils.IPAddressInfo",false]],"ipsourceinfo (class in parsedmarc.types)":[[0,"parsedmarc.types.IPSourceInfo",false]],"is_mbox() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.is_mbox",false]],"is_outlook_msg() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.is_outlook_msg",false]],"load_ip_db() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.load_ip_db",false]],"load_psl_overrides() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.load_psl_overrides",false]],"load_reverse_dns_map() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.load_reverse_dns_map",false]],"migrate_indexes() (in module parsedmarc.elastic)":[[0,"parsedmarc.elastic.migrate_indexes",false]],"migrate_indexes() (in module parsedmarc.opensearch)":[[0,"parsedmarc.opensearch.migrate_indexes",false]],"module":[[0,"module-parsedmarc",false],[0,"module-parsedmarc.elastic",false],[0,"module-parsedmarc.opensearch",false],[0,"module-parsedmarc.splunk",false],[0,"module-parsedmarc.types",false],[0,"module-parsedmarc.utils",false]],"opensearcherror":[[0,"parsedmarc.opensearch.OpenSearchError",false]],"parse_aggregate_report_file() (in module parsedmarc)":[[0,"parsedmarc.parse_aggregate_report_file",false]],"parse_aggregate_report_xml() (in module parsedmarc)":[[0,"parsedmarc.parse_aggregate_report_xml",false]],"parse_email() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.parse_email",false]],"parse_failure_report() (in module parsedmarc)":[[0,"parsedmarc.parse_failure_report",false]],"parse_forensic_report() (in module parsedmarc)":[[0,"parsedmarc.parse_forensic_report",false]],"parse_report_email() (in module parsedmarc)":[[0,"parsedmarc.parse_report_email",false]],"parse_report_file() (in module parsedmarc)":[[0,"parsedmarc.parse_report_file",false]],"parse_smtp_tls_report_json() (in module parsedmarc)":[[0,"parsedmarc.parse_smtp_tls_report_json",false]],"parsed_aggregate_reports_to_csv() (in module parsedmarc)":[[0,"parsedmarc.parsed_aggregate_reports_to_csv",false]],"parsed_aggregate_reports_to_csv_rows() (in module parsedmarc)":[[0,"parsedmarc.parsed_aggregate_reports_to_csv_rows",false]],"parsed_failure_reports_to_csv() (in module parsedmarc)":[[0,"parsedmarc.parsed_failure_reports_to_csv",false]],"parsed_failure_reports_to_csv_rows() (in module parsedmarc)":[[0,"parsedmarc.parsed_failure_reports_to_csv_rows",false]],"parsed_forensic_reports_to_csv() (in module parsedmarc)":[[0,"parsedmarc.parsed_forensic_reports_to_csv",false]],"parsed_forensic_reports_to_csv_rows() (in module parsedmarc)":[[0,"parsedmarc.parsed_forensic_reports_to_csv_rows",false]],"parsed_smtp_tls_reports_to_csv() (in module parsedmarc)":[[0,"parsedmarc.parsed_smtp_tls_reports_to_csv",false]],"parsed_smtp_tls_reports_to_csv_rows() (in module parsedmarc)":[[0,"parsedmarc.parsed_smtp_tls_reports_to_csv_rows",false]],"parsedemail (class in parsedmarc.types)":[[0,"parsedmarc.types.ParsedEmail",false]],"parsedmarc":[[0,"module-parsedmarc",false]],"parsedmarc.elastic":[[0,"module-parsedmarc.elastic",false]],"parsedmarc.opensearch":[[0,"module-parsedmarc.opensearch",false]],"parsedmarc.splunk":[[0,"module-parsedmarc.splunk",false]],"parsedmarc.types":[[0,"module-parsedmarc.types",false]],"parsedmarc.utils":[[0,"module-parsedmarc.utils",false]],"parsererror":[[0,"parsedmarc.ParserError",false]],"parsingresults (class in parsedmarc.types)":[[0,"parsedmarc.types.ParsingResults",false]],"query_dns() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.query_dns",false]],"reversednsservice (class in parsedmarc.utils)":[[0,"parsedmarc.utils.ReverseDNSService",false]],"save_aggregate_report_to_elasticsearch() (in module parsedmarc.elastic)":[[0,"parsedmarc.elastic.save_aggregate_report_to_elasticsearch",false]],"save_aggregate_report_to_opensearch() (in module parsedmarc.opensearch)":[[0,"parsedmarc.opensearch.save_aggregate_report_to_opensearch",false]],"save_aggregate_reports_to_splunk() (parsedmarc.splunk.hecclient method)":[[0,"parsedmarc.splunk.HECClient.save_aggregate_reports_to_splunk",false]],"save_failure_report_to_elasticsearch() (in module parsedmarc.elastic)":[[0,"parsedmarc.elastic.save_failure_report_to_elasticsearch",false]],"save_failure_report_to_opensearch() (in module parsedmarc.opensearch)":[[0,"parsedmarc.opensearch.save_failure_report_to_opensearch",false]],"save_failure_reports_to_splunk() (parsedmarc.splunk.hecclient method)":[[0,"parsedmarc.splunk.HECClient.save_failure_reports_to_splunk",false]],"save_forensic_report_to_elasticsearch() (in module parsedmarc.elastic)":[[0,"parsedmarc.elastic.save_forensic_report_to_elasticsearch",false]],"save_forensic_report_to_opensearch() (in module parsedmarc.opensearch)":[[0,"parsedmarc.opensearch.save_forensic_report_to_opensearch",false]],"save_forensic_reports_to_splunk() (parsedmarc.splunk.hecclient method)":[[0,"parsedmarc.splunk.HECClient.save_forensic_reports_to_splunk",false]],"save_output() (in module parsedmarc)":[[0,"parsedmarc.save_output",false]],"save_smtp_tls_report_to_elasticsearch() (in module parsedmarc.elastic)":[[0,"parsedmarc.elastic.save_smtp_tls_report_to_elasticsearch",false]],"save_smtp_tls_report_to_opensearch() (in module parsedmarc.opensearch)":[[0,"parsedmarc.opensearch.save_smtp_tls_report_to_opensearch",false]],"save_smtp_tls_reports_to_splunk() (parsedmarc.splunk.hecclient method)":[[0,"parsedmarc.splunk.HECClient.save_smtp_tls_reports_to_splunk",false]],"set_hosts() (in module parsedmarc.elastic)":[[0,"parsedmarc.elastic.set_hosts",false]],"set_hosts() (in module parsedmarc.opensearch)":[[0,"parsedmarc.opensearch.set_hosts",false]],"smtptlsfailuredetails (class in parsedmarc.types)":[[0,"parsedmarc.types.SMTPTLSFailureDetails",false]],"smtptlsfailuredetailsoptional (class in parsedmarc.types)":[[0,"parsedmarc.types.SMTPTLSFailureDetailsOptional",false]],"smtptlsparsedreport (class in parsedmarc.types)":[[0,"parsedmarc.types.SMTPTLSParsedReport",false]],"smtptlspolicy (class in parsedmarc.types)":[[0,"parsedmarc.types.SMTPTLSPolicy",false]],"smtptlspolicysummary (class in parsedmarc.types)":[[0,"parsedmarc.types.SMTPTLSPolicySummary",false]],"smtptlsreport (class in parsedmarc.types)":[[0,"parsedmarc.types.SMTPTLSReport",false]],"splunkerror":[[0,"parsedmarc.splunk.SplunkError",false]],"timestamp_to_datetime() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.timestamp_to_datetime",false]],"timestamp_to_human() (in module parsedmarc.utils)":[[0,"parsedmarc.utils.timestamp_to_human",false]],"watch_inbox() (in module parsedmarc)":[[0,"parsedmarc.watch_inbox",false]]},"objects":{"":[[0,0,0,"-","parsedmarc"]],"parsedmarc":[[0,1,1,"","InvalidAggregateReport"],[0,1,1,"","InvalidDMARCReport"],[0,1,1,"","InvalidFailureReport"],[0,2,1,"","InvalidForensicReport"],[0,1,1,"","InvalidSMTPTLSReport"],[0,1,1,"","ParserError"],[0,3,1,"","append_json"],[0,0,0,"-","elastic"],[0,3,1,"","email_results"],[0,3,1,"","extract_report"],[0,3,1,"","extract_report_from_file_path"],[0,3,1,"","get_dmarc_reports_from_mailbox"],[0,3,1,"","get_dmarc_reports_from_mbox"],[0,3,1,"","get_report_zip"],[0,0,0,"-","opensearch"],[0,3,1,"","parse_aggregate_report_file"],[0,3,1,"","parse_aggregate_report_xml"],[0,3,1,"","parse_failure_report"],[0,3,1,"","parse_forensic_report"],[0,3,1,"","parse_report_email"],[0,3,1,"","parse_report_file"],[0,3,1,"","parse_smtp_tls_report_json"],[0,3,1,"","parsed_aggregate_reports_to_csv"],[0,3,1,"","parsed_aggregate_reports_to_csv_rows"],[0,3,1,"","parsed_failure_reports_to_csv"],[0,3,1,"","parsed_failure_reports_to_csv_rows"],[0,3,1,"","parsed_forensic_reports_to_csv"],[0,3,1,"","parsed_forensic_reports_to_csv_rows"],[0,3,1,"","parsed_smtp_tls_reports_to_csv"],[0,3,1,"","parsed_smtp_tls_reports_to_csv_rows"],[0,3,1,"","save_output"],[0,0,0,"-","splunk"],[0,0,0,"-","types"],[0,0,0,"-","utils"],[0,3,1,"","watch_inbox"]],"parsedmarc.elastic":[[0,1,1,"","AlreadySaved"],[0,1,1,"","ElasticsearchError"],[0,3,1,"","create_indexes"],[0,3,1,"","migrate_indexes"],[0,3,1,"","save_aggregate_report_to_elasticsearch"],[0,3,1,"","save_failure_report_to_elasticsearch"],[0,3,1,"","save_forensic_report_to_elasticsearch"],[0,3,1,"","save_smtp_tls_report_to_elasticsearch"],[0,3,1,"","set_hosts"]],"parsedmarc.opensearch":[[0,1,1,"","AlreadySaved"],[0,1,1,"","OpenSearchError"],[0,3,1,"","create_indexes"],[0,3,1,"","migrate_indexes"],[0,3,1,"","save_aggregate_report_to_opensearch"],[0,3,1,"","save_failure_report_to_opensearch"],[0,3,1,"","save_forensic_report_to_opensearch"],[0,3,1,"","save_smtp_tls_report_to_opensearch"],[0,3,1,"","set_hosts"]],"parsedmarc.splunk":[[0,4,1,"","HECClient"],[0,1,1,"","SplunkError"]],"parsedmarc.splunk.HECClient":[[0,5,1,"","close"],[0,5,1,"","save_aggregate_reports_to_splunk"],[0,5,1,"","save_failure_reports_to_splunk"],[0,5,1,"","save_forensic_reports_to_splunk"],[0,5,1,"","save_smtp_tls_reports_to_splunk"]],"parsedmarc.types":[[0,4,1,"","AggregateAlignment"],[0,4,1,"","AggregateAuthResultDKIM"],[0,4,1,"","AggregateAuthResultSPF"],[0,4,1,"","AggregateAuthResults"],[0,4,1,"","AggregateIdentifiers"],[0,4,1,"","AggregateParsedReport"],[0,4,1,"","AggregatePolicyEvaluated"],[0,4,1,"","AggregatePolicyOverrideReason"],[0,4,1,"","AggregatePolicyPublished"],[0,4,1,"","AggregateRecord"],[0,4,1,"","AggregateReport"],[0,4,1,"","AggregateReportMetadata"],[0,4,1,"","EmailAddress"],[0,4,1,"","EmailAttachment"],[0,4,1,"","FailureParsedReport"],[0,4,1,"","FailureReport"],[0,2,1,"","ForensicParsedReport"],[0,2,1,"","ForensicReport"],[0,4,1,"","IPSourceInfo"],[0,4,1,"","ParsedEmail"],[0,4,1,"","ParsingResults"],[0,4,1,"","SMTPTLSFailureDetails"],[0,4,1,"","SMTPTLSFailureDetailsOptional"],[0,4,1,"","SMTPTLSParsedReport"],[0,4,1,"","SMTPTLSPolicy"],[0,4,1,"","SMTPTLSPolicySummary"],[0,4,1,"","SMTPTLSReport"]],"parsedmarc.utils":[[0,1,1,"","DownloadError"],[0,1,1,"","EmailParserError"],[0,4,1,"","IPAddressInfo"],[0,1,1,"","InvalidIPinfoAPIKey"],[0,4,1,"","ReverseDNSService"],[0,3,1,"","configure_ipinfo_api"],[0,3,1,"","convert_outlook_msg"],[0,3,1,"","decode_base64"],[0,3,1,"","get_base_domain"],[0,3,1,"","get_filename_safe_string"],[0,3,1,"","get_ip_address_country"],[0,3,1,"","get_ip_address_db_record"],[0,3,1,"","get_ip_address_info"],[0,3,1,"","get_reverse_dns"],[0,3,1,"","get_service_from_reverse_dns_base_domain"],[0,3,1,"","human_timestamp_to_datetime"],[0,3,1,"","human_timestamp_to_unix_timestamp"],[0,3,1,"","is_mbox"],[0,3,1,"","is_outlook_msg"],[0,3,1,"","load_ip_db"],[0,3,1,"","load_psl_overrides"],[0,3,1,"","load_reverse_dns_map"],[0,3,1,"","parse_email"],[0,3,1,"","query_dns"],[0,3,1,"","timestamp_to_datetime"],[0,3,1,"","timestamp_to_human"]]},"objnames":{"0":["py","module","Python module"],"1":["py","exception","Python exception"],"2":["py","attribute","Python attribute"],"3":["py","function","Python function"],"4":["py","class","Python class"],"5":["py","method","Python method"]},"objtypes":{"0":"py:module","1":"py:exception","2":"py:attribute","3":"py:function","4":"py:class","5":"py:method"},"terms":{"":[0,2,3,4,6,7,8,10,12],"0":[0,2,3,4,5,6,8,9,10,11,12],"00":10,"003":10,"00z":10,"00z_exampl":10,"01":10,"0200":10,"0240":10,"04":10,"08":10,"09":10,"09t00":10,"09t23":10,"1":[0,2,4,5,10,12],"10":[0,5,6,10,12],"100":[10,12],"1000":12,"11":[5,6,10],"1143":2,"12":[5,6],"12201":12,"127":[2,4,12],"13":5,"14":5,"150":10,"16":[3,8],"173":10,"176":10,"19":[10,12],"1d":12,"1g":4,"1w":12,"2":[0,4,10,12],"20":10,"2000":12,"201":10,"2010":[6,10],"2012":10,"2013":6,"2016":6,"2017a":[3,8],"2018":10,"2019":6,"2024":10,"2028":5,"2030":5,"2035":5,"208":10,"209":10,"21":6,"212":10,"22":6,"222":10,"23":10,"2369":[3,8],"24":0,"241":10,"25":12,"27":10,"28":10,"2919":[3,8],"2d":12,"2k":12,"3":[0,5,6,10,11,12],"30":[0,12],"300":2,"30937":10,"3128":6,"365":[2,4],"38":10,"3d":10,"3h":12,"4":[4,6,11],"400":[0,12],"401":0,"403":0,"4096":4,"41":10,"5":[2,4,9,12],"500":12,"514":12,"5432":12,"5601":4,"59":10,"59z":10,"5m":[2,12],"6":[0,4,5,12],"60":[0,12],"6514":12,"6591":5,"660":4,"7":[4,5],"7018":10,"72":10,"7480":10,"7489":5,"7d":12,"8":[2,4,5,7,10,12],"8080":12,"822":0,"8460":5,"85":10,"86399":10,"86400":10,"9":[5,12],"9200":[4,12],"932":12,"9391651994964116463":10,"94":10,"993":12,"9989":5,"9990":5,"9991":5,"A":[0,3,7,12],"AT":10,"And":0,"As":[4,7],"By":[7,12],"For":[4,12],"IF":12,"If":[0,3,4,6,7,8,12],"In":[0,2,3,7,8,12],"It":[2,4,7,10,12],"NOT":12,"No":[3,6,8],"On":[3,4,6,7,8,12],"Or":[4,6,12],"That":7,"The":[0,3,6,7,11,12],"Then":[2,3,4,6,8,12],"There":7,"These":[7,12],"To":[2,4,6,7,9,10,12],"With":[7,12],"_":12,"_attempt":0,"_cluster":12,"_input":0,"_ipdatabaserecord":0,"_serverless_rejected_set":0,"abl":6,"abort":12,"about":[0,5],"abov":[2,6,12],"accept":[0,3,4,7,8,12],"access":[0,4,5,12],"access_key_id":12,"access_token":0,"accessright":12,"accident":[3,8],"account":[6,7,12],"acm":10,"acquir":12,"across":7,"action":[3,8],"activ":[4,5,12],"active_primary_shard":12,"active_shard":12,"actual":[3,10],"ad":[3,8,12],"add":[2,3,4,6,7,8,12],"addit":[3,8,12],"address":[0,2,3,4,7,8,10,12],"addresse":7,"adkim":10,"admin":[3,8,12],"administr":[3,8],"after":[0,2,4,12],"against":[3,8],"agari":5,"agent":4,"aggreg":[0,5,11,12],"aggregate_csv_filenam":[0,12],"aggregate_index":0,"aggregate_json_filenam":[0,12],"aggregate_report":0,"aggregate_top":12,"aggregate_url":12,"aggregatealign":0,"aggregateauthresult":0,"aggregateauthresultdkim":0,"aggregateauthresultspf":0,"aggregateidentifi":0,"aggregateparsedreport":0,"aggregatepolicyevalu":0,"aggregatepolicyoverridereason":0,"aggregatepolicypublish":0,"aggregaterecord":0,"aggregatereport":0,"aggregatereportmetadata":0,"aggress":12,"alia":[0,12],"alias":12,"align":[5,7,10],"aliv":0,"all":[3,7,8,11,12],"allow":[2,3,8,12],"allow_unencrypted_storag":12,"allowremot":2,"alreadi":12,"alreadysav":0,"also":[0,2,3,6,7,8,12],"alter":[3,8],"altern":[5,12],"although":11,"alwai":[0,2,4,12],"always_use_local_fil":[0,12],"amazon":5,"amount":12,"an":[0,3,5,7,8,10,12],"analyt":[5,12],"analyz":12,"ani":[0,3,6,7,8,12],"anonym":10,"anoth":[6,12],"answer":[0,12],"apach":5,"api":[2,4,5,12],"api_kei":[0,12],"app":12,"appear":12,"append":[0,12],"append_json":0,"appendix":10,"appid":12,"appli":12,"applic":12,"applicationaccesspolici":12,"approach":12,"approxim":2,"apt":[2,4,6],"ar":[0,2,3,4,5,6,7,8,10,12],"archiv":[0,12],"archive_fold":[0,12],"argument":12,"around":12,"arrai":0,"arriv":12,"arrival_d":10,"arrival_date_utc":10,"artifact":4,"as_domain":[0,10],"as_nam":[0,10],"ask":3,"asmx":2,"asn":[0,10,12],"aspf":10,"assign":4,"associ":0,"assum":12,"att":10,"attach":[0,3,8,10,12],"attachment_filenam":0,"attempt":[0,12],"attribut":6,"auth":[0,2,10,12],"auth_failur":10,"auth_method":12,"auth_mod":12,"auth_result":10,"auth_typ":[0,12],"authent":[0,2,3,4,7,12],"authentication_mechan":10,"authentication_result":10,"authentication_typ":12,"auto":2,"automat":[6,12],"avail":[6,12],"avoid":[7,12],"aw":[0,12],"aws_region":[0,12],"aws_servic":[0,12],"awssigv4":[0,12],"azur":[5,12],"b":10,"b2c":7,"back":[0,12],"backend":12,"backfil":12,"backlog":12,"backward":12,"bare":12,"base":[0,2,3,4,6,7,8,10],"base64":0,"base_domain":[0,10],"basic":[0,2,12],"batch":12,"batch_siz":[0,12],"bcc":[0,10],"bd6e1bb5":10,"becaus":[2,3,7,8,12],"becom":12,"been":[0,12],"befor":[0,12],"begin_d":10,"behavior":0,"behind":6,"being":0,"below":[3,6,8,12],"benefit":5,"best":7,"between":[0,4,7,12],"beyond":0,"bin":[2,4,6,12],"binari":[0,12],"binaryio":0,"bind":2,"bindaddress":2,"blank":[3,8],"block":[2,12],"bodi":[0,3,8,10,12],"bool":[0,12],"both":12,"brand":[5,7],"break":[3,4,8],"broken":0,"browser":4,"bucket":12,"budget":0,"bug":5,"build":6,"built":0,"bundl":[0,6,7,12],"busi":7,"button":[3,8],"byte":0,"c":[10,12],"ca":[4,12],"cach":[0,12],"cafile_path":12,"call":[0,5,12],"callabl":0,"callback":0,"came":[3,8],"can":[0,2,3,4,5,6,7,8,12],"cannot":12,"cap":[0,12],"carri":0,"case":[2,3,8],"catch":[0,12],"caught":0,"caus":[3,4,7,8],"cc":[0,10],"center":7,"cento":[4,6],"cert":[4,12],"cert_path":12,"certain":[0,12],"certfile_path":12,"certif":[0,4,7,12],"certificate_password":12,"certificate_path":12,"cest":10,"chain":0,"chang":[4,7,11,12],"charact":[2,12],"charset":10,"chart":7,"check":[0,2,3,4,7,12],"check_timeout":[0,12],"checkbox":4,"checkdmarc":3,"chines":7,"chmod":[2,4,12],"choos":[3,8],"chown":[2,12],"ci":7,"cisco":12,"class":0,"cleanli":0,"clear":0,"cli":[0,5],"click":[4,7],"client":[2,3,4,8,12],"client_id":12,"client_secret":12,"clientsecret":12,"clientsotimeout":2,"close":[0,12],"cloud":[0,12],"cloudflar":[0,12],"cluster":[4,12],"co":4,"code":[0,4,12],"collect":[7,12],"collector":[11,12],"com":[1,2,3,8,9,10,12],"come":[0,7],"comma":[6,12],"command":[2,3,6,8,12],"comment":12,"commerci":[4,5],"common":[3,4,6,8],"commun":[3,8],"compat":[7,12],"complet":[3,4,12],"compli":[3,4,6,8,9],"compliant":[3,8],"compon":6,"compress":5,"conf":6,"config":[0,2,6],"config_fil":12,"config_reload":0,"configur":[0,3,4,5,6,7,8,9],"configure_ipinfo_api":0,"conform":4,"connect":[0,2,4,12],"connection_str":12,"consid":[5,7],"consist":[0,5,10],"consol":[4,12],"constant":0,"consum":[7,12],"contact":7,"contain":[0,7,11,12],"content":[0,3,8,10,11,12],"contrib":6,"contribut":5,"control":[4,12],"convent":12,"convert":[0,3,8],"convert_outlook_msg":0,"copi":[0,6,11],"core":[3,8],"correct":6,"correctli":[0,7,12],"correspond":12,"corrupt":0,"could":[3,4,8,12],"count":[2,7,10],"countri":[0,7,10,12],"country_cod":0,"cpan":6,"cr":12,"crash":[2,4,12],"creat":[0,2,3,4,6,8,12],"create_fold":0,"create_index":0,"creation":12,"creativ":6,"credenti":[6,12],"credentials_fil":12,"cron":6,"cross":0,"crt":4,"csr":4,"csv":[0,5,12],"cumul":6,"current":[2,4,12],"custom":[7,12],"d":[0,4,12],"daemon":[2,4,12],"dai":[0,4,9,12],"daili":[0,12],"dashboard":[4,5,9,11],"dat":0,"data":[0,4,5,6,7,9,11,12],"databas":[0,12],"date":[0,3,8,10],"date_utc":10,"datetim":0,"davmail":5,"db_path":0,"dbip":[0,12],"dbname":12,"dce":12,"dcr":12,"dcr_aggregate_stream":12,"dcr_failure_stream":12,"dcr_immutable_id":12,"dcr_smtp_tls_stream":12,"dd":0,"de":10,"dearmor":4,"deb":4,"debian":[4,5,6],"debug":12,"decemb":6,"decod":0,"decode_base64":0,"dedic":6,"default":[0,2,4,5,6,7,12],"defens":5,"delai":[2,10,12],"deleg":12,"delegated_us":12,"delet":[0,2,4,12],"delivery_result":10,"demystifi":3,"depend":[0,4,5,12],"deploi":[3,8],"deploy":12,"deprec":12,"describ":12,"descript":[2,6,12],"design":12,"destin":[0,12],"detail":[6,7,12],"dev":[6,12],"devel":6,"develop":5,"devicecod":12,"di":10,"dict":0,"dictionari":0,"differ":[7,12],"difficult":12,"dig":0,"digest":[3,8],"dir":6,"direct":12,"directli":7,"directori":[0,6,12],"disabl":[0,2,12],"disclaim":[3,8],"disk":[0,12],"displai":[3,7,11],"display_nam":10,"disposit":[7,10],"distribut":6,"dkim":[5,7,8,10],"dkim_align":10,"dkim_domain":10,"dkim_result":10,"dkim_selector":10,"dkm":3,"dmarc":[0,4,6,8,9,10,11,12],"dmarc_aggreg":4,"dmarc_align":10,"dmarc_failur":4,"dmarc_moderation_act":[3,8],"dmarc_none_moderation_act":[3,8],"dmarc_quarantine_moderation_act":[3,8],"dmarcian":5,"dmarcresport":12,"dn":[0,3,7,12],"dnf":6,"dns_retri":0,"dns_test_address":12,"dns_timeout":[0,12],"dnspython":0,"do":[0,2,6,7,12],"doc":[9,12],"doctyp":10,"document":[0,2,12],"doe":[3,8,12],"domain":[0,4,7,8,10,12],"domainawar":[1,3,12],"don":3,"doubl":12,"down":[7,12],"download":[0,2,4,6,12],"downloaderror":0,"draft":[5,10],"dsn":12,"dtd":10,"dummi":12,"dure":[2,12],"e":[0,2,3,4,6,8,12],"e7":10,"each":[0,4,6,9,11,12],"earlier":[0,7],"easi":[4,9],"easier":[11,12],"echo":4,"edit":[2,6,12],"editor":11,"effect":12,"effici":4,"either":[5,12],"elast":[4,5,12],"elasticsearch":[0,5,12],"elasticsearcherror":0,"elig":12,"elk":12,"els":4,"email":[0,3,5,6,7,8,10,11,12],"email_result":0,"emailaddress":0,"emailattach":0,"emailparsererror":0,"empti":[0,3,8,12],"en":[3,4,8,10],"enabl":[2,4,12],"enableew":2,"enablekeepal":2,"enableproxi":2,"encod":[0,10,12],"encount":0,"encrypt":[4,12],"encryptedsavedobject":4,"encryptionkei":4,"end":[0,3,4,5,12],"end_dat":10,"endpoint":[5,12],"endpoint_url":12,"enforc":[3,8],"enough":12,"enrol":4,"ensur":[3,6,8],"entir":[0,3,7,8,12],"entri":0,"envelop":3,"envelope_from":10,"envelope_to":10,"environ":[5,6],"eof":0,"eol":5,"epel":6,"equival":6,"error":[0,7,10,12],"escap":12,"especi":[7,12],"etc":[2,3,4,6,8,12],"even":[2,3,8,12],"event":[2,11,12],"everi":[0,2,6,7,12],"ew":5,"ex":12,"exactli":[3,8],"exampl":[3,4,6,8,10],"except":[0,12],"exchang":[2,10,12],"exclud":2,"execreload":12,"execstart":[2,12],"exhaust":12,"exist":[0,3,4,8,12],"exit":[0,12],"expiri":7,"expiringdict":0,"explain":[3,8],"explicit":[0,3,8],"export":[4,7,12],"extra":12,"extract":[0,2],"extract_report":0,"extract_report_from_file_path":0,"ey":[2,12],"f":4,"factor":2,"fail":[0,3,7,8,10,12],"fail_on_output_error":12,"failed_session_count":10,"failov":0,"failur":[0,5,11,12],"failure_csv_filenam":[0,12],"failure_detail":10,"failure_index":0,"failure_json_filenam":[0,12],"failure_report":0,"failure_top":12,"failure_url":12,"failureparsedreport":0,"failurereport":0,"fall":[0,12],"fallback":0,"fals":[0,2,10,12],"fantast":[3,8],"faster":12,"fatal":[0,12],"featur":[4,12],"feedback":0,"feedback_report":0,"feedback_typ":10,"fetch":[0,7,12],"few":[7,12],"field":[0,4],"file":[0,2,5,6,7,11],"file_path":[0,12],"filenam":[0,12],"filename_safe_subject":10,"filepath":12,"fill":[4,6],"filter":[0,3,7,8,11],"final":5,"financ":12,"find":[3,7,8,12],"fine":[3,8],"finish":12,"first":[0,3,6,8,12],"first_strip_reply_to":[3,8],"fit":[3,8,12],"fix":4,"flag":[0,2,6,12],"flat":0,"flexibl":11,"flight":12,"float":[0,12],"fo":10,"fold":0,"folder":[0,2,12],"foldersizelimit":2,"follow":[2,4,5,12],"footer":[3,8],"forens":[5,7,12],"forensicparsedreport":0,"forensicreport":0,"form":12,"format":[0,7,12],"formerli":[5,7,12],"forward":[3,7,8],"found":[0,6,12],"foundat":10,"fqdn":4,"fraud":5,"free":[6,12],"fresh":12,"freshest":12,"friendli":7,"from":[0,2,3,4,5,6,7,8,10,12],"from_is_list":[3,8],"ftp_proxi":6,"full":12,"fulli":[3,8,12],"function":0,"further":7,"g":[0,2,3,4,6,8,12],"gatewai":2,"gb":4,"gcc":6,"gdpr":[4,9],"gelf":[5,12],"gener":[3,4,8,10,12],"geoip":[6,12],"geoipupd":6,"geolite2":5,"geoloc":[0,12],"get":[0,2,4,6,12],"get_base_domain":0,"get_dmarc_reports_from_mailbox":0,"get_dmarc_reports_from_mbox":0,"get_filename_safe_str":0,"get_ip_address_countri":0,"get_ip_address_db_record":0,"get_ip_address_info":0,"get_report_zip":0,"get_reverse_dn":0,"get_service_from_reverse_dns_base_domain":0,"github":[1,6,10,12],"give":[0,4],"given":[0,12],"glass":7,"gmail":[5,7,12],"gmail_api":12,"go":[0,3,8],"goe":[3,8],"googl":[7,12],"googleapi":12,"got":12,"gov":12,"gpg":4,"grafana":5,"grant":12,"graph":[2,5,7,12],"graph_url":12,"graylog":5,"group":[2,7,12],"guid":[4,5],"guidanc":12,"gzip":[0,5],"h":[0,12],"ha":[0,4,6,12],"hamburg":4,"hand":[3,8],"handl":[5,12],"handler":7,"happen":0,"hard":12,"has_defect":10,"have":[3,4,6,7,8,11,12],"head":10,"header":[0,3,7,8,10,12],"header_from":10,"headless":2,"health":12,"healthcar":12,"heap":4,"heavi":[4,12],"hec":[0,11,12],"hecclient":0,"hectokengoesher":12,"help":5,"here":[0,3,8,10],"hh":0,"hi":[3,8],"high":[7,12],"higher":[3,8],"highli":12,"histori":12,"hit":[0,12],"home":6,"hop":10,"host":[0,2,3,4,5,8,12],"hostnam":[0,12],"hour":[0,12],"hover":7,"href":10,"html":[3,4,8,10],"http":[0,1,2,3,4,5,6,8,9,10,11,12],"http_proxi":6,"https_proxi":6,"human":[0,7],"human_timestamp":0,"human_timestamp_to_datetim":0,"human_timestamp_to_unix_timestamp":0,"hup":12,"i":[0,2,3,4,5,6,7,8,10,12],"icon":7,"id":[3,8,10,12],"ideal":[3,8],"ident":[3,8,12],"identifi":10,"idl":[0,2,12],"ignor":12,"imag":12,"imap":[0,2,5,12],"imap_password":12,"imapalwaysapproxmsgs":2,"imapautoexpung":2,"imapcli":5,"imapidledelai":2,"imapport":2,"immedi":2,"immut":12,"impli":12,"import":[4,7,12],"improv":12,"inbox":[0,3,5,8,12],"inc":10,"includ":[0,3,6,8,12],"include_list_post_head":[3,8],"include_rfc2369_head":[3,8],"include_sender_head":[3,8],"include_spam_trash":12,"incom":[7,12],"incorrect":12,"increas":[4,12],"increment":12,"indent":12,"index":[0,5,9,11,12],"index_prefix":[0,12],"index_prefix_domain_map":12,"index_suffix":[0,12],"indic":[3,5],"individu":12,"industri":12,"inform":[0,4,7,12],"infrequ":12,"ingest":12,"ini":[2,12],"initi":0,"inner":12,"input":0,"input_":0,"inspect":12,"instal":[2,5,12],"installed_app":12,"instanc":12,"instead":[0,3,6,8,12],"int":[0,12],"intend":[3,8],"interact":[2,4],"interakt":10,"interfer":[3,8],"interpret":6,"interrupt":12,"interv":12,"interval_begin":10,"interval_end":10,"invalid":[0,12],"invalidaggregatereport":0,"invaliddmarcreport":0,"invalidfailurereport":0,"invalidforensicreport":0,"invalidipinfoapikei":0,"invalidsmtptlsreport":0,"involv":7,"io":[0,12],"ip":[0,3,4,7,12],"ip_address":[0,10],"ip_db_path":[0,12],"ip_db_url":12,"ipaddressinfo":0,"ipinfo":[0,6,12],"ipinfo_api_token":12,"ipinfo_url":12,"ipsourceinfo":0,"ipv4":0,"ipv6":0,"is_mbox":0,"is_outlook_msg":0,"iso":0,"issu":1,"its":[0,6,12],"itself":12,"java":2,"job":[3,6,8],"joe":[3,8],"journalctl":[2,12],"jre":2,"json":[0,5,12],"june":5,"just":7,"jvm":4,"kafka":[5,12],"kb4099855":6,"kb4134118":6,"kb4295699":6,"keep":[0,12],"keep_al":0,"keepal":2,"kei":[0,3,4,6,12],"keyfile_path":12,"keyout":4,"keyr":4,"keystor":4,"kibana":[5,11],"kill":12,"kind":12,"know":3,"known":[3,7,8,12],"kubernet":12,"kwarg":0,"label":12,"languag":[3,8],"larg":[2,12],"larger":12,"last":6,"later":[4,6,12],"latest":[2,4,9,12],"layer":0,"layout":11,"leak":7,"least":[4,6,12],"leav":3,"left":7,"legaci":5,"legal":[3,8],"legitim":[7,12],"less":12,"level":[0,3,4,12],"lf":12,"lib":6,"libemail":6,"libpq":12,"libxml2":6,"libxslt":6,"licens":6,"life":5,"lifetim":0,"lifetimetimeout":0,"like":[0,3,6,8,12],"limit":[0,2,12],"line":[3,8,12],"link":[3,4,7,8],"linux":[3,6,8],"list":[0,2,4,5,7,12],"listen":[2,12],"lite":[0,6,12],"live":7,"ll":[3,8],"load":[0,4,12],"load_ip_db":0,"load_psl_overrid":0,"load_reverse_dns_map":0,"local":[0,2,4,6,10,12],"local_file_path":0,"local_psl_overrides_path":12,"local_reverse_dns_map_path":12,"localhost":12,"locat":[6,7,12],"log":[0,2,5,12],"log_analyt":12,"log_fil":12,"logger":12,"login":4,"logstash":4,"long":[3,12],"longer":[3,8],"look":[0,3,7],"lookup":[0,12],"loopback":2,"loss":0,"lot":7,"low":12,"lower":12,"lua":10,"m":[0,6,10,12],"m365":12,"maco":6,"magnifi":7,"mai":[5,7,12],"mail":[0,5,6,10,12],"mail_bcc":0,"mail_cc":0,"mail_from":0,"mail_to":0,"mailbox":[0,7,12],"mailbox_connect":0,"mailboxconnect":0,"maildir":12,"maildir_cr":12,"maildir_path":12,"mailer":10,"mailrelai":10,"mailto":6,"main":4,"mainpid":12,"maintain":5,"make":[0,3,4,6,8,9,12],"malici":[7,12],"manag":[4,7,12],"mandatori":12,"manual":12,"map":0,"mariadb":12,"market":7,"massiv":12,"match":[0,4,11,12],"max_ag":10,"max_shards_per_nod":12,"maximum":4,"maxmind":[0,5,12],"mbox":[0,12],"mean":12,"mechan":3,"member":[3,8],"memori":12,"mention":7,"menu":[4,7],"merg":0,"messag":[0,2,3,4,6,7,8,10,12],"message_id":10,"meta":10,"method":12,"mfrom":10,"microsoft":[2,5,10,12],"might":[0,3,7,8],"migrat":[7,12],"migrate_index":0,"mime":10,"min":0,"minimum":4,"minut":[0,2,12],"mirror":0,"miss":12,"mitig":[3,8],"mix":0,"mm":0,"mmdb":[0,6,12],"mobil":[3,8],"mode":[0,2,4,6,10],"modern":[2,3,8],"modifi":[0,3,8,12],"modul":[0,5,6,12],"mon":10,"monitor":[3,12],"monthli":[0,12],"monthly_index":[0,12],"more":[0,4,6,11,12],"most":[3,4,6,7,8,12],"mous":7,"move":[0,4,12],"msg":[0,6],"msg_byte":0,"msg_date":0,"msg_footer":[3,8],"msg_header":[3,8],"msgconvert":[0,6],"msgraph":12,"mta":7,"much":12,"multi":[2,5],"multipl":[0,12],"mung":[3,8],"must":[2,3,8,12],"mutual":[4,12],"mv":4,"mx":10,"my":[5,12],"n":[0,10,12],"n_proc":12,"name":[0,3,4,7,10,11],"nameserv":[0,12],"nano":[2,12],"nation":12,"navig":[3,8],"ncontent":10,"ndate":10,"ndjson":[4,7],"need":[0,2,3,4,6,7,8,12],"neither":12,"nelson":[3,8],"net":[2,12],"network":[0,2,4,12],"never":12,"new":[0,2,3,5,6,7,12],"newer":6,"newest":[2,12],"newkei":4,"next":[0,12],"nfrom":10,"nmessag":10,"nmime":10,"node":4,"nologin":6,"non":[0,3,8,12],"nonameserv":0,"none":[0,3,10,12],"noproxyfor":2,"nor":12,"norepli":[3,10],"normal":[0,10,12],"normalize_timespan_threshold_hour":0,"normalized_timespan":10,"nosecureimap":2,"notabl":7,"note":12,"notic":12,"now":[4,7],"nsubject":10,"nto":10,"null":[6,10],"number":[0,12],"number_of_replica":[0,12],"number_of_shard":[0,12],"nwettbewerb":10,"nx":10,"o":[0,2,4,12],"oR":6,"oauth2":12,"oauth2_port":12,"object":[0,4,7],"observ":7,"occur":[0,7],"occurr":11,"oct":10,"offic":2,"office365":2,"offici":12,"offlin":[0,6,12],"often":[7,12],"ol":[0,6],"old":7,"older":[6,10],"oldest":[2,12],"onc":12,"ondmarc":5,"one":[0,3,5,6,8,12],"ones":12,"onli":[0,2,3,6,7,8],"onlin":[0,2,12],"onto":0,"oor":0,"open":[0,3],"opendn":12,"opensearch":[5,7,12],"opensearch_dashboard":7,"opensearcherror":0,"openssl":4,"oper":12,"opt":[2,6,12],"option":[0,2,3,4,5,8,11,12],"order":12,"org":[0,6,9,10,12],"org_email":10,"org_extra_contact_info":10,"org_nam":10,"organ":[2,5,7,12],"organization_nam":10,"origin":[3,8,12],"original_envelope_id":10,"original_mail_from":10,"original_rcpt_to":10,"original_timespan_second":10,"oserror":0,"other":[0,3,4,7,8,12],"otherwis":12,"our":7,"out":[0,3,4,7],"outdat":7,"outgo":[3,8,12],"outlook":[0,2,6],"output":[0,5,12],"output_directori":0,"outsid":12,"over":[0,2,5,6,7,12],"overal":0,"overrid":[0,12],"overwrit":[0,4],"owa":5,"own":[6,7,11,12],"ownership":6,"p":[3,10],"p12":4,"pack":4,"packag":[0,4,6],"packet":0,"pad":0,"page":[3,4,6,7,8],"paginate_messag":12,"pan":10,"parallel":12,"paramet":[0,12],"parent":7,"pars":[0,3,5,6,10,12],"parse_aggregate_report_fil":0,"parse_aggregate_report_xml":0,"parse_email":0,"parse_failure_report":0,"parse_forensic_report":0,"parse_report_email":0,"parse_report_fil":0,"parse_smtp_tls_report_json":0,"parsed_aggregate_reports_to_csv":0,"parsed_aggregate_reports_to_csv_row":0,"parsed_failure_reports_to_csv":0,"parsed_failure_reports_to_csv_row":0,"parsed_forensic_reports_to_csv":0,"parsed_forensic_reports_to_csv_row":0,"parsed_sampl":10,"parsed_smtp_tls_reports_to_csv":0,"parsed_smtp_tls_reports_to_csv_row":0,"parsedemail":0,"parsedmarc":[4,9,10,11],"parsedmarc_":12,"parsedmarc_config_fil":12,"parsedmarc_debug":12,"parsedmarc_elasticsearch_":12,"parsedmarc_elasticsearch_host":12,"parsedmarc_elasticsearch_ssl":12,"parsedmarc_gelf_":12,"parsedmarc_general_":12,"parsedmarc_general_debug":12,"parsedmarc_general_ipinfo_api_token":12,"parsedmarc_general_ipinfo_url":12,"parsedmarc_general_offlin":12,"parsedmarc_general_save_aggreg":12,"parsedmarc_general_save_failur":12,"parsedmarc_gmail_api_":12,"parsedmarc_gmail_api_credentials_file_fil":12,"parsedmarc_imap_":12,"parsedmarc_imap_host":12,"parsedmarc_imap_password":12,"parsedmarc_imap_password_fil":12,"parsedmarc_imap_us":12,"parsedmarc_kafka_":12,"parsedmarc_log_analytics_":12,"parsedmarc_mailbox_":12,"parsedmarc_mailbox_watch":12,"parsedmarc_maildir_":12,"parsedmarc_msgraph_":12,"parsedmarc_opensearch_":12,"parsedmarc_s3_":12,"parsedmarc_smtp_":12,"parsedmarc_splunk_hec_":12,"parsedmarc_splunk_hec_index":12,"parsedmarc_splunk_hec_token":12,"parsedmarc_splunk_hec_url":12,"parsedmarc_syslog_":12,"parsedmarc_webhook_":12,"parser":0,"parsererror":0,"parsingresult":0,"part":[3,4,7,8],"particular":7,"particularli":12,"pass":[0,3,7,10,12],"passag":7,"passsword":12,"password":[0,4,6,12],"past":[4,11],"patch":6,"path":[0,4,6,12],"pathlik":0,"pattern":[0,5,7],"payload":[0,12],"pct":10,"peak":12,"pem":12,"per":[0,7,12],"percentag":7,"perform":[2,5],"period":12,"perl":[0,6],"permiss":[4,12],"persist":12,"peter":10,"pick":[6,12],"pid":12,"pie":7,"pip":[6,12],"pkcs12":12,"place":[0,4,7,12],"plain":[0,12],"plaintext":[3,8],"platform":[3,6,8,12],"pleas":[1,5,12],"plu":7,"plug":12,"point":12,"polici":[3,7,8,10,12],"policy_domain":10,"policy_evalu":10,"policy_override_com":10,"policy_override_reason":10,"policy_publish":10,"policy_str":10,"policy_typ":10,"policyscopegroupid":12,"poll":[2,12],"popul":0,"port":[0,2,12],"posit":[0,12],"posix":0,"possibl":12,"post":[3,8,12],"poster":[3,8],"postgr":12,"postgresql":[5,12],"postoriu":[3,8],"powershel":12,"ppa":6,"practic":12,"pre":[6,12],"prebuilt":12,"predict":12,"prefer":[2,6,12],"prefix":[0,3,8,12],"premad":[5,11],"prerequisit":5,"present":12,"pressur":12,"pretti":12,"prettifi":12,"previou":[0,2,4,12],"previous":4,"primari":0,"print":12,"printabl":10,"prioriti":12,"privaci":[3,6,7,8,12],"privat":12,"probe":0,"proc":12,"process":[0,2,5,6,12],"produc":[0,10],"program":12,"programdata":6,"progress":12,"project":[0,2,3,5,11,12],"prompt":4,"proofpoint":5,"properti":2,"protect":[2,3,5,8,12],"protocol":12,"provid":[0,4,7,12],"prox":6,"proxi":2,"proxyhost":2,"proxypassword":2,"proxyport":2,"proxyus":2,"pry":[2,12],"psl":[0,12],"psl_overrid":0,"psl_overrides_path":0,"psl_overrides_url":[0,12],"psycopg":12,"public":[0,3,10,12],"public_suffix_list":0,"publicbaseurl":4,"publicsuffix":0,"publish":[3,12],"pull":12,"put":[4,12],"python":[0,6],"python3":6,"qo":4,"quarantin":[3,8],"queri":[0,12],"query_dn":0,"quickli":0,"quickstart":12,"quot":10,"quota":[0,12],"r":[2,10,12],"rais":0,"ram":[4,12],"rate":[0,12],"rather":[3,8,12],"raw":12,"re":[6,12],"read":[0,12],"readabl":[0,12],"readwrit":12,"realli":3,"reason":[0,2,4,5,12],"receiv":[0,7,10,12],"receiving_ip":10,"receiving_mx_hostnam":10,"recent":0,"recipi":7,"recogn":7,"recommend":12,"recommended_dns_nameserv":0,"record":[0,5,6,10],"record_typ":0,"redi":12,"reduc":12,"refer":[4,5],"referenc":12,"refresh":6,"refresh_interv":12,"regard":12,"regardless":[0,10],"region":[0,12],"region_nam":12,"regist":6,"registr":12,"regul":[4,6,9,12],"regular":[3,8],"reject":[0,3,8,12],"relai":[3,8],"relat":[3,12],"releas":[4,6],"reli":7,"reliabl":12,"reload":[0,2,4],"remain":[0,7,12],"remot":2,"remov":[0,3,4,8,12],"repeat":[0,3,8],"replac":[0,3,4,8,12],"repli":[2,3,8],"replic":12,"replica":[0,12],"reply_goes_to_list":[3,8],"reply_to":10,"replyto":[3,8],"repopul":0,"report":[0,4,11,12],"report_id":10,"report_metadata":10,"report_typ":0,"reported_domain":10,"reports_fold":[0,12],"repositori":[6,11],"req":4,"request":[0,2,4,12],"requir":[0,2,3,4,5,6,8,12],"require_encrypt":0,"reserv":12,"resid":12,"resolv":[0,12],"resourc":[0,4,5,12],"respons":[0,12],"rest":[0,12],"restart":[2,3,4,6,8],"restartsec":[2,12],"restor":4,"restrict":12,"restrictaccess":12,"result":[0,5,7,10,12],"result_typ":10,"resum":12,"retain":[3,8,12],"retent":5,"retri":[0,12],"retriev":2,"retry_attempt":12,"retry_delai":12,"return":0,"revers":[0,7,12],"reverse_dn":[0,10],"reverse_dns_base_domain":0,"reverse_dns_map":0,"reverse_dns_map_path":0,"reverse_dns_map_url":[0,12],"reversednsservic":0,"review":7,"rewrit":[0,3,8],"rfc":[0,3,5,8,10],"rfc2369":[3,8],"rfc822":2,"rhel":[4,5,6],"right":[4,7],"rm":4,"ro":0,"rocki":6,"rollup":6,"root":[2,12],"rpm":4,"rpt":[5,7],"rsa":4,"rua":[5,6],"ruf":[5,6,7,12],"rule":[7,12],"run":[0,4,5,6],"rw":[2,12],"s3":[5,12],"safe":0,"safer":12,"same":[0,3,4,6,7,11,12],"sampl":[0,5,7,12],"sample_headers_onli":10,"save":[0,4,6,7,12],"save_aggreg":12,"save_aggregate_report_to_elasticsearch":0,"save_aggregate_report_to_opensearch":0,"save_aggregate_reports_to_splunk":0,"save_failur":12,"save_failure_report_to_elasticsearch":0,"save_failure_report_to_opensearch":0,"save_failure_reports_to_splunk":0,"save_forens":12,"save_forensic_report_to_elasticsearch":0,"save_forensic_report_to_opensearch":0,"save_forensic_reports_to_splunk":0,"save_output":0,"save_smtp_tl":12,"save_smtp_tls_report_to_elasticsearch":0,"save_smtp_tls_report_to_opensearch":0,"save_smtp_tls_reports_to_splunk":0,"sbin":6,"schedul":[6,12],"schema":[5,10,12],"scope":[10,12],"script":6,"scrub_nondigest":[3,8],"search":[0,3,8,12],"second":[0,2,12],"secret_access_kei":12,"secur":[0,4,12],"see":[2,3,4,6,7,12],"seek":0,"segment":7,"select":0,"selector":10,"self":[4,5],"send":[0,2,3,4,5,7,8,11,12],"sender":[5,7,8],"sending_mta_ip":10,"sensit":12,"sent":[3,8,12],"sentinel":5,"separ":[0,3,4,6,7,9,11,12],"sequenc":0,"server":[0,2,3,4,5,6,7,10,12],"server_ip":4,"serverless":[0,12],"servernameon":10,"servic":[0,3,4,5,6,7,8,10],"service_account":12,"service_account_us":12,"session":[0,7],"set":[0,2,3,4,6,7,8,9,12],"set_host":0,"setup":[4,6,9,12],"shard":[0,12],"share":[4,6,12],"sharealik":6,"sharepoint":10,"shell":6,"ship":6,"should":[3,7,8,12],"shouldn":[3,8],"show":[2,7,12],"shown":12,"side":7,"sighup":[0,6,12],"sign":[0,3,4,6,12],"signal":12,"signatur":[3,7,8],"sigv4":[0,12],"silent":12,"similar":7,"simpl":5,"simplifi":0,"sinc":[0,6,12],"singl":[0,12],"sink":12,"sister":3,"size":[2,4],"skel":6,"skip":[0,12],"skip_certificate_verif":[0,12],"slightli":11,"slow":0,"small":[4,12],"smaller":12,"smtp":[0,3,5,12],"smtp_tl":[0,12],"smtp_tls_csv_filenam":[0,12],"smtp_tls_json_filenam":[0,12],"smtp_tls_report":0,"smtp_tls_url":12,"smtptlsfailuredetail":0,"smtptlsfailuredetailsopt":0,"smtptlsparsedreport":0,"smtptlspolici":0,"smtptlspolicysummari":0,"smtptlsreport":0,"so":[0,3,6,7,8,12],"socket":2,"solut":6,"some":[0,2,3,4,7,8],"someon":4,"sometim":12,"sort":[7,12],"sourc":[0,3,4,6,7,10],"source_as_domain":10,"source_as_nam":10,"source_asn":10,"source_base_domain":10,"source_countri":10,"source_ip_address":10,"source_nam":10,"source_reverse_dn":10,"source_typ":10,"sourceforg":2,"sp":[3,10],"spam":12,"special":12,"specif":[3,6,7,12],"specifi":[2,3],"spf":[7,10],"spf_align":10,"spf_domain":10,"spf_result":10,"spf_scope":10,"splunk":[5,12],"splunk_hec":12,"splunkerror":0,"splunkhec":12,"sponsor":5,"spoof":[3,8],"ss":0,"ssl":[0,2,4,12],"ssl_cert_path":0,"st":[7,10,12],"stabl":4,"stack":[4,7,12],"stai":7,"standard":[0,5,6,10],"start":[0,2,4,7,9,11,12],"starttl":[7,12],"startup":6,"statu":[2,12],"stdout":12,"step":[3,4,6,8],"still":[0,3,8,10,12],"storag":[0,12],"store":[2,4,9,12],"str":[0,12],"straight":12,"stream":12,"string":[0,12],"strip":[0,3,8,12],"strip_attachment_payload":[0,12],"strongli":12,"structur":5,"stsv1":10,"style":0,"subdomain":[0,3,12],"subject":[0,3,8,10,12],"subject_prefix":[3,8],"subsidiari":7,"substitut":6,"success":12,"successful_session_count":10,"sudo":[2,4,6,12],"suffix":0,"suggest":7,"suitabl":0,"summari":[3,8],"supervis":12,"suppli":[0,7,12],"support":[2,5,7,10,11],"sure":4,"surfac":7,"sw50zxjha3rpdmugv2v0dgjld2vyymvylcocymvyc2ljahq":10,"switch":7,"syslog":[2,5,12],"system":[2,3,4,6,8,12],"systemctl":[2,4,12],"systemd":5,"systemdr":6,"t":[5,8,10,12],"tab":[3,4,8],"tabl":[5,7,12],"tag":6,"take":[0,12],"target":[0,2,12],"task":6,"tby":10,"tcp":12,"tee":4,"tell":[3,7,8],"templat":[3,8],"temporari":7,"tenant":5,"tenant_id":12,"term":6,"test":[0,10,12],"text":[0,10],"than":[3,4,8,12],"thank":10,"thei":[3,7,8,12],"theirs":3,"them":[0,4,7,12],"therebi":[3,8],"thi":[0,2,3,4,5,6,7,8,10,12],"those":[0,12],"thousand":12,"three":7,"through":[0,3,12],"throughput":12,"tier":12,"time":[0,2,4,6,7,12],"timeout":[0,2,12],"timespan":0,"timespan_requires_norm":10,"timestamp":0,"timestamp_to_datetim":0,"timestamp_to_human":0,"timezon":10,"tl":[0,5,12],"tld":3,"to_domain":10,"to_utc":0,"togeth":7,"token":[0,4,12],"token_fil":12,"tool":12,"top":[3,7],"topic":12,"touch":[3,8],"tracker":1,"trade":12,"tradit":[3,8],"trail":12,"transfer":10,"transient":0,"transpar":5,"transport":[4,12],"trash":12,"tri":0,"true":[0,2,4,10,12],"trust":12,"truststor":4,"try":12,"tuesdai":6,"tune":5,"two":6,"txt":[0,12],"type":[5,7,10,12],"typo":12,"u":[2,6,10,12],"ubuntu":[4,6],"udp":[0,12],"ui":[3,8],"unchang":[0,12],"uncondition":[3,8],"under":[4,6,7,12],"underli":0,"underneath":7,"underscor":12,"understand":[5,7],"unencrypt":12,"unfortun":[3,8],"unit":[0,2,12],"unix":0,"unknown":0,"unless":[6,12],"unreach":12,"unread":12,"unsubscrib":[3,8],"until":[0,5,12],"unzip":2,"up":[0,2,4,6,7,9,12],"updat":[0,4,6,12],"upersecur":12,"upgrad":[2,5,6,12],"upload":12,"upper":7,"uppercas":12,"uri":[6,12],"url":[0,2,12],"us":[0,3,4,5,8,10],"usag":12,"use_ssl":0,"user":[0,2,3,4,6,7,8,10,12],"user_ag":10,"useradd":[2,6],"usernam":[0,12],"usernamepassword":12,"usesystemproxi":2,"usr":[4,6],"utc":0,"utf":10,"util":5,"v":12,"valid":[0,7,10,12],"valimail":5,"valu":[0,3,4,7,8,12],"var":[3,6,8,12],"variabl":5,"variant":12,"variou":6,"vendor":3,"venv":[6,12],"verbos":12,"veri":[4,7,12],"verif":[0,4,12],"verifi":0,"verification_mod":4,"version":[0,2,4,5,9,10,11,12],"vew":2,"via":[0,2],"view":[7,12],"vim":4,"virtualenv":6,"visual":[4,9],"volum":[7,12],"vulner":3,"w":[0,12],"w3c":10,"wa":[3,4,8],"wai":[0,4,7],"wait":[0,12],"want":[2,12],"wantedbi":[2,12],"warn":12,"watch":[0,2,4,6,12],"watch_inbox":0,"watcher":12,"web":[2,4],"webdav":2,"webhook":[5,12],"webmail":[3,7,8],"week":[0,12],"weekli":6,"well":[2,12],"were":12,"wettbewerb":10,"wget":4,"whalensolut":12,"what":5,"wheel":12,"when":[0,3,5,7,8,12],"whenev":[0,2,12],"where":[0,2,3,8,12],"wherea":7,"wherev":12,"whether":0,"which":[0,2,4,5,6,7,12],"while":[7,12],"who":7,"whole":0,"whose":12,"why":[3,7,12],"wide":[6,10,12],"wiki":10,"win":12,"window":[6,12],"within":0,"without":[3,4,7,8],"won":5,"work":[2,3,5,6,7,8,12],"worker":12,"workstat":2,"worst":3,"would":[3,6,8,12],"wrap":[3,8],"wrapper":12,"write":[0,12],"written":12,"www":[4,6,12],"x":[4,7,10],"x509":4,"xennn":10,"xml":[0,11],"xml_schema":10,"xms4g":4,"xmx4g":4,"xpack":4,"xxxx":4,"y":[4,6],"yahoo":7,"yaml":12,"ye":[3,8],"year":12,"yet":3,"yml":4,"you":[2,3,4,5,6,7,8,12],"your":[3,4,5,6,7,8,11,12],"yyyi":0,"zero":12,"zip":[0,2,5,12],"\u00fcbersicht":10},"titles":["API reference","Contributing to parsedmarc","Accessing an inbox using OWA/EWS","Understanding DMARC","Elasticsearch and Kibana","parsedmarc documentation - Open source DMARC report analyzer and visualizer","Installation","Using the Kibana dashboards","What about mailing lists?","OpenSearch and Grafana","Sample outputs","Splunk","Using parsedmarc"],"titleterms":{"2":[3,8],"3":[3,8],"_file":12,"about":[3,8],"access":2,"aggreg":[7,10],"align":3,"an":2,"analyz":[5,6],"api":0,"best":[3,8],"bug":1,"cli":12,"compat":5,"compos":12,"config":12,"configur":[2,12],"content":5,"contribut":1,"countri":6,"csv":10,"dashboard":7,"databas":6,"davmail":2,"depend":6,"dkim":3,"dmarc":[3,5,7],"do":[3,8],"docker":12,"document":5,"domain":3,"elast":0,"elasticsearch":4,"env":12,"environ":12,"ew":2,"exampl":12,"exchang":6,"failur":[7,10],"featur":5,"file":12,"geolite2":6,"grafana":9,"guid":3,"help":12,"inbox":2,"index":4,"indic":0,"instal":[4,6,9],"ip":6,"json":10,"kibana":[4,7],"list":[3,8],"listserv":[3,8],"lookalik":3,"mail":[3,8],"mailman":[3,8],"map":12,"maxmind":6,"microsoft":6,"mode":12,"multi":12,"multipl":6,"name":12,"onli":12,"open":5,"opensearch":[0,9],"option":6,"output":10,"owa":2,"parsedmarc":[0,1,2,5,6,12],"pattern":4,"perform":12,"practic":[3,8],"prerequisit":6,"proxi":6,"python":5,"record":[3,4,9],"refer":0,"reload":12,"report":[1,5,6,7,10],"resourc":3,"restart":12,"retent":[4,9],"run":[2,12],"sampl":10,"secret":12,"section":12,"sender":3,"servic":[2,12],"smtp":[7,10],"sourc":5,"specifi":12,"spf":3,"splunk":[0,11],"suffix":12,"support":[3,12],"systemd":[2,12],"t":3,"tabl":0,"tenant":12,"test":6,"tl":[7,10],"tune":12,"type":0,"understand":3,"upgrad":4,"us":[2,6,7,12],"util":0,"valid":3,"variabl":12,"via":12,"visual":5,"web":6,"what":[3,8],"without":12,"won":3,"workaround":[3,8]}}) \ No newline at end of file diff --git a/splunk.html b/splunk.html index d96d487..8379f1f 100644 --- a/splunk.html +++ b/splunk.html @@ -6,14 +6,14 @@ - Splunk — parsedmarc 9.11.2 documentation + Splunk — parsedmarc 10.0.0 documentation - + @@ -84,9 +84,9 @@

    Splunk

    Starting in version 4.3.0 parsedmarc supports sending aggregate and/or -forensic DMARC data to a Splunk HTTP Event collector (HEC).

    +failure DMARC data to a Splunk HTTP Event collector (HEC).

    The project repository contains XML files for premade Splunk -dashboards for aggregate and forensic DMARC reports.

    +dashboards for aggregate and failure DMARC reports.

    Copy and paste the contents of each file into a separate Splunk dashboard XML editor.

    diff --git a/usage.html b/usage.html index 72326f5..3ab54a2 100644 --- a/usage.html +++ b/usage.html @@ -6,14 +6,14 @@ - Using parsedmarc — parsedmarc 9.11.2 documentation + Using parsedmarc — parsedmarc 10.0.0 documentation - + @@ -53,6 +53,7 @@
  • Specifying the config file via environment variable
  • Running without a config file (env-only mode)
  • Docker Compose example
    • +
  • Docker secrets (_FILE suffix)
  • Section name mapping
    • @@ -104,9 +105,9 @@

    CLI help

    usage: parsedmarc [-h] [-c CONFIG_FILE] [--strip-attachment-payloads] [-o OUTPUT]
-                  [--aggregate-json-filename AGGREGATE_JSON_FILENAME] [--forensic-json-filename FORENSIC_JSON_FILENAME]
+                  [--aggregate-json-filename AGGREGATE_JSON_FILENAME] [--failure-json-filename FAILURE_JSON_FILENAME]
                   [--smtp-tls-json-filename SMTP_TLS_JSON_FILENAME] [--aggregate-csv-filename AGGREGATE_CSV_FILENAME]
-                  [--forensic-csv-filename FORENSIC_CSV_FILENAME] [--smtp-tls-csv-filename SMTP_TLS_CSV_FILENAME]
+                  [--failure-csv-filename FAILURE_CSV_FILENAME] [--smtp-tls-csv-filename SMTP_TLS_CSV_FILENAME]
                   [-n NAMESERVERS [NAMESERVERS ...]] [-t DNS_TIMEOUT] [--offline] [-s] [-w] [--verbose] [--debug]
                   [--log-file LOG_FILE] [--no-prettify-json] [-v]
                   [file_path ...]
@@ -114,26 +115,26 @@
 Parses DMARC reports
 
 positional arguments:
-  file_path             one or more paths to aggregate or forensic report files, emails, or mbox files'
+  file_path             one or more paths to aggregate or failure report files, emails, or mbox files'
 
 options:
   -h, --help            show this help message and exit
   -c CONFIG_FILE, --config-file CONFIG_FILE
                         a path to a configuration file (--silent implied)
   --strip-attachment-payloads
-                        remove attachment payloads from forensic report output
+                        remove attachment payloads from failure report output
   -o OUTPUT, --output OUTPUT
                         write output files to the given directory
   --aggregate-json-filename AGGREGATE_JSON_FILENAME
                         filename for the aggregate JSON output file
-  --forensic-json-filename FORENSIC_JSON_FILENAME
-                        filename for the forensic JSON output file
+  --failure-json-filename FAILURE_JSON_FILENAME
+                        filename for the failure JSON output file
   --smtp-tls-json-filename SMTP_TLS_JSON_FILENAME
                         filename for the SMTP TLS JSON output file
   --aggregate-csv-filename AGGREGATE_CSV_FILENAME
                         filename for the aggregate CSV output file
-  --forensic-csv-filename FORENSIC_CSV_FILENAME
-                        filename for the forensic CSV output file
+  --failure-csv-filename FAILURE_CSV_FILENAME
+                        filename for the failure CSV output file
   --smtp-tls-csv-filename SMTP_TLS_CSV_FILENAME
                         filename for the SMTP TLS CSV output file
   -n NAMESERVERS [NAMESERVERS ...], --nameservers NAMESERVERS [NAMESERVERS ...]
@@ -167,7 +168,7 @@ configuration file, described below.
    
 
 [general]
 save_aggregate = True
-save_forensic = True
+save_failure = True
 
 [imap]
 host = imap.example.com
@@ -206,7 +207,7 @@ configuration file, described below.
    
 
 [webhook]
 aggregate_url = https://aggregate_url.example.com
-forensic_url = https://forensic_url.example.com
+failure_url = https://failure_url.example.com
 smtp_tls_url = https://smtp_tls_url.example.com
 timeout = 60
    @@ -217,7 +218,7 @@ configuration file, described below.

    • save_aggregate - bool: Save aggregate report data to Elasticsearch, Splunk and/or S3

      • -

    • save_forensic - bool: Save forensic report data to +

    • save_failure - bool: Save failure report data to Elasticsearch, Splunk and/or S3

    • save_smtp_tls - bool: Save SMTP-STS report data to Elasticsearch, Splunk and/or S3

      • @@ -228,7 +229,7 @@ payloads from results

    • output - str: Directory to place JSON and CSV files in. This is required if you set either of the JSON output file options.

    • aggregate_json_filename - str: filename for the aggregate JSON output file

      • -

    • forensic_json_filename - str: filename for the forensic +

    • failure_json_filename - str: filename for the failure JSON output file

    • ip_db_path - str: An optional custom path to a MMDB file from IPinfo, MaxMind, or DBIP

      • @@ -406,6 +407,12 @@ verification (not recommended)

      creating the index (Default: 1)

    • number_of_replicas - int: The number of replicas to use when creating the index (Default: 0)

      • +

    • serverless - bool: Set to True when targeting an Elastic Cloud +Serverless project. Serverless manages sharding and replication itself +and rejects the number_of_shards / number_of_replicas index settings +with HTTP 400. With this flag set, parsedmarc strips those keys from the +settings sent at index creation; any other settings (e.g. +refresh_interval) are passed through unchanged (Default: False)

  • opensearch

    @@ -459,7 +466,7 @@ verification (not recommended)

  • skip_certificate_verification - bool: Skip certificate verification (not recommended)

  • aggregate_topic - str: The Kafka topic for aggregate reports

    • -

  • forensic_topic - str: The Kafka topic for forensic reports

    • +

  • failure_topic - str: The Kafka topic for failure reports

  • smtp

    @@ -486,6 +493,47 @@ so use %%
    • +

  • postgresql

    +
      +

    • host - str: The PostgreSQL server hostname or IP address. +Required unless connection_string is provided.

      • +

    • port - int: The PostgreSQL server port (Default: 5432)

      • +

    • user - str: The database user name (Optional)

      • +

    • password - str: The database user password (Optional)

      • +

    • database - str: The database name (Optional)

      • +

    • connection_string - str: A full libpq connection string or URI +(e.g. postgresql://user:pass@host/dbname). When provided, +all individual parameters above are ignored.

      • +
    +

    The PostgreSQL backend is an optional extra. Install it with +pip install parsedmarc[postgresql] (it pulls in psycopg); the +prebuilt binary wheels are not available for every platform, which is +why it is not a mandatory dependency.

    +

    Tables are created automatically on first run using +CREATE TABLE IF NOT EXISTS, so no manual schema migration is needed +for fresh installations.

    +

    Example configuration:

    +
    [postgresql]
+host = localhost
+port = 5432
+user = parsedmarc
+password = secret
+database = parsedmarc
+
    +
    +

    Or using a DSN/URI:

    +
    [postgresql]
+connection_string = postgresql://parsedmarc:secret@localhost/parsedmarc
+
    +
    +

    Saving parsed data to PostgreSQL is controlled by the [general] +options save_aggregate, save_failure, and save_smtp_tls +(save_forensic is still accepted as a deprecated alias for +save_failure). These flags must be set to True for the +corresponding report types (aggregate DMARC, failure DMARC, and +SMTP TLS reports) or no data will be written to PostgreSQL, even if +this section is configured.

    +

  • s3

    • bucket - str: The S3 bucket name

      • @@ -586,7 +634,7 @@ When False

      dce - str: The Data Collection Endpoint (DCE). Example: https://{DCE-NAME}.{REGION}.ingest.monitor.azure.com.

    • dcr_immutable_id - str: The immutable ID of the Data Collection Rule (DCR)

    • dcr_aggregate_stream - str: The stream name for aggregate reports in the DCR

      • -

    • dcr_forensic_stream - str: The stream name for the forensic reports in the DCR

      • +

    • dcr_failure_stream - str: The stream name for the failure reports in the DCR

    • dcr_smtp_tls_stream - str: The stream name for the SMTP TLS reports in the DCR

    @@ -603,14 +651,14 @@ When False

  • maildir

      -

    • maildir_path - str: Full path for mailbox maidir location (Default: INBOX)

      • +

    • maildir_path - str: Full path for mailbox maildir location (Default: INBOX)

    • maildir_create - bool: Create maildir if not present (Default: False)

  • webhook - Post the individual reports to a webhook url with the report as the JSON body

    • aggregate_url - str: URL of the webhook which should receive the aggregate reports

      • -

    • forensic_url - str: URL of the webhook which should receive the forensic reports

      • +

    • failure_url - str: URL of the webhook which should receive the failure reports

    • smtp_tls_url - str: URL of the webhook which should receive the smtp_tls reports

    • timeout - int: Interval in which the webhook call should timeout

    @@ -627,23 +675,23 @@ blocks DNS requests to outside resolvers.

    • Note

    -

    save_aggregate and save_forensic are separate options -because you may not want to save forensic reports -(also known as failure reports) to your Elasticsearch instance, +

    save_aggregate and save_failure are separate options +because you may not want to save failure reports +(formerly known as forensic reports) to your Elasticsearch instance, particularly if you are in a highly-regulated industry that handles sensitive data, such as healthcare or finance. If your legitimate outgoing email fails DMARC, it is possible -that email may appear later in a forensic report.

    -

    Forensic reports contain the original headers of an email that +that email may appear later in a failure report.

    +

    Failure reports contain the original headers of an email that failed a DMARC check, and sometimes may also include the full message body, depending on the policy of the reporting organization.

    -

    Most reporting organizations do not send forensic reports of any +

    Most reporting organizations do not send failure reports of any kind for privacy reasons. While aggregate DMARC reports are sent -at least daily, it is normal to receive very few forensic reports.

    -

    An alternative approach is to still collect forensic/failure/ruf +at least daily, it is normal to receive very few failure reports.

    +

    An alternative approach is to still collect failure/ruf reports in your DMARC inbox, but run parsedmarc with -save_forensic = True manually on a separate IMAP folder (using +save_failure = True manually on a separate IMAP folder (using the reports_folder option), after you have manually moved known samples you want to save to that folder (e.g. malicious samples and non-sensitive legitimate samples).

    @@ -736,10 +784,51 @@ parsedmarc /path/to/reports/* PARSEDMARC_MAILBOX_WATCH: "true" PARSEDMARC_ELASTICSEARCH_HOSTS: http://elasticsearch:9200 PARSEDMARC_GENERAL_SAVE_AGGREGATE: "true" - PARSEDMARC_GENERAL_SAVE_FORENSIC: "true" + PARSEDMARC_GENERAL_SAVE_FAILURE: "true"
    +
    +

    Docker secrets (_FILE suffix)

    +

    Any PARSEDMARC_{SECTION}_{KEY} environment variable can also be supplied +via a file by appending _FILE to its name. The file’s contents (with any +trailing CR/LF characters stripped) are used as the value. This is the +same convention used by the official Postgres, MariaDB, and Redis container +images, and is designed to plug straight into Docker / Docker Compose / +Kubernetes secrets so credentials never appear in plain environment: +blocks (where they would be readable via docker inspect, container logs, +and /proc/<pid>/environ).

    +

    The bare DEBUG / PARSEDMARC_DEBUG aliases and PARSEDMARC_CONFIG_FILE +do not have a _FILE form; only PARSEDMARC_{SECTION}_{KEY} vars resolved +to a known config section are eligible.

    +

    If both the direct env var and the _FILE variant are set, the _FILE +variant wins. If the file does not exist or is unreadable, parsedmarc +exits with a configuration error rather than silently falling back to an +empty value.

    +
    secrets:
+  imap_password:
+    file: ./secrets/imap_password.txt
+
+services:
+  parsedmarc:
+    image: parsedmarc:latest
+    secrets:
+      - imap_password
+    environment:
+      PARSEDMARC_IMAP_HOST: imap.example.com
+      PARSEDMARC_IMAP_USER: dmarc@example.com
+      PARSEDMARC_IMAP_PASSWORD_FILE: /run/secrets/imap_password
+
    +
    +

    Note that a small set of config keys whose own names already end in +_file ([general] log_file, [msgraph] token_file, +[gmail_api] credentials_file, [gmail_api] token_file) keep their +pre-existing meaning when set via PARSEDMARC_..._FILE — that env var is +the path itself, not a wrapper around a file containing the path. To pass +those paths via a Docker secret, double up the suffix +(PARSEDMARC_GMAIL_API_CREDENTIALS_FILE_FILE); the inner contents are +then read and stored as the credentials_file value.

    +

    Section name mapping

    For sections with underscores in the name, the full section name is used:

    @@ -817,7 +906,7 @@ a safer starting point for large backfills than aggressive parallelism.

  • Use mailbox.since to process reports in smaller time windows such as 1d, 7d, or another interval that fits the backlog. This makes it easier to catch up incrementally instead of loading an entire mailbox history in one run.

    • -

  • Set strip_attachment_payloads = True when forensic reports contain large +

  • Set strip_attachment_payloads = True when failure reports contain large attachments and you do not need to retain the raw payloads in the parsed output.

  • Prefer running parsedmarc separately from Elasticsearch or OpenSearch, or