amorfo77/parsedmarc
1
0 Fork 0
mirror of https://github.com/domainaware/parsedmarc.git synced 2026-11-11 11:21:32 +00:00
Code Issues Projects Releases Wiki Activity
Files
70a0ea0470d04649e33e6b09ce20da991d424f72
parsedmarc/parsedmarc/__init__.py
T
Sean WhalenandClaude Fable 5.2 771c6df0bd PostgreSQL connection follow-ups and the message_count mapping (#959) 
- Accept a URI whose host has an empty port (postgresql://db:/d?
  sslrootcert=/a@b): libpq parses it as no port, but _port_is_numeric()
  treated it as non-numeric and the early refusal rejected the string.
- Close the connection when setting autocommit or registering the
  NUL-safe dumpers fails after psycopg.connect() succeeded, including
  on KeyboardInterrupt; an error from close() is dropped.
- With a connection string, report a create_tables() failure with a
  fixed message (plus a well-formed SQLSTATE), since the server's text
  can quote a session setting from the string's options; with the
  separate settings keep the server's text, URL credentials masked.
  Both raise from None. The CLI now closes a PostgreSQL client whose
  create_tables() raised, as it is not yet in the clients to roll back.
- Declare the Elasticsearch and OpenSearch aggregate documents'
  message_count and original_timespan_seconds as Long() instead of the
  bare Integer class, matching the long type dynamic mapping has always
  given them (the document classes' mappings are never installed).
  Correct the MAX_MESSAGE_COUNT comment, the _smtp_tls_count docstring
  and the GHSA-fxr7 changelog entry, which described an integer
  Elasticsearch/OpenSearch mapping of the counts that is not installed.

Co-authored-by: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-10 19:15:01 -05:00

5181 lines
215 KiB
Python
Raw Blame History

# -*- coding: utf-8 -*-
"""A Python package for parsing DMARC reports"""
from __future__ import annotations
import binascii
import email
import email.errors
import email.message
import email.utils
import functools
import hashlib
import ipaddress
import json
import logging
import mailbox
import os
import quopri
import re
import shutil
import tempfile
import time
import traceback
import xml.parsers.expat as expat
import zipfile
import zlib
from base64 import b64decode
from collections import deque
from collections.abc import Callable, Sequence
from csv import DictWriter
from dataclasses import dataclass, field
from datetime import date, datetime, timedelta, timezone, tzinfo
from io import BytesIO, StringIO
from typing import (
Any,
BinaryIO,
cast,
)
import lxml.etree as etree
import mailparser
import xmltodict
from expiringdict import ExpiringDict
from mailsuite.smtp import send_email
from mailsuite.utils import MessageLimitExceeded, check_message_limits
from tqdm import tqdm
from parsedmarc.attempts import MAX_PARSE_ATTEMPTS, AttemptStore, attempt_key
from parsedmarc.config import (
IP_ADDRESS_CACHE,
REVERSE_DNS_MAP,
SEEN_AGGREGATE_REPORT_IDS,
ParserConfig,
)
from parsedmarc.constants import (
DEFAULT_DNS_MAX_RETRIES,
DEFAULT_DNS_TIMEOUT,
MAX_FAILED_REVERSE_DNS_SECONDS_PER_REPORT,
MAX_REVERSE_DNS_SECONDS_PER_REPORT,
SLOW_REVERSE_DNS_SECONDS,
MAX_MESSAGE_COUNT,
MAX_NORMALIZED_RECORDS,
MAX_REPORT_FUTURE_DAYS,
MAX_REPORT_SPAN_DAYS,
MAX_XML_DEPTH,
MAX_XML_NODES,
MAX_XML_TAG_LENGTH,
__version__,
)
from parsedmarc.log import logger
from parsedmarc.mail import (
GmailConnection,
IMAPConnection,
MailboxConnection,
MaildirConnection,
MSGraphConnection,
)
from parsedmarc.types import (
AggregateReport,
FailureReport,
ForensicReport as ForensicReport,
ParsedReport,
ParsingResults,
ReportType,
SMTPTLSReport,
)
from parsedmarc.utils import (
IPAddressInfo,
convert_outlook_msg,
get_base_domain,
get_filename_safe_string,
get_ip_address_info,
human_timestamp_to_datetime,
human_timestamp_to_unix_timestamp,
is_outlook_msg,
parse_email,
sanitize_for_log,
timestamp_to_human,
)
logger.debug(f"parsedmarc v{__version__}")
# A message/feedback-report part is a MIME body part, so its lines end with
# CRLF per RFC 5322 §2.1. In re.MULTILINE, ``$`` matches before the LF but not
# before the CR, so the value must exclude the line ending explicitly rather
# than relying on ``.`` — otherwise every field value keeps a trailing CR.
feedback_report_regex = re.compile(r"^([\w\-]+): ([^\r\n]+)\r?$", re.MULTILINE)
xml_header_regex = re.compile(r"^<\?xml .*?>", re.MULTILINE)
# Every pattern here runs on attacker-controlled input; the next two were
# rewritten to match in linear time (GHSA-4h9r-c435-j834, GHSA-7grx-mj6g-qxmc):
# a repeated character class excludes the delimiter it stops on, adjacent
# quantifiers never overlap, and text_report_regex is anchored to a line start,
# so each candidate match is scanned once instead of being rescanned from every
# offset on every backtrack. See "Regular expressions on untrusted input" in
# AGENTS.md.
xml_schema_regex = re.compile(r"</?xs:schema[^<>]*>")
text_report_regex = re.compile(
r"^[ \t]*([A-Za-z]+(?: [A-Za-z]+)*):[ \t]+([^\r\n]+)", re.MULTILINE
)
# Captures the value of any xmlns (default or prefixed) declaration so the
# RFC 9990 namespace can be detected before xmltodict drops it.
xml_namespace_regex = re.compile(
r"""xmlns(?::[a-zA-Z_][\w.-]*)?\s*=\s*["']([^"']+)["']"""
)
# The XML namespace assigned to DMARC aggregate reports by RFC 9990.
RFC_9990_NAMESPACE = "urn:ietf:params:xml:ns:dmarc-2.0"
# PolicyOverrideType enumeration from RFC 9990. Compared to RFC 7489,
# `policy_test_mode` was added (emitted when t=y suppresses enforcement)
# and `forwarded` / `sampled_out` were removed.
RFC_9990_POLICY_OVERRIDE_TYPES = frozenset(
{
"local_policy",
"mailing_list",
"other",
"policy_test_mode",
"trusted_forwarder",
}
)
RFC_7489_REMOVED_POLICY_OVERRIDE_TYPES = frozenset({"forwarded", "sampled_out"})
MAGIC_ZIP = b"\x50\x4b\x03\x04"
MAGIC_GZIP = b"\x1f\x8b"
MAGIC_XML = b"\x3c\x3f\x78\x6d\x6c\x20"
MAGIC_XML_TAG = b"\x3c" # '<' - XML starting with an element tag (no declaration)
# 0x7B, "{" -- a JSON text that is an object begins with it (RFC 8259).
# Previously written as b"\7b", which Python reads as the octal escape
# \7 (BEL) followed by a literal "b", so the branch never matched real
# JSON; every in-tree caller happened to pre-guard with its own zip/gzip
# or "{" check, which masked it.
MAGIC_JSON = b"\x7b"
# Maximum size, in bytes, of the data extracted from one compressed report
# attachment, and of an uncompressed report or report file measured as UTF-8
# (_exceeds_size_cap()), so a non-ASCII text cannot be four times the cap in
# bytes. Real reports compress at roughly 10:1, but the attachment's
# content is chosen by whoever sent it, and deflate (used by both gzip and
# zip) reaches about 1000:1 on degenerate input, so a 32 KB attachment from
# any sender a monitored mailbox accepts could inflate to 32 MB -- and the
# decoded str coexists with the decompressed bytes, so the peak is roughly
# twice that. Deliberately not configurable: an aggregate report is a day of
# one reporter's traffic, the largest sample is 909 KB, and the XML node
# limit (MAX_XML_NODES, 1,000,000 nodes at the samples' 35 nodes per KB)
# already rejects a report of more than about 28 MB (GHSA-q7gm-8cj9-94hr).
MAX_DECOMPRESSED_REPORT_SIZE = 32 * 1024 * 1024
# Per-message count of consecutive failed saves, keyed on
# ``(reports_folder, str(message_uid))``. Populated only when
# ``get_dmarc_reports_from_mailbox()`` is given a ``save_callback`` that
# reports a batch as unsaved; a message whose count exceeds
# ``max_unsaved_retries`` is moved to ``{archive_folder}/Unsaved`` instead
# of being retried forever, and its entry is dropped. A successful save
# clears the entries for that batch's messages.
#
# Process-local and deliberately not persisted: a restart re-attempts every
# message still in the reports folder, which is the safe direction (retry
# rather than shelve). The key carries no connection identity, so a library
# caller processing two connections that share a folder name (two IMAP
# servers, both "INBOX") in one process could collide counters if UIDs
# happen to match; the CLI uses one connection per process. It is not a
# ``ParserConfig`` field for the same reason ``batch_size`` and the
# ``delete`` flags are not -- it governs mailbox orchestration, not
# parsing.
_FAILED_SAVE_ATTEMPTS: dict[tuple[str, str], int] = {}
EMAIL_SAMPLE_CONTENT_TYPES = (
"text/rfc822",
"text-rfc-822",
"text/rfc822-headers",
"text/rfc-822-headers",
"message/rfc822",
"message/rfc-822",
"message/rfc822-headers",
"message/rfc-822-headers",
)
class ParserError(RuntimeError):
"""Raised whenever the parser fails for some reason"""
class InvalidDMARCReport(ParserError):
"""Raised when an invalid DMARC report is encountered"""
class InvalidSMTPTLSReport(ParserError):
"""Raised when an invalid SMTP TLS report is encountered"""
class ParseTimeoutError(InvalidDMARCReport):
"""Returned in place of a report whose parsing exceeded ``parse_timeout``
An ``InvalidDMARCReport``, so a timed-out message is handled like any
other message that cannot be parsed: the mbox loop warns and continues,
and the mailbox loop moves it to the ``Invalid`` archive folder. It is
the value for every timed-out input, including one that would have
been an SMTP TLS report or not a report at all, since a parse that was
stopped cannot tell what the input held.
"""
class InvalidAggregateReport(InvalidDMARCReport):
"""Raised when an invalid DMARC aggregate report is encountered"""
class InvalidFailureReport(InvalidDMARCReport):
"""Raised when an invalid DMARC failure report is encountered"""
# Backward-compatible alias
InvalidForensicReport = InvalidFailureReport
_DOCTYPE_ERROR = "Aggregate reports must not contain a DOCTYPE declaration"
class _XMLDoctypeFound(Exception):
"""Raised by :class:`_NoDoctypeExpat` parsers as soon as expat reports
the start of a ``<!DOCTYPE`` declaration."""
def _raise_doctype_found(*args: Any) -> None:
raise _XMLDoctypeFound(_DOCTYPE_ERROR)
class _NoDoctypeExpat:
"""A minimal stand-in for the ``xml.parsers.expat`` module, passed to
``xmltodict.parse(expat=...)``, whose parsers abort on any DOCTYPE.
An internal DTD subset can declare default attributes
(``<!ATTLIST z a0 CDATA "v" ... aK ...>``) that expat then reports on
every matching element, so K declared defaults on N empty elements make
xmltodict build K x N dict entries from a few kilobytes of input
(GHSA-m47j-wp78-fqjg). Entity expansion is already off
(xmltodict's ``disable_entities=True``) but does not cover this.
DMARC aggregate reports never carry a DOCTYPE, so the declaration is
rejected outright, before expat reads any of the subset.
``xmltodict.parse()`` only ever calls ``ParserCreate()`` on the module it
is given.
"""
@staticmethod
def ParserCreate(*args: Any, **kwargs: Any) -> Any:
parser = expat.ParserCreate(*args, **kwargs)
parser.StartDoctypeDeclHandler = _raise_doctype_found
return _LimitedExpatParser(parser)
class _XMLLimitExceeded(Exception):
"""Raised by :class:`_LimitedExpatParser` as soon as a document goes over
``MAX_XML_DEPTH`` or ``MAX_XML_NODES``."""
class _LimitedExpatParser:
"""Wraps an expat parser so that the element handlers xmltodict installs
count nesting depth and nodes (elements plus attributes), and raise
:class:`_XMLLimitExceeded` over ``MAX_XML_DEPTH`` or ``MAX_XML_NODES``.
xmltodict keeps one stack entry per open element, so a document of
millions of unclosed ``<a>`` tags, or of millions of siblings, could cost
gigabytes before expat reported the error at the end of the input
(GHSA-q7gm-8cj9-94hr). The check runs before xmltodict's handler, so the
tree never grows past the limits.
pyexpat parser objects cannot be subclassed, and xmltodict assigns its
handlers after ``ParserCreate()`` returns, so this proxy wraps the
handlers in ``__setattr__`` and forwards every other attribute. The
wrappers keep their counts in a :class:`_XMLCounts`, not on the proxy:
a wrapper referring to the proxy would close a reference cycle through
the parser and keep xmltodict's whole tree alive until the cyclic
garbage collector ran, doubling peak memory across the two parses.
"""
def __init__(self, parser: Any) -> None:
object.__setattr__(self, "_parser", parser)
object.__setattr__(self, "_counts", _XMLCounts())
def __getattr__(self, name: str) -> Any:
return getattr(self._parser, name)
def __setattr__(self, name: str, value: Any) -> None:
if name == "StartElementHandler":
value = _limit_start(value, self._counts)
elif name == "EndElementHandler":
value = _limit_end(value, self._counts)
setattr(self._parser, name, value)
class _XMLCounts:
"""The open-element depth and node count of one expat parse."""
def __init__(self) -> None:
self.depth = 0
self.nodes = 0
def _limit_start(handler: Callable[..., Any], counts: _XMLCounts) -> Callable[..., Any]:
"""Wrap an expat start-element handler so that it counts depth and
nodes in ``counts`` and raises :class:`_XMLLimitExceeded` over the
limits before ``handler`` runs."""
def start(name: str, attributes: list[str]) -> Any:
depth = counts.depth + 1
# xmltodict sets ordered_attributes, so expat passes attributes as
# a flat [name, value, name, value, ...] list.
nodes = counts.nodes + 1 + len(attributes) // 2
if depth > MAX_XML_DEPTH:
raise _XMLLimitExceeded(
f"elements nested more than {MAX_XML_DEPTH} levels deep"
)
if nodes > MAX_XML_NODES:
raise _XMLLimitExceeded(
f"more than {MAX_XML_NODES:,} elements and attributes"
)
counts.depth = depth
counts.nodes = nodes
return handler(name, attributes)
return start
def _limit_end(handler: Callable[..., Any], counts: _XMLCounts) -> Callable[..., Any]:
"""Wrap an expat end-element handler so that it decrements the depth in
``counts``."""
def end(name: str) -> Any:
counts.depth -= 1
return handler(name)
return end
class _LxmlNodeCounter:
"""An lxml parser target that counts elements, attributes and namespace
declarations without building a tree, and raises
:class:`_XMLLimitExceeded` past ``MAX_XML_NODES``.
It runs before the lxml recovery parse, whose tree would otherwise cost
gigabytes for a wide document behind a malformed XML declaration
(GHSA-q7gm-8cj9-94hr). Depth is not counted here: libxml2 stops at 256
levels without ``huge_tree``, and the second ``xmltodict.parse()``
enforces ``MAX_XML_DEPTH`` on the recovered document.
"""
def __init__(self) -> None:
self.nodes = 0
def _add(self, count: int) -> None:
self.nodes += count
if self.nodes > MAX_XML_NODES:
raise _XMLLimitExceeded(
f"more than {MAX_XML_NODES:,} elements and attributes"
)
def start(self, tag: str, attrib: dict[str, str]) -> None:
self._add(1 + len(attrib))
def start_ns(self, prefix: str | None, uri: str) -> None:
# xmltodict, which does not process namespaces, sees each xmlns
# declaration as an attribute.
self._add(1)
def end(self, tag: str) -> None:
pass
def close(self) -> int:
return self.nodes
def _parse_xml_with_recovery(xml: bytes) -> Any:
"""Parse ``xml`` with lxml in recovery mode and return the tree, after a
counting pass that raises :class:`_XMLLimitExceeded` over
``MAX_XML_NODES``, so the tree is only built for a document within the
limit."""
etree.parse(
BytesIO(xml),
etree.XMLParser(
recover=True, resolve_entities=False, target=_LxmlNodeCounter()
),
)
return etree.parse(
BytesIO(xml), etree.XMLParser(recover=True, resolve_entities=False)
)
def _resolve_config(
config: ParserConfig | None,
*,
offline: bool = False,
ip_db_path: str | None = None,
always_use_local_files: bool = False,
reverse_dns_map_path: str | None = None,
reverse_dns_map_url: str | None = None,
nameservers: list[str] | None = None,
dns_timeout: float = DEFAULT_DNS_TIMEOUT,
dns_retries: int = DEFAULT_DNS_MAX_RETRIES,
strip_attachment_payloads: bool = False,
normalize_timespan_threshold_hours: float = 24.0,
) -> ParserConfig:
"""Resolve the effective :class:`~parsedmarc.config.ParserConfig` for a
public parsing call, from either an explicit ``config`` or the caller's
individual option keyword arguments.
If ``config`` is not ``None``, it is returned unchanged: per the
documented ``config=`` contract, the individual option keyword arguments
are ignored in favor of the config's own values, so no merging happens
here.
Otherwise, a new ``ParserConfig`` is built from the given keyword
arguments. Its three cache fields are deliberately *not* left to their
``default_factory`` -- doing so would hand back a config with brand new,
empty caches on every call, silently defeating cross-call IP-info
caching and aggregate-report dedup. Instead, the module-default caches
(:data:`IP_ADDRESS_CACHE`, :data:`SEEN_AGGREGATE_REPORT_IDS`,
:data:`REVERSE_DNS_MAP`) are injected explicitly, so kwargs-style calls
keep observing and mutating the same shared caches they always have.
``dataclasses.replace`` is deliberately not used here: it would need an
existing ``ParserConfig`` to start from, and there isn't one on the
kwargs path -- this function's job is to build the first one.
``psl_overrides_path`` / ``psl_overrides_url`` have no corresponding
keyword arguments on the public functions (see AGENTS.md's guidance on
justifying new config options), so they stay ``None`` on this path; they
are only ever set via an explicitly constructed ``config=``.
"""
if config is not None:
return config
return ParserConfig(
offline=offline,
ip_db_path=ip_db_path,
always_use_local_files=always_use_local_files,
reverse_dns_map_path=reverse_dns_map_path,
reverse_dns_map_url=reverse_dns_map_url,
nameservers=nameservers,
dns_timeout=dns_timeout,
dns_retries=dns_retries,
strip_attachment_payloads=strip_attachment_payloads,
normalize_timespan_threshold_hours=float(normalize_timespan_threshold_hours),
ip_address_cache=IP_ADDRESS_CACHE,
seen_aggregate_report_ids=SEEN_AGGREGATE_REPORT_IDS,
reverse_dns_map=REVERSE_DNS_MAP,
)
def _exc_origin(error: BaseException) -> str:
"""Returns a ``" (raised at <file>:<line>)"`` suffix pointing at where an
unexpected exception actually originated, but only when the parsedmarc
logger is at ``DEBUG`` level. Returns ``""`` otherwise, so normal output is
unchanged.
The deepest traceback frame is the line that raised, which is the useful
breadcrumb when a catch-all ``except Exception`` turns an unforeseen error
into a generic ``Invalid*Report`` -- without it, the message says *what*
failed but not *where*.
"""
if not logger.isEnabledFor(logging.DEBUG):
return ""
frames = traceback.extract_tb(error.__traceback__)
if not frames:
return ""
last = frames[-1]
return f" (raised at {last.filename}:{last.lineno})"
def _text(value: Any) -> str | None:
"""Unwrap a value parsed by xmltodict that may be a dict.
RFC 9990 changed several aggregate-report elements (extra_contact_info,
error, comment, human_result) to type ``langAttrString``, a string with
an optional ``lang`` attribute. xmltodict parses the report without
attributes (``xml_attribs=False``), so such an element arrives as a
string; a dict now arises only from child elements, and gives its
``#text``. Returns ``None`` for unset values and leaves other scalar
shapes untouched.
"""
if value is None:
return None
if isinstance(value, dict):
text = value.get("#text")
return None if text is None else str(text)
return value
def _normalize_result_word(value: Any) -> Any:
"""Lowercase a reporter-supplied enum word; RFC 7489 Appendix C and
RFC 9990 define result/disposition types as lowercase tokens. Its
callers pass strings or ``None`` (``_scalar()`` has already run), and a
non-string passes through unchanged.
"""
return value.lower() if isinstance(value, str) else value
def _bucket_interval_by_day(
begin: datetime,
end: datetime,
total_count: int,
) -> list[dict[str, Any]]:
"""
Split the interval [begin, end) into daily buckets and distribute
`total_count` proportionally across those buckets.
The function:
1. Identifies each calendar day touched by [begin, end)
2. Computes how many seconds of the interval fall into each day
3. Assigns counts in proportion to those overlaps
4. Ensures the final counts sum exactly to total_count
Args:
begin: timezone-aware datetime, inclusive start of interval
end: timezone-aware datetime, exclusive end of interval
total_count: number of messages to distribute
Returns:
A list of dicts like:
{
"begin": datetime,
"end": datetime,
"count": int
}
"""
# --- Input validation ----------------------------------------------------
if begin > end:
raise ValueError("begin must be earlier than end")
if begin.tzinfo is None or end.tzinfo is None:
raise ValueError("begin and end must be timezone-aware")
if begin.tzinfo is not end.tzinfo:
raise ValueError("begin and end must have the same tzinfo")
if total_count < 0:
raise ValueError("total_count must be non-negative")
# --- Short-circuit trivial cases -----------------------------------------
interval_seconds = (end - begin).total_seconds()
if interval_seconds <= 0 or total_count == 0:
return []
tz: tzinfo = begin.tzinfo
# --- Step 1: Determine all calendar days touched by [begin, end) ----------
#
# For example:
# begin = Jan 1 12:00
# end = Jan 3 06:00
#
# We need buckets for:
# Jan 1 12:00 → Jan 2 00:00
# Jan 2 00:00 → Jan 3 00:00
# Jan 3 00:00 → Jan 3 06:00
#
# Start at midnight on the day of `begin`.
day_cursor = datetime(begin.year, begin.month, begin.day, tzinfo=tz)
# If `begin` is earlier on that day (e.g. 10:00), we want that midnight.
# If `begin` is past that midnight (e.g. 00:30), this is correct.
# If `begin` is BEFORE that midnight (rare unless tz shifts), adjust:
if day_cursor > begin:
day_cursor -= timedelta(days=1)
day_buckets: list[dict[str, Any]] = []
while day_cursor < end:
day_start = day_cursor
day_end = day_cursor + timedelta(days=1)
# Overlap between [begin, end) and this day
overlap_start = max(begin, day_start)
overlap_end = min(end, day_end)
overlap_seconds = (overlap_end - overlap_start).total_seconds()
if overlap_seconds > 0:
day_buckets.append(
{
"begin": overlap_start,
"end": overlap_end,
"seconds": overlap_seconds,
}
)
day_cursor = day_end
# --- Step 2: Pro-rate counts across buckets -------------------------------
#
# Compute the exact fractional count for each bucket:
# bucket_fraction = bucket_seconds / interval_seconds
# bucket_exact = total_count * bucket_fraction
#
# Then apply a "largest remainder" rounding strategy to ensure the sum
# equals exactly total_count.
exact_values: list[float] = [
(b["seconds"] / interval_seconds) * total_count for b in day_buckets
]
floor_values: list[int] = [int(x) for x in exact_values]
fractional_parts: list[float] = [x - int(x) for x in exact_values]
# How many counts do we still need to distribute after flooring?
remainder = total_count - sum(floor_values)
# Sort buckets by descending fractional remainder
indices_by_fraction = sorted(
range(len(day_buckets)),
key=lambda i: fractional_parts[i],
reverse=True,
)
# Start with floor values
final_counts = floor_values[:]
# Add +1 to the buckets with the largest fractional parts
for idx in indices_by_fraction[:remainder]:
final_counts[idx] += 1
# --- Step 3: Build the final per-day result list -------------------------
results: list[dict[str, Any]] = []
for bucket, count in zip(day_buckets, final_counts):
if count > 0:
results.append(
{
"begin": bucket["begin"],
"end": bucket["end"],
"count": count,
}
)
return results
def _append_parsed_record(
parsed_record: dict[str, Any],
records: list[dict[str, Any]],
begin_dt: datetime,
end_dt: datetime,
normalize: bool,
) -> None:
"""
Append a parsed DMARC record either unchanged or normalized.
Args:
parsed_record: The record returned by _parse_report_record().
records: Accumulating list of output records.
begin_dt: Report-level begin datetime (UTC).
end_dt: Report-level end datetime (UTC).
normalize: Whether this report exceeded the allowed timespan
and should be normalized per-day.
"""
if not normalize:
parsed_record["normalized_timespan"] = False
parsed_record["interval_begin"] = begin_dt.strftime("%Y-%m-%d %H:%M:%S")
parsed_record["interval_end"] = end_dt.strftime("%Y-%m-%d %H:%M:%S")
records.append(parsed_record)
return
# Normalization path: break record into daily buckets
total_count = int(parsed_record.get("count", 0))
buckets = _bucket_interval_by_day(begin_dt, end_dt, total_count)
if not buckets:
return
for part_index, bucket in enumerate(buckets):
new_rec = parsed_record.copy()
new_rec["count"] = bucket["count"]
new_rec["normalized_timespan"] = True
new_rec["interval_begin"] = bucket["begin"].strftime("%Y-%m-%d %H:%M:%S")
new_rec["interval_end"] = bucket["end"].strftime("%Y-%m-%d %H:%M:%S")
records.append(new_rec)
def _scalar(value: Any) -> str | None:
"""Return the string an element the schema types as a string should hold.
A string is returned as it is. A dict (an element with child elements,
possibly beside text) gives its ``#text`` or ``None``; a list (a
repeated element) or any other value gives ``None``. Without this a
child element reached Elasticsearch and OpenSearch as an object and
mapped the field as one (GHSA-qpcg-29vr-jwm3).
"""
if value is None or isinstance(value, str):
return value
if isinstance(value, dict):
text = value.get("#text")
return text if isinstance(text, str) else None
return None
def _scalar_fields(mapping: Any, keys: tuple[str, ...]) -> None:
"""Replace each of ``keys`` present in ``mapping`` with its
:func:`_scalar` value, in place; a ``mapping`` that is not a dict is
left to the caller's own error handling."""
if not isinstance(mapping, dict):
return
for key in keys:
if key in mapping:
mapping[key] = _scalar(mapping[key])
_REPORT_METADATA_SCALARS = (
"org_name",
"email",
"extra_contact_info",
"report_id",
"generator",
)
_POLICY_PUBLISHED_SCALARS = (
"domain",
"adkim",
"aspf",
"p",
"sp",
"pct",
"fo",
"np",
"testing",
"discovery_method",
)
def _scalarize_report(report: dict[str, Any]) -> None:
"""Make the report-level fields the schema types as strings strings or
``None``, in place: ``version``, the ``report_metadata`` strings and
every ``policy_published`` field of the RFC 7489 and RFC 9990 schemas
(other children are not copied to the output). Records are handled by
:func:`_parse_report_record`. Anything that is not a dict is left to
the caller's own error handling."""
if not isinstance(report, dict):
return
_scalar_fields(report, ("version",))
_scalar_fields(report.get("report_metadata"), _REPORT_METADATA_SCALARS)
policies = report.get("policy_published")
for policy in policies if isinstance(policies, list) else [policies]:
_scalar_fields(policy, _POLICY_PUBLISHED_SCALARS)
def _strip_zone_id(value: str, field_name: str) -> str:
"""Return an IP address without its IPv6 zone ID, if it has one.
A zone ID (``fe80::1%eth0``) is local to the reporting host, and the
PostgreSQL ``inet`` type cannot hold one (GHSA-fxr7-7h9q-v949). ``%``
appears in no other form of an IP address, so everything from the
first ``%`` is removed (``fe80::1%25eth0`` and ``fe80::1%`` also give
``fe80::1``), and a warning naming the field is logged. The remaining
text is not checked here; the caller's own handling of an invalid IP
address applies to it.
Args:
value: The address, as the report gives it
field_name: The report field that gave it, for the warning
Returns:
str: ``value`` up to its first ``%``
"""
address, sep, _zone = value.partition("%")
if sep:
logger.warning(
f"Removed the IPv6 zone ID from {field_name}: {sanitize_for_log(value)}"
)
return address
@dataclass
class _ReverseDNSBudget:
"""Per-report state that bounds the time an aggregate report spends in
reverse DNS lookups (GHSA-rwfh-ww9c-jm9m).
Attributes:
seconds_left: Seconds of failed or slow lookups the report may still
spend (``MAX_FAILED_REVERSE_DNS_SECONDS_PER_REPORT``)
total_seconds_left: Seconds of lookups of any kind the report may
still spend (``MAX_REVERSE_DNS_SECONDS_PER_REPORT``)
memo: Source information already produced in this report, by
lowercase IP address, so each distinct IP is looked up at most
once per report
exhausted: Whether either budget is spent; the report's remaining
IPs are then enriched from the local IP database and reverse
DNS map only
"""
seconds_left: float = field(
default_factory=lambda: MAX_FAILED_REVERSE_DNS_SECONDS_PER_REPORT
)
total_seconds_left: float = field(
default_factory=lambda: MAX_REVERSE_DNS_SECONDS_PER_REPORT
)
memo: dict[str, IPAddressInfo] = field(default_factory=dict)
exhausted: bool = False
_REVERSE_DNS_BUDGET_FALLBACK = (
"; the remaining records are enriched from the local IP database and "
"reverse DNS map only"
)
def _enrich_record_source(
source_ip: str, config: ParserConfig, budget: _ReverseDNSBudget
) -> IPAddressInfo:
"""Return the source information for one record's ``source_ip``.
A record's source IP is chosen by the report's sender, and a reverse
zone that never answers costs the whole DNS lifetime (8 s with the
default nameservers), so the time of every online lookup is charged to
``budget``. The time of a lookup that returns no PTR, or that takes at
least ``SLOW_REVERSE_DNS_SECONDS`` even though it found one (the sender
may control the reverse zone and answer every query just before the
timeout), is charged to the failed-or-slow budget; the time of every
lookup, however fast, is charged to the total budget as well, because a
zone that answers just under the slow threshold is otherwise never
charged and the number of records is not bounded. A lookup answered
from the IP address cache is charged too, but takes microseconds. Once
either budget is spent, one warning names the limit reached, and the
remaining IPs are enriched from the local IP database and reverse DNS
map only; those results are not cached, because the IP may well have a
PTR. An IP already seen in this report is answered from ``budget.memo``
and not charged.
"""
ip = source_ip.lower()
if ip in budget.memo:
return budget.memo[ip]
common: dict[str, Any] = dict(
ip_db_path=config.ip_db_path,
always_use_local_files=config.always_use_local_files,
reverse_dns_map_path=config.reverse_dns_map_path,
reverse_dns_map_url=config.reverse_dns_map_url,
reverse_dns_map=config.reverse_dns_map,
nameservers=config.nameservers,
timeout=config.dns_timeout,
retries=config.dns_retries,
psl_overrides_path=config.psl_overrides_path,
psl_overrides_url=config.psl_overrides_url,
)
if budget.exhausted or config.offline:
result = get_ip_address_info(
ip,
offline=True,
cache=None if budget.exhausted else config.ip_address_cache,
**common,
)
else:
start = time.monotonic()
result = get_ip_address_info(
ip, offline=False, cache=config.ip_address_cache, **common
)
elapsed = time.monotonic() - start
budget.total_seconds_left -= elapsed
if result["reverse_dns"] is None or elapsed >= SLOW_REVERSE_DNS_SECONDS:
budget.seconds_left -= elapsed
if budget.seconds_left <= 0:
budget.exhausted = True
logger.warning(
"Reverse DNS lookups for this report failed or were slow for "
f"{MAX_FAILED_REVERSE_DNS_SECONDS_PER_REPORT:.0f} seconds"
+ _REVERSE_DNS_BUDGET_FALLBACK
)
elif budget.total_seconds_left <= 0:
budget.exhausted = True
logger.warning(
"Reverse DNS lookups for this report took more than "
f"{MAX_REVERSE_DNS_SECONDS_PER_REPORT:.0f} seconds"
+ _REVERSE_DNS_BUDGET_FALLBACK
)
budget.memo[ip] = result
return result
def _parse_report_record(
record: dict[str, Any],
*,
config: ParserConfig,
is_rfc_9990: bool = False,
dns_budget: _ReverseDNSBudget | None = None,
) -> dict[str, Any]:
"""
Converts a record from a DMARC aggregate report into a more consistent
format
Args:
record (dict): The record to convert
config (ParserConfig): Parsing and enrichment options, plus caches
is_rfc_9990 (bool): Whether the enclosing report was detected as
RFC 9990-shaped, for RFC 9990-aware validation warnings
dns_budget (_ReverseDNSBudget | None): The enclosing report's
reverse DNS budget; ``None`` gives the record a budget of its own
Returns:
dict: The converted record
Raises:
ValueError: If the source IP is empty or not an IP address (after
its IPv6 zone ID, if any, is removed), or the count is not an
integer between 0 and ``MAX_MESSAGE_COUNT``
"""
record = record.copy()
new_record: dict[str, Any] = {}
# Fields the schema types as strings become strings or None
# (GHSA-qpcg-29vr-jwm3); see _scalar().
_scalar_fields(record["row"], ("source_ip",))
if record["row"]["source_ip"] is None:
raise ValueError("Source IP address is empty")
source_ip = _strip_zone_id(record["row"]["source_ip"], "source_ip")
# A <count> with no text of its own (child elements only, or repeated)
# becomes "", so int() raises the ValueError of a non-numeric count.
count = int(_scalar(record["row"]["count"]) or "")
# A count no output's 32-bit integer column can hold is rejected here,
# not by the output (GHSA-fxr7-7h9q-v949), and before the reverse DNS
# lookup, which a record that is dropped anyway should not wait on.
if not 0 <= count <= MAX_MESSAGE_COUNT:
raise ValueError(f"count {count} is outside 0..{MAX_MESSAGE_COUNT}")
if dns_budget is None:
dns_budget = _ReverseDNSBudget()
new_record["source"] = _enrich_record_source(source_ip, config, dns_budget)
new_record["count"] = count
policy_evaluated = record["row"]["policy_evaluated"].copy()
_scalar_fields(policy_evaluated, ("disposition", "dkim", "spf"))
new_policy_evaluated: dict[str, Any] = {
"disposition": "none",
"dkim": "fail",
"spf": "fail",
"policy_override_reasons": [],
}
if "disposition" in policy_evaluated:
new_policy_evaluated["disposition"] = _normalize_result_word(
policy_evaluated["disposition"]
)
for key in ("dkim", "spf"):
if key in policy_evaluated:
new_policy_evaluated[key] = _normalize_result_word(policy_evaluated[key])
reasons = []
spf_aligned = (
policy_evaluated["spf"] is not None
and policy_evaluated["spf"].lower() == "pass"
)
dkim_aligned = (
policy_evaluated["dkim"] is not None
and policy_evaluated["dkim"].lower() == "pass"
)
dmarc_aligned = spf_aligned or dkim_aligned
new_record["alignment"] = dict()
new_record["alignment"]["spf"] = spf_aligned
new_record["alignment"]["dkim"] = dkim_aligned
new_record["alignment"]["dmarc"] = dmarc_aligned
if "reason" in policy_evaluated:
if type(policy_evaluated["reason"]) is list:
reasons = policy_evaluated["reason"]
else:
reasons = [policy_evaluated["reason"]]
normalized_reasons = []
for reason in reasons:
# `comment` and `type` are strings in both schemas, but a sender can
# give them child elements (a dict) or repeat them (a list), and the
# CSV writer joins them as strings (GHSA-4272-4r8g-6c5h). They follow
# the rule of every other string field: see _scalar().
reason_type = _scalar(reason.get("type"))
normalized_reasons.append(
{"type": reason_type, "comment": _scalar(reason.get("comment"))}
)
if is_rfc_9990 and reason_type in RFC_7489_REMOVED_POLICY_OVERRIDE_TYPES:
logger.warning(
"Policy override reason type %r was removed in RFC 9990; "
"expected one of %s",
reason_type,
sorted(RFC_9990_POLICY_OVERRIDE_TYPES),
)
elif (
is_rfc_9990
and reason_type is not None
and reason_type not in RFC_9990_POLICY_OVERRIDE_TYPES
):
logger.warning(
"Unknown policy override reason type %r per RFC 9990; "
"expected one of %s",
reason_type,
sorted(RFC_9990_POLICY_OVERRIDE_TYPES),
)
new_policy_evaluated["policy_override_reasons"] = normalized_reasons
new_record["policy_evaluated"] = new_policy_evaluated
raw_identifiers = (
record["identities"] if "identities" in record else record["identifiers"]
)
new_record["identifiers"] = {
key: _scalar(value)
for key, value in raw_identifiers.items()
if key in ("header_from", "envelope_from", "envelope_to")
}
new_record["auth_results"] = {"dkim": [], "spf": []}
if type(new_record["identifiers"]["header_from"]) is str:
lowered_from = new_record["identifiers"]["header_from"].lower()
else:
lowered_from = ""
new_record["identifiers"]["header_from"] = lowered_from
if isinstance(record["auth_results"], dict):
auth_results = record["auth_results"].copy()
if "spf" not in auth_results:
auth_results["spf"] = []
if "dkim" not in auth_results:
auth_results["dkim"] = []
else:
auth_results = new_record["auth_results"].copy()
if not isinstance(auth_results["dkim"], list):
auth_results["dkim"] = [auth_results["dkim"]]
for result in auth_results["dkim"]:
_scalar_fields(result, ("domain", "selector", "result", "human_result"))
if "domain" in result and result["domain"] is not None:
new_result: dict[str, Any] = {"domain": result["domain"]}
if "selector" in result and result["selector"] is not None:
new_result["selector"] = result["selector"]
else:
if is_rfc_9990:
logger.warning(
"DKIM auth result for %r is missing the 'selector' "
"element, which is REQUIRED by RFC 9990",
result["domain"],
)
new_result["selector"] = "none"
if "result" in result and result["result"] is not None:
new_result["result"] = _normalize_result_word(result["result"])
else:
new_result["result"] = "none"
new_result["human_result"] = _text(result.get("human_result"))
new_record["auth_results"]["dkim"].append(new_result)
if not isinstance(auth_results["spf"], list):
auth_results["spf"] = [auth_results["spf"]]
for result in auth_results["spf"]:
_scalar_fields(result, ("domain", "scope", "result", "human_result"))
if "domain" in result and result["domain"] is not None:
new_result: dict[str, Any] = {"domain": result["domain"]}
if "scope" in result and result["scope"] is not None:
new_result["scope"] = result["scope"]
else:
new_result["scope"] = "mfrom"
if "result" in result and result["result"] is not None:
new_result["result"] = _normalize_result_word(result["result"])
else:
new_result["result"] = "none"
new_result["human_result"] = _text(result.get("human_result"))
new_record["auth_results"]["spf"].append(new_result)
# Backfill envelope_from from the last SPF result's domain when the
# reporter omitted the identifier or sent it empty.
if new_record["identifiers"].get("envelope_from") is None:
envelope_from = None
if len(auth_results["spf"]) > 0:
spf_result = auth_results["spf"][-1]
if "domain" in spf_result:
envelope_from = spf_result["domain"]
if envelope_from is not None:
envelope_from = str(envelope_from).lower()
new_record["identifiers"]["envelope_from"] = envelope_from
envelope_to = None
if "envelope_to" in new_record["identifiers"]:
envelope_to = new_record["identifiers"]["envelope_to"]
del new_record["identifiers"]["envelope_to"]
new_record["identifiers"]["envelope_to"] = envelope_to
return new_record
def _non_blank(value: Any) -> Any:
"""Treat ``None`` or a blank/whitespace-only string as absent.
Some SMTP TLS (RFC 8460) reporters send an empty string for an optional
failure-details field (e.g. ``sending-mta-ip``/``receiving-ip`` on a
``sts-policy-fetch-error``) instead of omitting the key. An empty string
is not a valid IP address, so passing it through crashed the PostgreSQL
save (``INET`` columns) and would also have crashed the Elasticsearch
and OpenSearch saves: their ``Ip()`` fields' validation
(``elasticsearch.dsl``/``opensearchpy``'s ``Document.save()``) rejects
an empty string with a bare ``ValueError`` (issue #915). Non-string,
non-blank values pass through unchanged.
"""
if value is None:
return None
if isinstance(value, str) and value.strip() == "":
return None
return value
# RFC 8460 §4.4 gives every SMTP TLS report date as an RFC 3339 date-time,
# which is at most about 35 characters even with a long fractional second.
# dateutil's parser is super-linear on a long run of digits (12.4 s for
# 1,000,000 of them, Python 3.14), so a longer value is rejected unparsed.
_SMTP_TLS_MAX_DATETIME_LENGTH = 64
def _smtp_tls_string(value: Any, field: str) -> str:
"""Return ``value`` if it is a string, as RFC 8460 §4.4 requires of
``field``; raise ``InvalidSMTPTLSReport`` otherwise."""
if not isinstance(value, str):
raise InvalidSMTPTLSReport(
f"{field} must be a string, not {type(value).__name__}"
)
_smtp_tls_check_utf8(value, field)
return value
def _smtp_tls_check_utf8(value: str, field: str) -> None:
"""Raise ``InvalidSMTPTLSReport`` when ``value`` holds a lone surrogate
(JSON ``"\\ud800"``). RFC 8460 §4 reports are I-JSON, whose RFC 7493
§2.1 forbids surrogate code points, and the outputs cannot encode one."""
try:
value.encode("utf-8")
except UnicodeEncodeError as error:
raise InvalidSMTPTLSReport(f"{field} is not valid UTF-8 text") from error
def _smtp_tls_string_list(value: Any, field: str) -> list[str]:
"""Return ``value`` if it is a list of strings (RFC 8460 §4.4's "JSON
array of strings"); raise ``InvalidSMTPTLSReport`` otherwise."""
if not isinstance(value, list) or not all(isinstance(item, str) for item in value):
raise InvalidSMTPTLSReport(f"{field} must be a list of strings")
for item in value:
_smtp_tls_check_utf8(item, field)
return value
def _smtp_tls_mx_host_patterns(value: Any, field: str) -> list[str]:
"""Return a policy's MX host patterns as a list of strings.
RFC 8460 §4.4 describes the value as "a JSON array of strings", but its
Appendix B example gives one pattern as a plain string, so a string
becomes a one-element list. ``null`` and an empty string give an empty
list; any other type raises ``InvalidSMTPTLSReport``.
"""
if value is None or value == "":
return []
if isinstance(value, str):
return [_smtp_tls_string(value, field)]
return _smtp_tls_string_list(value, field)
def _smtp_tls_count(value: Any, field: str) -> int | None:
"""Return ``value`` if it is null or an integer from 0 to
``MAX_MESSAGE_COUNT``.
RFC 8460 §4.4 encodes each session count as "an integer, encoded as a
JSON number". ``null`` is tolerated as an unreported count. A JSON
``true``/``false`` is rejected even though ``bool`` subclasses ``int``.
A count outside ``0..MAX_MESSAGE_COUNT`` is rejected because the
PostgreSQL output stores counts as 32-bit integers
(GHSA-fxr7-7h9q-v949). Raises ``InvalidSMTPTLSReport`` otherwise.
"""
if value is None:
return value
if not isinstance(value, int) or isinstance(value, bool):
raise InvalidSMTPTLSReport(
f"{field} must be an integer, not {type(value).__name__}"
)
if not 0 <= value <= MAX_MESSAGE_COUNT:
raise InvalidSMTPTLSReport(f"{field} must be between 0 and {MAX_MESSAGE_COUNT}")
return value
def _smtp_tls_datetime(value: Any, field: str) -> str:
"""Return ``value`` if it is a date-time string every output can convert.
RFC 8460 §4.4 gives ``start-datetime`` and ``end-datetime`` as strings in
RFC 3339 format. The value is checked with the conversions the
Elasticsearch, OpenSearch and S3 outputs (a UTC ``datetime``) and the
Splunk output (a UNIX timestamp) make later. dateutil also accepts a
partial date such as ``"2016"``, which PostgreSQL may still reject; that
failure is raised as ``PostgreSQLRejectedReport``, which skips the
report in the PostgreSQL output only (an output error with
``fail_on_output_error``). A ``start-datetime`` more than
``MAX_REPORT_FUTURE_DAYS`` days in the future is also rejected, because
it names the dated index (#947); ``end-datetime`` is not checked. Raises
``InvalidSMTPTLSReport`` otherwise.
"""
value = _smtp_tls_string(value, field)
if len(value) > _SMTP_TLS_MAX_DATETIME_LENGTH:
raise InvalidSMTPTLSReport(
f"{field} is longer than {_SMTP_TLS_MAX_DATETIME_LENGTH} characters"
)
try:
moment = human_timestamp_to_datetime(value, to_utc=True)
human_timestamp_to_unix_timestamp(value)
except (ValueError, OverflowError) as e:
raise InvalidSMTPTLSReport(
f"{field} is not a valid date-time: {sanitize_for_log(value)}"
) from e
if field == "start-datetime":
try:
_check_not_future(moment, field)
except ValueError as error:
raise InvalidSMTPTLSReport(str(error)) from error
return value
def _parse_smtp_tls_failure_details(failure_details: dict[str, Any]):
try:
new_failure_details: dict[str, Any] = {
"result_type": _smtp_tls_string(
failure_details["result-type"], "result-type"
),
"failed_session_count": _smtp_tls_count(
failure_details["failed-session-count"], "failed-session-count"
),
}
optional_fields = [
("sending-mta-ip", "sending_mta_ip"),
("receiving-ip", "receiving_ip"),
("receiving-mx-hostname", "receiving_mx_hostname"),
("receiving-mx-helo", "receiving_mx_helo"),
("failure-reason-code", "failure_reason_code"),
]
for src_key, dest_key in optional_fields:
value = _non_blank(failure_details.get(src_key))
if value is not None:
new_failure_details[dest_key] = _smtp_tls_string(value, src_key)
# RFC 8460 §4.4 defines both as the IP address of the sending or
# receiving MTA, and both are optional. Drop a value that is not an
# IP address, which the PostgreSQL inet column and the
# Elasticsearch/OpenSearch ip mappings reject (GHSA-fxr7-7h9q-v949).
# An IPv6 zone ID (fe80::1%eth0) is local to the reporting host and
# the inet type cannot hold one, and ipaddress.ip_address() accepts
# it, so such a value is dropped as well. The aggregate and failure
# paths strip the zone ID instead (_strip_zone_id()), because the
# source IP is required there; these fields are optional, so
# dropping one loses nothing a report needs.
for src_key, dest_key in (
("sending-mta-ip", "sending_mta_ip"),
("receiving-ip", "receiving_ip"),
):
value = new_failure_details.get(dest_key)
if value is None:
continue
try:
if "%" in value:
raise ValueError("zone ID")
ipaddress.ip_address(value)
except ValueError:
logger.warning(
f"Ignoring {src_key} that is not an IP address: "
f"{sanitize_for_log(value)}"
)
del new_failure_details[dest_key]
# RFC 8460 §4.4 names the JSON key "additional-information" (its
# value is described as an "additional-info-uri"). Some reporters
# instead send the non-RFC key "additional-info-uri" directly; fall
# back to it for backward compatibility when the RFC key is absent
# or blank.
additional_info_uri = _non_blank(failure_details.get("additional-information"))
if additional_info_uri is None:
additional_info_uri = _non_blank(failure_details.get("additional-info-uri"))
if additional_info_uri is not None:
new_failure_details["additional_info_uri"] = _smtp_tls_string(
additional_info_uri, "additional-information"
)
return new_failure_details
except KeyError as e:
raise InvalidSMTPTLSReport(
f"Missing required failure details field: {e}"
) from e
except Exception as e:
raise InvalidSMTPTLSReport(str(e) + _exc_origin(e)) from e
def _parse_smtp_tls_report_policy(policy: dict[str, Any]):
policy_types = ["tlsa", "sts", "no-policy-found"]
try:
policy_domain = _smtp_tls_string(
policy["policy"]["policy-domain"], "policy-domain"
)
policy_type = _smtp_tls_string(policy["policy"]["policy-type"], "policy-type")
failure_details = []
if policy_type not in policy_types:
raise InvalidSMTPTLSReport(
f"Invalid policy type {sanitize_for_log(str(policy_type))}"
)
new_policy: dict[str, Any] = {
"policy_domain": policy_domain,
"policy_type": policy_type,
}
if policy["policy"].get("policy-string") is not None:
policy_strings = _smtp_tls_string_list(
policy["policy"]["policy-string"], "policy-string"
)
if len(policy_strings) > 0:
new_policy["policy_strings"] = policy_strings
# RFC 8460 §4.4 keys the policy's MX host patterns "mx-host";
# "mx-host-pattern" names its value type, not a key, but was read as
# the key before and is kept as a fallback (issue #942).
mx_host_patterns = _smtp_tls_mx_host_patterns(
policy["policy"].get("mx-host"), "mx-host"
)
if not mx_host_patterns:
mx_host_patterns = _smtp_tls_mx_host_patterns(
policy["policy"].get("mx-host-pattern"), "mx-host-pattern"
)
if mx_host_patterns:
new_policy["mx_host_patterns"] = mx_host_patterns
new_policy["successful_session_count"] = _smtp_tls_count(
policy["summary"]["total-successful-session-count"],
"total-successful-session-count",
)
new_policy["failed_session_count"] = _smtp_tls_count(
policy["summary"]["total-failure-session-count"],
"total-failure-session-count",
)
if "failure-details" in policy:
if not isinstance(policy["failure-details"], list) or not all(
isinstance(details, dict) for details in policy["failure-details"]
):
raise InvalidSMTPTLSReport("failure-details must be a list of objects")
for details in policy["failure-details"]:
failure_details.append(_parse_smtp_tls_failure_details(details))
new_policy["failure_details"] = failure_details
return new_policy
except KeyError as e:
raise InvalidSMTPTLSReport(f"Missing required policy field: {e}") from e
except Exception as e:
raise InvalidSMTPTLSReport(str(e) + _exc_origin(e)) from e
def parse_smtp_tls_report_json(report: str | bytes) -> SMTPTLSReport:
"""Parses and validates an SMTP TLS report"""
required_fields = [
"organization-name",
"date-range",
"contact-info",
"report-id",
"policies",
]
if _exceeds_size_cap(report):
# json.loads() holds the whole document at several times its size
# (GHSA-q7gm-8cj9-94hr).
raise InvalidSMTPTLSReport(_size_cap_text(report))
try:
if isinstance(report, bytes):
report = report.decode("utf-8", errors="replace")
policies = []
report_dict = json.loads(report)
for required_field in required_fields:
if required_field not in report_dict:
raise Exception(f"Missing required field: {required_field}]")
if not isinstance(report_dict["policies"], list):
policies_type = type(report_dict["policies"])
raise InvalidSMTPTLSReport(f"policies must be a list, not {policies_type}")
for policy in report_dict["policies"]:
policies.append(_parse_smtp_tls_report_policy(policy))
# RFC 8460 §4.4 gives contact-info as one string; some reporters
# send a list of them, which is kept as it is.
contact_info = report_dict["contact-info"]
if isinstance(contact_info, str):
contact_info = _smtp_tls_string(contact_info, "contact-info")
else:
contact_info = _smtp_tls_string_list(contact_info, "contact-info")
new_report: SMTPTLSReport = {
"organization_name": _smtp_tls_string(
report_dict["organization-name"], "organization-name"
),
"begin_date": _smtp_tls_datetime(
report_dict["date-range"]["start-datetime"], "start-datetime"
),
"end_date": _smtp_tls_datetime(
report_dict["date-range"]["end-datetime"], "end-datetime"
),
"contact_info": contact_info,
"report_id": _smtp_tls_string(report_dict["report-id"], "report-id"),
"policies": policies,
}
return new_report
except KeyError as e:
raise InvalidSMTPTLSReport(f"Missing required field: {e}") from e
except Exception as e:
raise InvalidSMTPTLSReport(str(e) + _exc_origin(e)) from e
# Characters that make a spreadsheet application read a cell as a formula.
_CSV_FORMULA_PREFIXES = ("=", "+", "-", "@", "\t", "\r")
def _csv_safe(value: Any) -> Any:
"""Makes a string CSV field safe to write and to open in a spreadsheet.
Two things happen to a string, in this order:
1. NUL characters are stripped. Python 3.10's ``csv`` writer raised
``_csv.Error: need to escape, but no escapechar set`` on any field
containing NUL (CPython issue 97503, fixed in 3.11; the floor is now
3.11), and NUL is still dropped to match
``get_filename_safe_string``.
2. A string starting with ``=``, ``+``, ``-``, ``@``, tab, or carriage
return gets a leading ``'``. Spreadsheet applications evaluate such
a cell as a formula, and report text fields come from untrusted
mail, so a reporter could otherwise plant a formula like
``=HYPERLINK(...)`` in the output (CWE-1236). The apostrophe is the
OWASP CSV Injection recommendation, and makes the application
treat the cell as text:
https://owasp.org/www-community/attacks/CSV_Injection
Non-string values (ASN integers, counts) are returned unchanged.
"""
if isinstance(value, str):
value = value.replace("\x00", "")
if value.startswith(_CSV_FORMULA_PREFIXES):
return "'" + value
return value
return value
def parsed_smtp_tls_reports_to_csv_rows(
reports: SMTPTLSReport | list[SMTPTLSReport],
) -> list[dict[str, Any]]:
"""Converts one or more parsed SMTP TLS reports into a list of
single-layer dict objects suitable for use in a CSV"""
if isinstance(reports, dict):
reports = [reports]
rows = []
for report in reports:
common_fields: dict[str, Any] = {
"organization_name": report["organization_name"],
"begin_date": report["begin_date"],
"end_date": report["end_date"],
"report_id": report["report_id"],
}
record: dict[str, Any] = common_fields.copy()
for policy in report["policies"]:
if "policy_strings" in policy:
record["policy_strings"] = "|".join(policy["policy_strings"])
if "mx_host_patterns" in policy:
record["mx_host_patterns"] = "|".join(policy["mx_host_patterns"])
successful_record = record.copy()
successful_record["policy_domain"] = policy["policy_domain"]
successful_record["policy_type"] = policy["policy_type"]
successful_record["successful_session_count"] = policy[
"successful_session_count"
]
successful_record["failed_session_count"] = policy["failed_session_count"]
rows.append(successful_record)
if "failure_details" in policy:
for failure_details in policy["failure_details"]:
failure_record = record.copy()
for key in failure_details.keys():
failure_record[key] = failure_details[key]
rows.append(failure_record)
return rows
def parsed_smtp_tls_reports_to_csv(
reports: SMTPTLSReport | list[SMTPTLSReport],
) -> str:
"""
Converts one or more parsed SMTP TLS reports to flat CSV format, including
headers
Args:
reports: A parsed SMTP TLS report or list of parsed SMTP TLS reports
Returns:
str: Parsed SMTP TLS report data in flat CSV format, including headers
"""
fields = [
"organization_name",
"begin_date",
"end_date",
"report_id",
"result_type",
"successful_session_count",
"failed_session_count",
"policy_domain",
"policy_type",
"policy_strings",
"mx_host_patterns",
"sending_mta_ip",
"receiving_ip",
"receiving_mx_hostname",
"receiving_mx_helo",
"additional_info_uri",
"failure_reason_code",
]
csv_file_object = StringIO(newline="\n")
writer = DictWriter(csv_file_object, fields)
writer.writeheader()
rows = parsed_smtp_tls_reports_to_csv_rows(reports)
for row in rows:
writer.writerow({key: _csv_safe(value) for key, value in row.items()})
csv_file_object.flush()
return csv_file_object.getvalue()
# No field of an aggregate report carries an attribute parsedmarc keeps, and
# an attribute xmltodict kept (<p a="1">none</p> as {"@a": "1", "#text":
# "none"}) would reach Elasticsearch or OpenSearch as an object and map the
# field as one (GHSA-qpcg-29vr-jwm3). With xml_attribs=False xmltodict builds
# no attribute keys, while expat still hands the attributes to the start
# handler, so _LimitedExpatParser counts them.
def _parse_limited_xml(xml: str) -> Any:
"""Parse ``xml`` with xmltodict through the limited expat parser, without
attributes, and raise ``InvalidAggregateReport`` when it goes over
``MAX_XML_DEPTH`` or ``MAX_XML_NODES``."""
try:
return xmltodict.parse(xml, expat=_NoDoctypeExpat, xml_attribs=False)
except _XMLLimitExceeded as error:
raise InvalidAggregateReport(f"Invalid XML: {error}") from error
def _recover_xml(xml: str) -> str:
"""Return ``xml`` repaired by lxml's recovery parser, for a document
expat rejected: ``""`` when nothing is recoverable, ``"<a/>"`` when the
recovery itself fails. Raises ``InvalidAggregateReport`` for a document
over the node limit or holding a DOCTYPE."""
has_doctype = False
try:
tree = _parse_xml_with_recovery(xml.encode("utf-8"))
root = tree.getroot()
if root is None:
# Nothing recoverable; the second parse reports it as
# invalid XML.
return ""
# The first parse can fail before expat reaches a DOCTYPE (e.g. on
# a malformed XML declaration, which is only rewritten later), so
# the recovered tree gets the same DOCTYPE check, made before
# serializing. The rejection is raised after this try, which would
# otherwise swallow it.
has_doctype = bool(tree.docinfo.doctype) or (
tree.docinfo.internalDTD is not None
)
if not has_doctype:
# Serializing the root element rather than the tree leaves any
# internal DTD subset out of the XML handed to the second parse.
# str() is a no-op at runtime; lxml ships no type stubs, so
# pyright would widen the result to str | bytes.
return str(etree.tostring(root, encoding="unicode"))
except _XMLLimitExceeded as limit:
raise InvalidAggregateReport(f"Invalid XML: {limit}") from limit
except Exception:
return "<a/>"
raise InvalidAggregateReport(_DOCTYPE_ERROR)
def _size_cap_text(content: str | bytes) -> str:
"""Return the reason a report over the size cap is rejected: bytes are
measured as they are, text as UTF-8."""
size = f"{MAX_DECOMPRESSED_REPORT_SIZE // 2**20} MiB"
if isinstance(content, bytes):
return f"the report is larger than {size}"
return f"the report is larger than {size} of UTF-8"
def _exceeds_size_cap(content: str | bytes) -> bool:
"""Return whether ``content`` is more than ``MAX_DECOMPRESSED_REPORT_SIZE``
bytes, a ``str`` measured as UTF-8, as the cap on a compressed report
measures its decompressed bytes. ASCII text and text too long or too
short to matter are decided by length; other text is encoded 1 MiB of
characters at a time, so the measure needs no full copy."""
cap = MAX_DECOMPRESSED_REPORT_SIZE
if isinstance(content, bytes) or len(content) > cap:
return len(content) > cap
# A character is at most 4 bytes of UTF-8.
if len(content) <= cap // 4 or content.isascii():
return False
size = 0
for offset in range(0, len(content), 2**20):
size += len(content[offset : offset + 2**20].encode("utf-8", "surrogatepass"))
if size > cap:
return True
return False
def _report_text(xml: str | bytes) -> str:
"""Return the report as text, after :func:`_check_xml_text`; bytes over
the size cap are rejected before they are decoded (errors ignored)."""
if isinstance(xml, bytes):
if _exceeds_size_cap(xml):
raise InvalidAggregateReport(f"Invalid XML: {_size_cap_text(xml)}")
xml = xml.decode(errors="ignore")
_check_xml_text(xml)
return xml
def _check_xml_text(xml: str) -> None:
"""Reject, before any parser reads it, an aggregate report whose text
alone would make parsing it too costly (GHSA-q7gm-8cj9-94hr): more than
``MAX_DECOMPRESSED_REPORT_SIZE`` bytes of UTF-8 (an uncompressed
attachment or file is held to the cap a compressed one is), more
attributes than ``MAX_XML_NODES``, or a tag longer than
``MAX_XML_TAG_LENGTH``. Raises ``InvalidAggregateReport``."""
if _exceeds_size_cap(xml):
raise InvalidAggregateReport(f"Invalid XML: {_size_cap_text(xml)}")
# The = count is one C pass, so it runs before the slower tag scan.
_check_attribute_count(xml)
_check_tag_lengths(xml)
def _check_tag_lengths(xml: str) -> None:
"""Raise ``InvalidAggregateReport`` when any text between a ``<`` and
the next ``>`` is longer than ``MAX_XML_TAG_LENGTH``, before a parser
builds the tag's attribute list (GHSA-q7gm-8cj9-94hr).
One linear pass without a regex; the text after the last ``>`` is
checked too, so an unterminated tag is caught. A comment or CDATA
section is treated as a tag, which is harmless at this length. XML
allows a raw ``>`` inside a quoted attribute value, which ends a piece
early, so this scan does not bound how many attributes a tag holds;
:func:`_check_attribute_count` does.
"""
# The text is split on ">" one chunk at a time: one split of the whole
# text made a string per tag, +1,318 MiB for 100 MiB of "<a/>". A
# chunk's pieces are measured with max() before any is examined, and
# the unfinished last piece is carried into the next chunk from its
# last "<", so memory stays near one chunk.
carry = ""
for offset in range(0, len(xml), _TAG_SCAN_CHUNK):
pieces = (carry + xml[offset : offset + _TAG_SCAN_CHUNK]).split(">")
carry = pieces.pop()
if max(map(len, pieces), default=0) > MAX_XML_TAG_LENGTH:
for piece in pieces:
_check_tag_piece(piece)
carry = _check_tag_piece(carry)
def _check_attribute_count(xml: str) -> None:
"""Raise ``InvalidAggregateReport`` when ``xml`` holds more ``=``
characters than ``MAX_XML_NODES``: every attribute needs one, and
expat and libxml2 build a tag's attribute list before the node counter
sees it (GHSA-q7gm-8cj9-94hr). ``str.count`` is one linear pass."""
if xml.count("=") > MAX_XML_NODES:
raise InvalidAggregateReport(
f"Invalid XML: more than {MAX_XML_NODES:,} `=` characters, so more "
"attributes than the node limit allows"
)
def _check_tag_piece(piece: str) -> str:
"""Raise the tag-length error when the text of ``piece`` from its last
``<`` is longer than ``MAX_XML_TAG_LENGTH``; return that text, or ``""``
when ``piece`` holds no ``<``."""
start = piece.rfind("<")
if start < 0:
return ""
if len(piece) - start > MAX_XML_TAG_LENGTH:
raise InvalidAggregateReport(
f"Invalid XML: a tag is longer than {MAX_XML_TAG_LENGTH:,} characters"
)
return piece[start:]
_TAG_SCAN_CHUNK = 1 << 16
def _check_report_span(span_seconds: int) -> None:
"""Raise ``InvalidAggregateReport`` when a report's date range is negative
or longer than ``MAX_REPORT_SPAN_DAYS``, before normalization splits it
into one record per day (GHSA-p85v-578x-3wcf). The message holds no
number from the report, whose dates can be thousands of digits long."""
if span_seconds < 0:
raise InvalidAggregateReport("Report date range is negative")
if span_seconds > MAX_REPORT_SPAN_DAYS * 86400:
raise InvalidAggregateReport(
f"Report date range spans more than {MAX_REPORT_SPAN_DAYS} days"
)
def _check_not_future(moment: datetime, what: str) -> None:
"""Raise ``ValueError`` when ``moment`` is more than
``MAX_REPORT_FUTURE_DAYS`` days ahead of the current time, because the
date that names a report's dated index would then create an index for a
date far in the future (#947). ``moment`` must be an aware UTC datetime;
each parser converts the error to its own ``Invalid*Report``."""
if moment > datetime.now(timezone.utc) + timedelta(days=MAX_REPORT_FUTURE_DAYS):
raise ValueError(
f"{what} {moment.isoformat()} is more than "
f"{MAX_REPORT_FUTURE_DAYS} days in the future"
)
def _days_touched(begin: datetime, end: datetime) -> int:
"""Return how many calendar days of ``begin``'s timezone the interval
[begin, end) touches: the number of buckets ``_bucket_interval_by_day()``
makes, before it drops the buckets that receive no messages."""
day_start = datetime(begin.year, begin.month, begin.day, tzinfo=begin.tzinfo)
return max(0, -(-int((end - day_start).total_seconds()) // 86400))
def _normalize_within_limit(
normalize: bool,
xml_records: Any,
begin_dt: datetime,
end_dt: datetime,
report_id: str,
) -> bool:
"""Return whether per-day normalization stays enabled for a report.
Normalization makes one record per record and day, so a report whose
records times days exceed ``MAX_NORMALIZED_RECORDS`` keeps its records
whole, over the report's own range, and one warning is logged
(GHSA-p85v-578x-3wcf).
"""
if not normalize:
return False
record_count = len(xml_records) if isinstance(xml_records, list) else 1
days = _days_touched(begin_dt, end_dt)
if record_count * days <= MAX_NORMALIZED_RECORDS:
return True
logger.warning(
f"Report {sanitize_for_log(str(report_id))} is not normalized to daily "
f"records: {record_count} records over {days} days would make "
f"{record_count * days} records, more than {MAX_NORMALIZED_RECORDS}"
)
return False
def _parse_report_records(
xml_records: Any,
*,
config: ParserConfig,
is_rfc_9990: bool,
begin_dt: datetime,
end_dt: datetime,
normalize: bool,
keep_alive: Callable | None,
dns_budget: _ReverseDNSBudget | None = None,
) -> list[dict[str, Any]]:
"""Parse an aggregate report's ``<record>`` elements (a list, or one
record) into output records, split per day when ``normalize`` is set.
A record of a list that fails to parse is skipped with a warning.
``dns_budget`` is shared by every record (``None`` creates one); see
:class:`_ReverseDNSBudget`."""
if dns_budget is None:
dns_budget = _ReverseDNSBudget()
records: list[dict[str, Any]] = []
if type(xml_records) is list:
for i in range(len(xml_records)):
if keep_alive is not None and i > 0 and i % 20 == 0:
logger.debug("Sending keepalive cmd")
keep_alive()
logger.debug(f"Processed {i}/{len(xml_records)}")
try:
report_record = _parse_report_record(
xml_records[i],
config=config,
is_rfc_9990=is_rfc_9990,
dns_budget=dns_budget,
)
_append_parsed_record(
parsed_record=report_record,
records=records,
begin_dt=begin_dt,
end_dt=end_dt,
normalize=normalize,
)
except Exception as e:
logger.warning(f"Could not parse record: {sanitize_for_log(str(e))}")
else:
report_record = _parse_report_record(
xml_records,
config=config,
is_rfc_9990=is_rfc_9990,
dns_budget=dns_budget,
)
_append_parsed_record(
parsed_record=report_record,
records=records,
begin_dt=begin_dt,
end_dt=end_dt,
normalize=normalize,
)
return records
def parse_aggregate_report_xml(
xml: str | bytes,
*,
ip_db_path: str | None = None,
always_use_local_files: bool = False,
reverse_dns_map_path: str | None = None,
reverse_dns_map_url: str | None = None,
offline: bool = False,
nameservers: list[str] | None = None,
timeout: float = DEFAULT_DNS_TIMEOUT,
retries: int = DEFAULT_DNS_MAX_RETRIES,
keep_alive: Callable | None = None,
normalize_timespan_threshold_hours: float = 24.0,
config: ParserConfig | None = None,
) -> AggregateReport:
"""Parses a DMARC XML report string and returns a consistent dict
Args:
xml (str | bytes): DMARC aggregate report XML (bytes are decoded
with errors ignored)
ip_db_path (str): Path to a MMDB file from IPinfo, MaxMind, or DBIP
always_use_local_files (bool): Do not download files
reverse_dns_map_path (str): Path to a reverse DNS map file
reverse_dns_map_url (str): URL to a reverse DNS map file
offline (bool): Do not query online for geolocation or DNS
nameservers (list): A list of one or more nameservers to use
(Cloudflare's public DNS resolvers by default)
timeout (float): Sets the DNS timeout in seconds
retries (int): Number of times to retry DNS queries on timeout or
other transient errors
keep_alive (callable): Keep alive function. Not part of ``config``;
always applies.
normalize_timespan_threshold_hours (float): Normalize timespans beyond this
config (ParserConfig): a single object carrying all parsing and
enrichment options plus the caches; when provided, the
individual option keyword arguments listed above are ignored in
favor of the config's values.
Returns:
dict: The parsed aggregate DMARC report
"""
cfg = _resolve_config(
config,
offline=offline,
ip_db_path=ip_db_path,
always_use_local_files=always_use_local_files,
reverse_dns_map_path=reverse_dns_map_path,
reverse_dns_map_url=reverse_dns_map_url,
nameservers=nameservers,
dns_timeout=timeout,
dns_retries=retries,
normalize_timespan_threshold_hours=normalize_timespan_threshold_hours,
)
errors = []
# Parse XML and recover from errors
xml = _report_text(xml)
# Detect the XML namespace before any rewriting strips it. The dmarc-2.0
# namespace is one of the indicators for an RFC 9990 report but it is
# NOT a reliable sole discriminator: the <version> element value is
# ambiguous (RFC 9990's appendix sample uses <version>1.0</version>
# inside the dmarc-2.0 namespace), and real-world reporters frequently
# emit RFC 9990-shaped reports without declaring the namespace at all.
# The final `is_rfc_9990` decision is made post-parse so that
# RFC 9990-only fields (np, testing, discovery_method, generator,
# human_result) can also vote it in.
xml_namespace: str | None = None
namespace_match = xml_namespace_regex.search(xml)
if namespace_match:
xml_namespace = namespace_match.group(1)
first_error: str | None = None
try:
xmltodict.parse(xml, expat=_NoDoctypeExpat, xml_attribs=False)["feedback"]
except _XMLDoctypeFound as e:
raise InvalidAggregateReport(_DOCTYPE_ERROR) from e
except _XMLLimitExceeded as e:
# Rejected outright: the lxml recovery below would be a second pass
# over the same oversized document.
raise InvalidAggregateReport(f"Invalid XML: {e}") from e
except Exception as e:
first_error = f"Invalid XML: {e.__str__()}"
# The recovery runs after the except block, not inside it: while a
# handler runs, the handled exception's traceback keeps the failed
# parse's frames, and with them its partial tree, alive.
if first_error is not None:
errors.append(first_error)
xml = _recover_xml(xml)
try:
# Replace XML header (sometimes they are invalid)
xml = xml_header_regex.sub('<?xml version="1.0"?>', xml)
# Remove invalid schema tags
xml = xml_schema_regex.sub("", xml)
report = _parse_limited_xml(xml)["feedback"]
_scalarize_report(report)
report_metadata = report["report_metadata"]
# The report_metadata strings are strings or None here:
# xmltodict kept no xml:lang attribute (xml_attribs=False) and
# _scalarize_report() any child element. Text in place of the
# section's elements is a missing section.
if not isinstance(report_metadata, dict):
raise AttributeError("report_metadata holds no elements")
schema = "draft"
if "version" in report:
schema = report["version"]
new_report: dict[str, Any] = {
"xml_schema": schema,
"xml_namespace": xml_namespace,
}
new_report_metadata: dict[str, Any] = {}
if report_metadata["org_name"] is None:
if report_metadata["email"] is not None:
report_metadata["org_name"] = report_metadata["email"].split("@")[-1]
org_name = report_metadata["org_name"]
if org_name is not None and " " not in org_name:
new_org_name = get_base_domain(org_name)
if new_org_name is not None:
org_name = new_org_name
if not org_name:
logger.debug(f"Could not parse org_name from XML.\r\n{report.__str__()}")
raise KeyError(
"Organization name is missing. This field is a requirement "
"for saving the report"
)
new_report_metadata["org_name"] = org_name
new_report_metadata["org_email"] = report_metadata["email"]
# extra_contact_info is langAttrString in RFC 9990 (xs:string in
# RFC 7489); its lang attribute is dropped when the report is parsed.
extra = _text(report_metadata.get("extra_contact_info"))
new_report_metadata["org_extra_contact_info"] = extra
new_report_metadata["report_id"] = report_metadata["report_id"]
report_id = new_report_metadata["report_id"]
report_id = report_id.replace("<", "").replace(">", "").split("@")[0]
new_report_metadata["report_id"] = report_id
date_range = report["report_metadata"]["date_range"]
begin_ts = int(date_range["begin"].split(".")[0])
end_ts = int(date_range["end"].split(".")[0])
span_seconds = end_ts - begin_ts
_check_report_span(span_seconds)
# The end is checked as well as the begin: a normalized report's
# per-day records each name an index from their own interval start,
# and every interval start lies at or before the report's end.
try:
for stamp, what in (
(begin_ts, "Report begin date"),
(end_ts, "Report end date"),
):
_check_not_future(datetime.fromtimestamp(stamp, timezone.utc), what)
except ValueError as error:
raise InvalidAggregateReport(str(error)) from error
normalize_timespan = (
span_seconds > cfg.normalize_timespan_threshold_hours * 3600
)
date_range["begin"] = timestamp_to_human(begin_ts)
date_range["end"] = timestamp_to_human(end_ts)
new_report_metadata["begin_date"] = date_range["begin"]
new_report_metadata["end_date"] = date_range["end"]
new_report_metadata["timespan_requires_normalization"] = normalize_timespan
new_report_metadata["original_timespan_seconds"] = span_seconds
begin_dt = human_timestamp_to_datetime(
new_report_metadata["begin_date"], to_utc=True
)
end_dt = human_timestamp_to_datetime(
new_report_metadata["end_date"], to_utc=True
)
# <error> is langAttrString in RFC 9990 (xs:string in RFC 7489) and
# was cardinality-narrowed from "unbounded" to "1" in RFC 9990, but
# the parser still accepts a list for backward compatibility with
# RFC 7489 reports that carry multiple errors.
if "error" in report["report_metadata"]:
raw_errors = report["report_metadata"]["error"]
if not isinstance(raw_errors, list):
raw_errors = [raw_errors]
errors = [text for text in (_text(e) for e in raw_errors) if text]
new_report_metadata["errors"] = errors
# <generator> is a plain xs:string in RFC 9990; attributes and child
# elements are already removed, and _text() keeps it a string.
generator = _text(report_metadata.get("generator"))
new_report_metadata["generator"] = generator
new_report["report_metadata"] = new_report_metadata
policy_published = report["policy_published"]
if type(policy_published) is list:
policy_published = policy_published[0]
# Final RFC 9990 detection: the dmarc-2.0 XML namespace OR any
# RFC 9990-only field. Real-world reporters that follow the schema
# without declaring the namespace still get RFC 9990-aware
# warnings (missing DKIM selector, removed override-reason types,
# etc.) and a truthful audit trail in `xml_namespace`.
rfc_9990_only_policy_fields = {"np", "testing", "discovery_method"}
is_rfc_9990 = (
xml_namespace == RFC_9990_NAMESPACE
or "generator" in report_metadata
or any(f in policy_published for f in rfc_9990_only_policy_fields)
)
new_policy_published: dict[str, Any] = {}
new_policy_published["domain"] = policy_published["domain"]
adkim = "r"
if "adkim" in policy_published:
if policy_published["adkim"] is not None:
adkim = policy_published["adkim"]
new_policy_published["adkim"] = adkim
aspf = "r"
if "aspf" in policy_published:
if policy_published["aspf"] is not None:
aspf = policy_published["aspf"]
new_policy_published["aspf"] = aspf
new_policy_published["p"] = policy_published["p"]
sp = new_policy_published["p"]
if "sp" in policy_published:
if policy_published["sp"] is not None:
sp = policy_published["sp"]
new_policy_published["sp"] = sp
pct = None
if "pct" in policy_published:
if policy_published["pct"] is not None:
pct = policy_published["pct"]
new_policy_published["pct"] = pct
fo = None
if "fo" in policy_published:
if policy_published["fo"] is not None:
fo = policy_published["fo"]
new_policy_published["fo"] = fo
np_ = None
if "np" in policy_published:
if policy_published["np"] is not None:
np_ = policy_published["np"]
if np_ not in ("none", "quarantine", "reject"):
logger.warning(f"Invalid np value: {sanitize_for_log(str(np_))}")
new_policy_published["np"] = np_
testing = None
if "testing" in policy_published:
if policy_published["testing"] is not None:
testing = policy_published["testing"]
if testing not in ("n", "y"):
logger.warning(
f"Invalid testing value: {sanitize_for_log(str(testing))}"
)
new_policy_published["testing"] = testing
discovery_method = None
if "discovery_method" in policy_published:
if policy_published["discovery_method"] is not None:
discovery_method = policy_published["discovery_method"]
if discovery_method not in ("psl", "treewalk"):
logger.warning(
f"Invalid discovery_method value: "
f"{sanitize_for_log(str(discovery_method))}"
)
new_policy_published["discovery_method"] = discovery_method
new_report["policy_published"] = new_policy_published
normalize_timespan = _normalize_within_limit(
normalize_timespan, report["record"], begin_dt, end_dt, report_id
)
new_report_metadata["timespan_requires_normalization"] = normalize_timespan
records = _parse_report_records(
report["record"],
config=cfg,
is_rfc_9990=is_rfc_9990,
begin_dt=begin_dt,
end_dt=end_dt,
normalize=normalize_timespan,
keep_alive=keep_alive,
dns_budget=_ReverseDNSBudget(),
)
new_report["records"] = records
return cast(AggregateReport, new_report)
except InvalidAggregateReport:
# Already specific (an XML or date range limit); do not re-wrap it
# as an "Unexpected error" below.
raise
except expat.ExpatError as error:
raise InvalidAggregateReport(f"Invalid XML: {error.__str__()}") from error
except KeyError as error:
raise InvalidAggregateReport(f"Missing field: {error.__str__()}") from error
except AttributeError as error:
raise InvalidAggregateReport("Report missing required section") from error
except Exception as error:
raise InvalidAggregateReport(
f"Unexpected error: {error.__str__()}{_exc_origin(error)}"
) from error
def _decompress_gzip_bounded(data: bytes) -> bytes:
"""
Decompresses a gzip stream, refusing to produce more than
``MAX_DECOMPRESSED_REPORT_SIZE`` bytes.
The limit is read from the module global on every call so that it stays
a single source of truth (and can be patched in tests).
Args:
data: The gzip stream
Returns:
bytes: The decompressed data
Raises:
ParserError: The decompressed data exceeds the limit, or the gzip
stream ends before the decompressor reaches its end marker.
zlib.error: The stream is corrupt (bad header, deflate data, or
CRC); ``extract_report()`` wraps it in ``ParserError``.
"""
limit = MAX_DECOMPRESSED_REPORT_SIZE
decompressor = zlib.decompressobj(zlib.MAX_WBITS | 16)
decompressed = decompressor.decompress(data, max_length=limit + 1)
if len(decompressed) > limit:
raise ParserError(f"Decompressed report exceeds the {limit} byte limit")
# ``flush()`` ignores ``max_length``, so it must not run until the
# bounded read above has been accepted. It is bounded here: a bounded
# ``decompress()`` that returns fewer than ``max_length`` bytes has
# either consumed all of its input or hit the end of the stream (after
# which any trailing bytes are ``unused_data``, not input), so nothing
# unbounded is left for ``flush()`` to produce.
decompressed += decompressor.flush()
if not decompressor.eof:
# A one-shot zlib.decompress() raises on a stream that ends early;
# a decompressobj just stops, so the truncation is detected here.
raise ParserError("Incomplete or truncated gzip stream")
return decompressed
def extract_report(content: bytes | str | BinaryIO) -> str:
"""
Extracts report text from zip- or gzip-compressed content, and returns
plain XML or JSON content decoded as-is.
A caller-supplied file-like object is read from but never closed by
this function, on either the success or the exception path; it is
left open and positioned wherever the function's own reads left it,
so the caller is free to seek(0) and reuse it. A seekable stream is
seeked to position 0 unconditionally (not back to wherever the caller
had left it) after its 6-byte header is sniffed, and then read from
there, so a successful call typically leaves it at EOF. A non-seekable
stream is drained into a buffer this function creates and closes
itself; the caller's stream is left open but exhausted.
Args:
content: The report as a base64-encoded string, file-like object,
or bytes. A string that is not valid base64 is returned
unchanged.
Returns:
str: The extracted text
"""
file_object: BinaryIO | None = None
# True while file_object is a BytesIO this function created itself
# (from a str/bytes input, or as a buffer copied from a non-seekable
# caller stream); it is closed in the ``finally`` below. It is set to
# False when file_object instead aliases a caller-supplied seekable
# stream, which this function must never close.
owns_file_object = True
header: bytes
try:
if isinstance(content, str):
try:
file_object = BytesIO(
b64decode(
content.replace("\n", "").replace("\r", ""), validate=True
)
)
except binascii.Error:
return content
header = file_object.read(6)
file_object.seek(0)
elif isinstance(content, (bytes)):
file_object = BytesIO(bytes(content))
header = file_object.read(6)
file_object.seek(0)
else:
stream = cast(BinaryIO, content)
seekable = getattr(stream, "seekable", None)
can_seek = False
if callable(seekable):
try:
can_seek = bool(seekable())
except Exception:
can_seek = False
if can_seek:
header_raw = stream.read(6)
if isinstance(header_raw, str):
raise ParserError("File objects must be opened in binary (rb) mode")
header = bytes(header_raw)
stream.seek(0)
# file_object aliases the caller's own stream here, so it
# must not be closed by this function.
file_object = stream
owns_file_object = False
else:
header_raw = stream.read(6)
if isinstance(header_raw, str):
raise ParserError("File objects must be opened in binary (rb) mode")
header = bytes(header_raw)
remainder = stream.read()
file_object = BytesIO(header + bytes(remainder))
if header[: len(MAGIC_ZIP)] == MAGIC_ZIP:
_zip = zipfile.ZipFile(file_object)
limit = MAX_DECOMPRESSED_REPORT_SIZE
member = _zip.open(_zip.namelist()[0]).read(limit + 1)
if len(member) > limit:
raise ParserError(f"Decompressed report exceeds the {limit} byte limit")
report = member.decode(errors="ignore")
elif header[: len(MAGIC_GZIP)] == MAGIC_GZIP:
report = _decompress_gzip_bounded(file_object.read()).decode(
errors="ignore"
)
elif (
header[: len(MAGIC_XML)] == MAGIC_XML
or header[: len(MAGIC_XML_TAG)] == MAGIC_XML_TAG
or header[: len(MAGIC_JSON)] == MAGIC_JSON
):
# An uncompressed report is held to the decompressed cap, read
# at most one byte past it rather than decoded whole.
limit = MAX_DECOMPRESSED_REPORT_SIZE
data = file_object.read(limit + 1)
if len(data) > limit:
raise ParserError(f"Report exceeds the {limit} byte limit")
report = data.decode(errors="ignore")
else:
raise ParserError("Not a valid zip, gzip, json, or xml file")
except Exception as error:
raise ParserError(
f"Invalid archive file: {error.__str__()}{_exc_origin(error)}"
) from error
finally:
if file_object and owns_file_object:
try:
file_object.close()
except Exception:
# Best-effort close; a failure here shouldn't shadow
# whatever exception (or successful result) is already
# propagating out of this function.
pass
return report
def extract_report_from_file_path(
file_path: str | bytes | os.PathLike[str] | os.PathLike[bytes],
) -> str:
"""Extracts report from a file at the given file_path"""
try:
with open(os.fspath(file_path), "rb") as report_file:
return extract_report(report_file.read())
except FileNotFoundError:
raise ParserError("File was not found")
def parse_aggregate_report_file(
_input: str | bytes | BinaryIO,
*,
offline: bool = False,
always_use_local_files: bool = False,
reverse_dns_map_path: str | None = None,
reverse_dns_map_url: str | None = None,
ip_db_path: str | None = None,
nameservers: list[str] | None = None,
dns_timeout: float = DEFAULT_DNS_TIMEOUT,
dns_retries: int = DEFAULT_DNS_MAX_RETRIES,
keep_alive: Callable | None = None,
normalize_timespan_threshold_hours: float = 24.0,
config: ParserConfig | None = None,
) -> AggregateReport:
"""Parses a file at the given path, a file-like object, or bytes as an
aggregate DMARC report
``_input`` is forwarded to ``extract_report()`` unchanged, so a
caller-supplied file-like object is read from but never closed, on
either the success or the exception path; see ``extract_report()``
for exactly how such an object is left positioned afterward.
Args:
_input (str | bytes | IO): A path to a file, a file-like object, or bytes
offline (bool): Do not query online for geolocation or DNS
always_use_local_files (bool): Do not download files
reverse_dns_map_path (str): Path to a reverse DNS map file
reverse_dns_map_url (str): URL to a reverse DNS map file
ip_db_path (str): Path to a MMDB file from IPinfo, MaxMind, or DBIP
nameservers (list): A list of one or more nameservers to use
(Cloudflare's public DNS resolvers by default)
dns_timeout (float): Sets the DNS timeout in seconds
dns_retries (int): Number of times to retry DNS queries on timeout
or other transient errors
keep_alive (callable): Keep alive function. Not part of ``config``;
always applies.
normalize_timespan_threshold_hours (float): Normalize timespans beyond this
config (ParserConfig): a single object carrying all parsing and
enrichment options plus the caches; when provided, the
individual option keyword arguments listed above are ignored in
favor of the config's values.
Returns:
dict: The parsed DMARC aggregate report
"""
cfg = _resolve_config(
config,
offline=offline,
ip_db_path=ip_db_path,
always_use_local_files=always_use_local_files,
reverse_dns_map_path=reverse_dns_map_path,
reverse_dns_map_url=reverse_dns_map_url,
nameservers=nameservers,
dns_timeout=dns_timeout,
dns_retries=dns_retries,
normalize_timespan_threshold_hours=normalize_timespan_threshold_hours,
)
try:
xml = extract_report(_input)
except Exception as e:
raise InvalidAggregateReport(str(e) + _exc_origin(e)) from e
return parse_aggregate_report_xml(
xml,
config=cfg,
keep_alive=keep_alive,
)
def parsed_aggregate_reports_to_csv_rows(
reports: AggregateReport | list[AggregateReport],
) -> list[dict[str, Any]]:
"""
Converts one or more parsed aggregate reports to list of dicts in flat CSV
format
Args:
reports: A parsed aggregate report or list of parsed aggregate reports
Returns:
list: Parsed aggregate report data as a list of dicts in flat CSV
format
"""
def to_str(obj):
return str(obj).lower()
if isinstance(reports, dict):
reports = [reports]
rows = []
for report in reports:
xml_schema = report["xml_schema"]
org_name = report["report_metadata"]["org_name"]
org_email = report["report_metadata"]["org_email"]
org_extra_contact = report["report_metadata"]["org_extra_contact_info"]
report_id = report["report_metadata"]["report_id"]
begin_date = report["report_metadata"]["begin_date"]
end_date = report["report_metadata"]["end_date"]
normalized_timespan = report["report_metadata"][
"timespan_requires_normalization"
]
errors = "|".join(report["report_metadata"]["errors"])
domain = report["policy_published"]["domain"]
adkim = report["policy_published"]["adkim"]
aspf = report["policy_published"]["aspf"]
p = report["policy_published"]["p"]
sp = report["policy_published"]["sp"]
pct = report["policy_published"]["pct"]
fo = report["policy_published"]["fo"]
np_ = report["policy_published"].get("np", None)
testing = report["policy_published"].get("testing", None)
discovery_method = report["policy_published"].get("discovery_method", None)
report_dict: dict[str, Any] = dict(
xml_schema=xml_schema,
org_name=org_name,
org_email=org_email,
org_extra_contact_info=org_extra_contact,
report_id=report_id,
begin_date=begin_date,
end_date=end_date,
normalized_timespan=normalized_timespan,
errors=errors,
domain=domain,
adkim=adkim,
aspf=aspf,
p=p,
sp=sp,
np=np_,
pct=pct,
fo=fo,
testing=testing,
discovery_method=discovery_method,
)
for record in report["records"]:
row: dict[str, Any] = report_dict.copy()
row["begin_date"] = record["interval_begin"]
row["end_date"] = record["interval_end"]
row["source_ip_address"] = record["source"]["ip_address"]
row["source_country"] = record["source"]["country"]
row["source_reverse_dns"] = record["source"]["reverse_dns"]
row["source_base_domain"] = record["source"]["base_domain"]
row["source_name"] = record["source"]["name"]
row["source_type"] = record["source"]["type"]
row["source_asn"] = record["source"]["asn"]
row["source_as_name"] = record["source"]["as_name"]
row["source_as_domain"] = record["source"]["as_domain"]
row["count"] = record["count"]
row["spf_aligned"] = record["alignment"]["spf"]
row["dkim_aligned"] = record["alignment"]["dkim"]
row["dmarc_aligned"] = record["alignment"]["dmarc"]
row["disposition"] = record["policy_evaluated"]["disposition"]
policy_override_reasons = list(
map(
lambda r_: r_.get("type") or "none",
record["policy_evaluated"]["policy_override_reasons"],
)
)
policy_override_comments = list(
map(
lambda r_: r_.get("comment") or "none",
record["policy_evaluated"]["policy_override_reasons"],
)
)
row["policy_override_reasons"] = ",".join(policy_override_reasons)
row["policy_override_comments"] = "|".join(policy_override_comments)
row["envelope_from"] = record["identifiers"]["envelope_from"]
row["header_from"] = record["identifiers"]["header_from"]
envelope_to = record["identifiers"]["envelope_to"]
row["envelope_to"] = envelope_to
dkim_domains = []
dkim_selectors = []
dkim_results = []
for dkim_result in record["auth_results"]["dkim"]:
dkim_domains.append(dkim_result["domain"])
if "selector" in dkim_result:
dkim_selectors.append(dkim_result["selector"])
dkim_results.append(dkim_result["result"])
row["dkim_domains"] = ",".join(map(to_str, dkim_domains))
row["dkim_selectors"] = ",".join(map(to_str, dkim_selectors))
row["dkim_results"] = ",".join(map(to_str, dkim_results))
spf_domains = []
spf_scopes = []
spf_results = []
for spf_result in record["auth_results"]["spf"]:
spf_domains.append(spf_result["domain"])
spf_scopes.append(spf_result["scope"])
spf_results.append(spf_result["result"])
row["spf_domains"] = ",".join(map(to_str, spf_domains))
row["spf_scopes"] = ",".join(map(to_str, spf_scopes))
row["spf_results"] = ",".join(map(to_str, spf_results))
rows.append(row)
for r in rows:
for k, v in r.items():
if type(v) not in [str, int, bool]:
r[k] = ""
return rows
def parsed_aggregate_reports_to_csv(
reports: AggregateReport | list[AggregateReport],
) -> str:
"""
Converts one or more parsed aggregate reports to flat CSV format, including
headers
Args:
reports: A parsed aggregate report or list of parsed aggregate reports
Returns:
str: Parsed aggregate report data in flat CSV format, including headers
"""
fields = [
"xml_schema",
"org_name",
"org_email",
"org_extra_contact_info",
"report_id",
"begin_date",
"end_date",
"normalized_timespan",
"errors",
"domain",
"adkim",
"aspf",
"p",
"sp",
"np",
"pct",
"fo",
"testing",
"discovery_method",
"source_ip_address",
"source_country",
"source_reverse_dns",
"source_base_domain",
"source_name",
"source_type",
"source_asn",
"source_as_name",
"source_as_domain",
"count",
"spf_aligned",
"dkim_aligned",
"dmarc_aligned",
"disposition",
"policy_override_reasons",
"policy_override_comments",
"envelope_from",
"header_from",
"envelope_to",
"dkim_domains",
"dkim_selectors",
"dkim_results",
"spf_domains",
"spf_scopes",
"spf_results",
]
csv_file_object = StringIO(newline="\n")
writer = DictWriter(csv_file_object, fields)
writer.writeheader()
rows = parsed_aggregate_reports_to_csv_rows(reports)
for row in rows:
writer.writerow({key: _csv_safe(value) for key, value in row.items()})
csv_file_object.flush()
return csv_file_object.getvalue()
def parse_failure_report(
feedback_report: str,
sample: str,
msg_date: datetime,
*,
always_use_local_files: bool = False,
reverse_dns_map_path: str | None = None,
reverse_dns_map_url: str | None = None,
offline: bool = False,
ip_db_path: str | None = None,
nameservers: list[str] | None = None,
dns_timeout: float = DEFAULT_DNS_TIMEOUT,
dns_retries: int = DEFAULT_DNS_MAX_RETRIES,
strip_attachment_payloads: bool = False,
config: ParserConfig | None = None,
) -> FailureReport:
"""
Converts a DMARC failure report and sample to a dict
Args:
feedback_report (str): A message's feedback report as a string
sample (str): The RFC 822 headers or RFC 822 message sample
msg_date (datetime): The message's date, used as the arrival date
when the feedback report has no ``Arrival-Date`` field
ip_db_path (str): Path to a MMDB file from IPinfo, MaxMind, or DBIP
always_use_local_files (bool): Do not download files
reverse_dns_map_path (str): Path to a reverse DNS map file
reverse_dns_map_url (str): URL to a reverse DNS map file
offline (bool): Do not query online for geolocation or DNS
nameservers (list): A list of one or more nameservers to use
(Cloudflare's public DNS resolvers by default)
dns_timeout (float): Sets the DNS timeout in seconds
dns_retries (int): Number of times to retry DNS queries on timeout
or other transient errors
strip_attachment_payloads (bool): Remove attachment payloads from
failure report results
config (ParserConfig): a single object carrying all parsing and
enrichment options plus the caches; when provided, the
individual option keyword arguments listed above are ignored in
favor of the config's values.
Returns:
dict: A parsed report and sample
"""
cfg = _resolve_config(
config,
offline=offline,
ip_db_path=ip_db_path,
always_use_local_files=always_use_local_files,
reverse_dns_map_path=reverse_dns_map_path,
reverse_dns_map_url=reverse_dns_map_url,
nameservers=nameservers,
dns_timeout=dns_timeout,
dns_retries=dns_retries,
strip_attachment_payloads=strip_attachment_payloads,
)
delivery_results = ["delivered", "spam", "policy", "reject", "other"]
try:
parsed_report: dict[str, Any] = {}
report_values = feedback_report_regex.findall(feedback_report)
for report_value in report_values:
key = report_value[0].lower().replace("-", "_")
parsed_report[key] = report_value[1]
if "arrival_date" not in parsed_report:
if msg_date is None:
raise InvalidFailureReport("Failure sample is not a valid email")
parsed_report["arrival_date"] = msg_date.isoformat()
if "version" not in parsed_report:
parsed_report["version"] = None
if "user_agent" not in parsed_report:
parsed_report["user_agent"] = None
if "delivery_result" not in parsed_report:
parsed_report["delivery_result"] = None
else:
for delivery_result in delivery_results:
if delivery_result in parsed_report["delivery_result"].lower():
parsed_report["delivery_result"] = delivery_result
break
if parsed_report["delivery_result"] not in delivery_results:
parsed_report["delivery_result"] = "other"
try:
arrival_utc = human_timestamp_to_datetime(
parsed_report["arrival_date"], to_utc=True
)
except (ValueError, OverflowError) as error:
raise InvalidFailureReport(
f"Invalid Arrival-Date: {sanitize_for_log(str(error))}"
) from error
try:
_check_not_future(arrival_utc, "Arrival-Date")
except ValueError as error:
raise InvalidFailureReport(str(error)) from error
arrival_utc = arrival_utc.strftime("%Y-%m-%d %H:%M:%S")
parsed_report["arrival_date_utc"] = arrival_utc
ip_address = _strip_zone_id(
re.split(r"\s", parsed_report["source_ip"]).pop(0), "Source-IP"
)
parsed_report_source = get_ip_address_info(
ip_address,
cache=cfg.ip_address_cache,
ip_db_path=cfg.ip_db_path,
always_use_local_files=cfg.always_use_local_files,
reverse_dns_map_path=cfg.reverse_dns_map_path,
reverse_dns_map_url=cfg.reverse_dns_map_url,
reverse_dns_map=cfg.reverse_dns_map,
offline=cfg.offline,
nameservers=cfg.nameservers,
timeout=cfg.dns_timeout,
retries=cfg.dns_retries,
psl_overrides_path=cfg.psl_overrides_path,
psl_overrides_url=cfg.psl_overrides_url,
)
parsed_report["source"] = parsed_report_source
del parsed_report["source_ip"]
# Identity-Alignment is REQUIRED per RFC 9991 §4. Default silently for
# backward compatibility with pre-9991 reporters, but log so the
# offending reporter is visible. Values are CFWS-separated per the
# ABNF, so each mechanism is stripped after splitting.
if "identity_alignment" not in parsed_report:
logger.warning(
"Failure report missing required 'Identity-Alignment' "
"field (RFC 9991 §4); defaulting to no aligned mechanisms"
)
parsed_report["authentication_mechanisms"] = []
else:
raw_alignment = parsed_report["identity_alignment"].strip()
if raw_alignment.lower() == "none":
parsed_report["authentication_mechanisms"] = []
else:
parsed_report["authentication_mechanisms"] = [
m.strip() for m in raw_alignment.split(",") if m.strip()
]
del parsed_report["identity_alignment"]
# Auth-Failure is REQUIRED per RFC 9991 §4. Comma-separated per ABNF
# so strip each token.
if "auth_failure" not in parsed_report:
logger.warning(
"Failure report missing required 'Auth-Failure' field "
"(RFC 9991 §4); defaulting to 'dmarc'"
)
parsed_report["auth_failure"] = "dmarc"
parsed_report["auth_failure"] = [
f.strip() for f in parsed_report["auth_failure"].split(",") if f.strip()
]
# Feedback-Type is REQUIRED per RFC 5965 §3.1, but some gateways
# (e.g. Exim/cPanel-based ones that send a plain-text summary without
# a machine-readable message/feedback-report part) omit it. The
# Elasticsearch/OpenSearch outputs require the key, so default it
# instead of dropping the report there (see issue #332).
if "feedback_type" not in parsed_report:
logger.warning(
"Failure report missing required 'Feedback-Type' field "
"(RFC 5965 §3.1); defaulting to 'auth-failure'"
)
parsed_report["feedback_type"] = "auth-failure"
# Authentication-Results is likewise REQUIRED per RFC 6591 §3.1 and
# required by the Elasticsearch/OpenSearch outputs.
if "authentication_results" not in parsed_report:
logger.warning(
"Failure report missing required 'Authentication-Results' "
"field (RFC 6591 §3.1); defaulting to None"
)
parsed_report["authentication_results"] = None
optional_fields = [
"original_envelope_id",
"dkim_domain",
"original_mail_from",
"original_rcpt_to",
]
for optional_field in optional_fields:
if optional_field not in parsed_report:
parsed_report[optional_field] = None
parsed_sample = parse_email(
sample, strip_attachment_payloads=cfg.strip_attachment_payloads
)
if "reported_domain" not in parsed_report:
# reported_domain is a required str in the FailureReport contract
# (parsedmarc/types.py) and is read directly by the Elasticsearch
# and OpenSearch outputs, so it cannot be defaulted to None. When
# the report omits the Reported-Domain field and the sample's From
# header is missing or unparseable, the report is unusable.
sample_from = parsed_sample.get("from")
if not isinstance(sample_from, dict) or not sample_from.get("domain"):
raise InvalidFailureReport(
"The failure report has no Reported-Domain field, and no "
"domain could be parsed from the From header of the "
"included sample message"
)
parsed_report["reported_domain"] = sample_from["domain"]
sample_headers_only = False
number_of_attachments = len(parsed_sample["attachments"])
if number_of_attachments < 1 and parsed_sample["body"] is None:
sample_headers_only = True
if sample_headers_only and parsed_sample["has_defects"]:
del parsed_sample["defects"]
del parsed_sample["defects_categories"]
del parsed_sample["has_defects"]
parsed_report["sample_headers_only"] = sample_headers_only
parsed_report["sample"] = sample
parsed_report["parsed_sample"] = parsed_sample
return cast(FailureReport, parsed_report)
except InvalidFailureReport:
# Already a clear, specific message; do not re-wrap it as an
# "Unexpected error" below.
raise
except KeyError as error:
raise InvalidFailureReport(f"Missing value: {error.__str__()}") from error
except MessageLimitExceeded as error:
raise InvalidFailureReport(f"Sample message rejected: {error}") from error
except Exception as error:
raise InvalidFailureReport(
f"Unexpected error: {error.__str__()}{_exc_origin(error)}"
) from error
def parsed_failure_reports_to_csv_rows(
reports: FailureReport | list[FailureReport],
) -> list[dict[str, Any]]:
"""
Converts one or more parsed failure reports to a list of dicts in flat CSV
format
Args:
reports: A parsed failure report or list of parsed failure reports
Returns:
list: Parsed failure report data as a list of dicts in flat CSV format
"""
if isinstance(reports, dict):
reports = [reports]
rows = []
for report in reports:
row: dict[str, Any] = dict(report)
row["source_ip_address"] = report["source"]["ip_address"]
row["source_reverse_dns"] = report["source"]["reverse_dns"]
row["source_base_domain"] = report["source"]["base_domain"]
row["source_name"] = report["source"]["name"]
row["source_type"] = report["source"]["type"]
row["source_asn"] = report["source"]["asn"]
row["source_as_name"] = report["source"]["as_name"]
row["source_as_domain"] = report["source"]["as_domain"]
row["source_country"] = report["source"]["country"]
del row["source"]
row["subject"] = report["parsed_sample"].get("subject")
row["auth_failure"] = ",".join(report["auth_failure"])
authentication_mechanisms = report["authentication_mechanisms"]
row["authentication_mechanisms"] = ",".join(authentication_mechanisms)
del row["sample"]
del row["parsed_sample"]
rows.append(row)
return rows
def parsed_failure_reports_to_csv(
reports: FailureReport | list[FailureReport],
) -> str:
"""
Converts one or more parsed failure reports to flat CSV format, including
headers
Args:
reports: A parsed failure report or list of parsed failure reports
Returns:
str: Parsed failure report data in flat CSV format, including headers
"""
fields = [
"feedback_type",
"user_agent",
"version",
"original_envelope_id",
"original_mail_from",
"original_rcpt_to",
"arrival_date",
"arrival_date_utc",
"subject",
"message_id",
"authentication_results",
"dkim_domain",
"source_ip_address",
"source_country",
"source_reverse_dns",
"source_base_domain",
"source_name",
"source_type",
"source_asn",
"source_as_name",
"source_as_domain",
"delivery_result",
"auth_failure",
"reported_domain",
"authentication_mechanisms",
"sample_headers_only",
]
csv_file = StringIO()
csv_writer = DictWriter(csv_file, fieldnames=fields)
csv_writer.writeheader()
rows = parsed_failure_reports_to_csv_rows(reports)
for row in rows:
new_row: dict[str, Any] = {}
for key in fields:
new_row[key] = _csv_safe(row.get(key))
csv_writer.writerow(new_row)
return csv_file.getvalue()
def _decode_mime_payload(part: email.message.Message, fallback: str) -> str:
"""
Returns a MIME part's payload decoded per its ``Content-Transfer-Encoding``
Three non-obvious constraints shape this helper:
- Only ``quoted-printable`` and ``base64`` parts are touched at all. The
caller parses the message from a ``str``, so for a part with a 7bit,
8bit, or absent ``Content-Transfer-Encoding`` there is nothing to undo,
and ``compat32``'s ``get_payload(decode=True)`` would round-trip the
already-correct text through ``raw-unicode-escape`` bytes — re-decoding
those as the declared charset mangles every non-ASCII character. Such
parts return ``fallback``, which is the payload text as-is.
- The standard library's email parser nests **every** ``message/*``
subtype — including ``message/feedback-report`` and ``message/rfc822`` —
as a child ``Message`` object, so ``get_payload(decode=True)`` returns
``None`` for those parts rather than decoded bytes. The caller's
re-serialization of that child message is passed in as ``fallback``, and
any transfer encoding is undone here instead.
- A composite part carrying a real ``Content-Transfer-Encoding`` of
``quoted-printable`` or ``base64`` is illegal per RFC 2045 §6.4, but
real reporters send them anyway (issue
`#882 <https://github.com/domainaware/parsedmarc/issues/882>`_), and the
standard library nests the still-encoded text as-is. Parsing that text
as a message can insert a header/body separator that the encoded text
never had, which has to be undone before decoding; see below.
Any unexpected failure returns ``fallback`` unchanged, so a report that
parsed before cannot start failing because of this decoding step.
Args:
part: The MIME part
fallback: The payload text to use when the part carries no transfer
encoding to undo, or cannot be decoded with ``decode=True``
(i.e. nested ``message/*`` parts)
Returns:
str: The decoded payload
"""
try:
cte = (part.get("Content-Transfer-Encoding") or "").strip().lower()
if cte not in ("quoted-printable", "base64"):
return fallback
payload_bytes = part.get_payload(decode=True)
if isinstance(payload_bytes, bytes):
charset = part.get_content_charset() or "utf-8"
try:
return payload_bytes.decode(charset, errors="replace")
except LookupError:
return payload_bytes.decode("utf-8", errors="replace")
# Nested message/* part: undo the transfer encoding by hand.
#
# The still-encoded text was parsed as a message first. A soft line
# break (RFC 2045 §6.7) splits a long field across lines, and the
# continuation line has neither a colon nor leading whitespace, so the
# parser records a MissingHeaderBodySeparatorDefect and treats the
# remainder as the body -- which makes the re-serialized `fallback`
# carry a blank line that the encoded text never had. Decoding then
# consumes the soft break and leaves that inserted newline in the
# middle of the value, truncating it. Drop exactly that one separator:
# a header value cannot contain a blank line, so the first "\n\n" is
# the inserted boundary. Text with a genuine separator raises no
# defect, and nothing is removed.
text = fallback
child = part.get_payload()
if isinstance(child, list) and child:
child = child[0]
if isinstance(child, email.message.Message) and any(
isinstance(defect, email.errors.MissingHeaderBodySeparatorDefect)
for defect in child.defects
):
text = fallback.replace("\n\n", "\n", 1)
if cte == "quoted-printable":
return quopri.decodestring(text.encode("utf-8", errors="replace")).decode(
"utf-8", errors="replace"
)
return b64decode(text).decode("utf-8", errors="replace")
except Exception:
return fallback
def parse_report_email(
input_: bytes | str,
*,
offline: bool = False,
ip_db_path: str | None = None,
always_use_local_files: bool = False,
reverse_dns_map_path: str | None = None,
reverse_dns_map_url: str | None = None,
nameservers: list[str] | None = None,
dns_timeout: float = DEFAULT_DNS_TIMEOUT,
dns_retries: int = DEFAULT_DNS_MAX_RETRIES,
strip_attachment_payloads: bool = False,
keep_alive: Callable | None = None,
normalize_timespan_threshold_hours: float = 24.0,
config: ParserConfig | None = None,
) -> ParsedReport:
"""
Parses a DMARC or SMTP TLS report from an email
Args:
input_: An emailed DMARC report in RFC 822 format, as bytes or a string
ip_db_path (str): Path to a MMDB file from IPinfo, MaxMind, or DBIP
always_use_local_files (bool): Do not download files
reverse_dns_map_path (str): Path to a reverse DNS map
reverse_dns_map_url (str): URL to a reverse DNS map
offline (bool): Do not query online for geolocation or DNS
nameservers (list): A list of one or more nameservers to use
dns_timeout (float): Sets the DNS timeout in seconds
dns_retries (int): Number of times to retry DNS queries on timeout
or other transient errors
strip_attachment_payloads (bool): Remove attachment payloads from
failure report results
keep_alive (callable): keep alive function. Not part of ``config``;
always applies.
normalize_timespan_threshold_hours (float): Normalize timespans beyond this
config (ParserConfig): a single object carrying all parsing and
enrichment options plus the caches; when provided, the
individual option keyword arguments listed above are ignored in
favor of the config's values.
Returns:
dict:
* ``report_type``: ``aggregate``, ``failure``, or ``smtp_tls``
* ``report``: The parsed report
Raises:
ParserError: If the message is not a valid report. Whatever the
input, only ``ParserError`` subclasses are raised.
A message that is not a valid report, or whose attachment is
not one, raises ``InvalidDMARCReport``, or
``InvalidSMTPTLSReport`` for an SMTP TLS attachment (both are
``ParserError`` subclasses). Any other failure to parse the
message raises a plain ``ParserError``.
"""
cfg = _resolve_config(
config,
offline=offline,
ip_db_path=ip_db_path,
always_use_local_files=always_use_local_files,
reverse_dns_map_path=reverse_dns_map_path,
reverse_dns_map_url=reverse_dns_map_url,
nameservers=nameservers,
dns_timeout=dns_timeout,
dns_retries=dns_retries,
strip_attachment_payloads=strip_attachment_payloads,
normalize_timespan_threshold_hours=normalize_timespan_threshold_hours,
)
result: ParsedReport | None = None
msg_date: datetime = datetime.now(timezone.utc)
try:
input_data: str | bytes | bytearray | memoryview = input_
if isinstance(input_data, (bytes, bytearray, memoryview)):
input_bytes = bytes(input_data)
if is_outlook_msg(input_bytes):
converted = convert_outlook_msg(input_bytes)
if isinstance(converted, str):
input_str = converted
else:
input_str = bytes(converted).decode(
encoding="utf8", errors="replace"
)
else:
input_str = input_bytes.decode(encoding="utf8", errors="replace")
else:
input_str = input_data
# Reject a message whose structure makes parsing slow before any
# parser builds its tree (GHSA-qcc2-4x3v-v4g7)
check_message_limits(input_str)
msg = mailparser.parse_from_string(input_str)
msg_headers = json.loads(msg.headers_json)
if "Date" in msg_headers:
try:
msg_date = human_timestamp_to_datetime(msg_headers["Date"])
except (ValueError, OverflowError) as e:
raise InvalidDMARCReport(
f"Invalid Date header: {sanitize_for_log(str(e))}"
) from e
date = email.utils.format_datetime(msg_date)
msg = email.message_from_string(input_str)
except MessageLimitExceeded as e:
raise InvalidDMARCReport(f"Message rejected: {e}") from e
except InvalidDMARCReport:
raise
except Exception as e:
raise ParserError(e.__str__() + _exc_origin(e)) from e
subject_text = "None"
feedback_report = None
smtp_tls_report = None
sample = None
# id() of every sample candidate and every part inside one
sample_parts: set[int] = set()
is_feedback_report: bool = False
if "From" in msg_headers:
logger.info(
"Parsing mail from {} on {}".format(
sanitize_for_log(str(msg_headers["From"])), date
)
)
if "Subject" in msg_headers:
# RFC 2047 decoding can yield raw CR/LF, so escape control
# characters before the subject reaches an error message or log.
subject_text = sanitize_for_log(str(msg_headers["Subject"]))
# Serializing a part (``str(payload_obj[0])``) re-renders any
# ``message/rfc822`` sub-message, which can raise e.g.
# ``email.errors.HeaderWriteError`` on a control character in a header or
# ``RecursionError`` on deeply nested multiparts. Callers rely on this
# function raising only ``ParserError`` subclasses so that one bad message
# is skipped (and moved to ``Invalid``) rather than aborting a whole
# mailbox batch (GHSA-qxwv-2vwj-v828).
try:
for part in msg.walk():
if id(part) in sample_parts:
# The sample is the reported message, spoofer's content
# included; an attachment inside it is not a report.
continue
content_type = part.get_content_type().lower()
payload_obj = part.get_payload()
if not isinstance(payload_obj, list):
payload_obj = [payload_obj]
if content_type.startswith("multipart/") or content_type == "text/html":
continue
# A part is serialized only where its text is used: str() of a
# composite part renders its whole subtree, and walk() visits
# every level, so serializing each one was quadratic in the
# nesting (98 multiparts around a 20 MiB leaf took 26.6 s;
# GHSA-qcc2-4x3v-v4g7).
if is_feedback_report and content_type in EMAIL_SAMPLE_CONTENT_TYPES:
# A feedback report has one sample (RFC 6591 §3.1); a message
# nested inside it belongs to it and must not replace it.
if sample is None:
sample = _decode_mime_payload(part, str(payload_obj[0]))
# A later candidate is not read as the sample, and neither
# are the parts inside it.
sample_parts.update(id(sample_part) for sample_part in part.walk())
continue
if content_type == "message/feedback-report" and is_feedback_report:
# A feedback report has one message/feedback-report part
# (RFC 6591 §3); a nested one must not be rendered again.
continue
if content_type != "message/feedback-report" and isinstance(
payload_obj[0], email.message.Message
):
# Another message/* body; walk() visits its parts.
continue
payload = str(payload_obj[0])
if content_type == "message/feedback-report":
is_feedback_report = True
# The RFC 6591 part wins over a legacy text report read
# earlier; its sample follows it.
sample = None
decoded_payload = _decode_mime_payload(part, payload)
try:
if "Feedback-Type" in decoded_payload:
feedback_report = decoded_payload
else:
feedback_report = b64decode(decoded_payload).__str__()
feedback_report = feedback_report.lstrip("b'").rstrip("'")
feedback_report = feedback_report.replace("\\r", "")
feedback_report = feedback_report.replace("\\n", "\n")
except (ValueError, TypeError, binascii.Error):
feedback_report = decoded_payload
elif content_type == "application/tlsrpt+json":
if not payload.strip().startswith("{"):
payload = b64decode(payload).decode("utf-8", errors="replace")
smtp_tls_report = parse_smtp_tls_report_json(payload)
return {"report_type": "smtp_tls", "report": smtp_tls_report}
elif content_type == "application/tlsrpt+gzip":
payload = extract_report(payload)
smtp_tls_report = parse_smtp_tls_report_json(payload)
return {"report_type": "smtp_tls", "report": smtp_tls_report}
elif content_type == "text/plain":
# A legacy text report has no message/feedback-report part.
# Once one is read, a text part (which may sit inside the
# sample, the spoofer's own text) must not replace the
# report's Source-IP or its sample.
if (
not is_feedback_report
and "A message claiming to be from you has failed" in payload
):
try:
parts = payload.split("detected.", 1)
field_matches = text_report_regex.findall(parts[0])
fields = dict()
for match in field_matches:
field_name = match[0].lower().replace(" ", "-")
fields[field_name] = match[1].strip()
feedback_report = "Arrival-Date: {}\nSource-IP: {}".format(
fields["received-date"], fields["sender-ip-address"]
)
except Exception as e:
error = f'Unable to parse message with subject "{subject_text}": {e}{_exc_origin(e)}'
raise InvalidDMARCReport(error) from e
sample = parts[1].lstrip()
logger.debug(sample)
else:
try:
payload_bytes = b64decode(payload)
if payload_bytes.startswith(MAGIC_ZIP) or payload_bytes.startswith(
MAGIC_GZIP
):
payload_text = extract_report(payload_bytes)
else:
payload_text = payload_bytes.decode("utf-8", errors="replace")
if payload_text.strip().startswith("{"):
smtp_tls_report = parse_smtp_tls_report_json(payload_text)
result = {"report_type": "smtp_tls", "report": smtp_tls_report}
return result
elif payload_text.strip().startswith("<"):
aggregate_report = parse_aggregate_report_xml(
payload_text,
config=cfg,
keep_alive=keep_alive,
)
result = {
"report_type": "aggregate",
"report": aggregate_report,
}
return result
except (TypeError, ValueError, binascii.Error):
# b64decode() rejected this MIME part's payload as not
# base64-decodable, so it isn't a report attachment; fall
# through and keep walking the message for a part (or the
# feedback-report/sample pair below) that is.
pass
except InvalidDMARCReport as e:
error = f'Message with subject "{subject_text}" is not a valid DMARC report: {e}'
raise InvalidDMARCReport(error) from e
except InvalidSMTPTLSReport:
# Named with the message's subject by the handler below.
raise
except Exception as e:
error = f'Unable to parse message with subject "{subject_text}": {e}{_exc_origin(e)}'
raise ParserError(error) from e
except InvalidSMTPTLSReport as e:
# From either the tlsrpt content-type branches or a JSON attachment.
error = (
f'Message with subject "{subject_text}" is not a valid SMTP TLS report: {e}'
)
raise InvalidSMTPTLSReport(error) from e
except ParserError:
raise
except Exception as e:
error = f'Unable to parse message with subject "{subject_text}": {e}{_exc_origin(e)}'
raise ParserError(error) from e
if feedback_report and sample:
try:
failure_report = parse_failure_report(
feedback_report,
sample,
msg_date,
config=cfg,
)
except InvalidFailureReport as e:
error = (
f'Message with subject "{subject_text}" '
"is not a valid "
f"failure DMARC report: {e}"
)
raise InvalidFailureReport(error) from e
result = {"report_type": "failure", "report": failure_report}
return result
# Every branch that sets ``result`` returns immediately, so reaching this
# point always means no report was found.
error = f'Message with subject "{subject_text}" is not a valid report'
raise InvalidDMARCReport(error)
# An RFC 5322 header field name (printable ASCII excluding the colon) followed
# by a colon at the start of a line, or an mbox "From " separator.
_email_header_regex = re.compile(r"^(From |[\x21-\x39\x3b-\x7e]+:)")
def _looks_like_email(text: str) -> bool:
"""Returns True if the first line looks like an email header.
Callers pass already-``lstrip()``-ed text, so the first line is the first
meaningful line.
"""
first_line = text.split("\n", 1)[0]
return _email_header_regex.match(first_line) is not None
def _leading_text(content: str | bytes) -> str:
"""Return the start of ``content`` (512 characters at most) without
leading whitespace, to sniff its format."""
if isinstance(content, (bytes, bytearray, memoryview)):
sniff = bytes(content)[:512].decode("utf-8", errors="replace")
else:
sniff = content[:512]
return sniff.lstrip()
def _describe_parse_failure(
content: str | bytes,
aggregate_error: str,
smtp_tls_error: str,
email_error: str,
) -> str:
"""Builds a human-readable reason for a parse_report_file failure.
parse_report_file tries the aggregate XML, SMTP TLS JSON, and report-email
parsers in turn; when all three reject the input, only the parser for the
format the content actually resembles produced a meaningful error. The
other two are noise -- a malformed aggregate report is also "not JSON" and
"not an email". Sniff the leading non-whitespace byte to surface the single
relevant reason.
"""
sniff = _leading_text(content)
if sniff.startswith("<"):
return f"Invalid aggregate report: {aggregate_error}"
if sniff.startswith("{"):
return f"Invalid SMTP TLS report: {smtp_tls_error}"
if _looks_like_email(sniff):
return f"Invalid report email: {email_error}"
return (
"Not a recognized report format (not a DMARC aggregate XML report, "
"an SMTP TLS JSON report, or a DMARC report email)"
)
def _read_up_to(file_object: BinaryIO, limit: int) -> bytes:
"""Read from ``file_object`` until end of file or ``limit`` bytes,
whichever comes first. A raw or unbuffered stream (a socket file, a
pipe) may return fewer bytes than asked for without being at its end, so
one sized ``read()`` could hand the parser a truncated report."""
# One growing buffer, not a list of chunks: a stream that returns a byte
# at a time would otherwise cost one bytes object per byte read.
content = bytearray()
while len(content) < limit:
chunk = file_object.read(limit - len(content))
if not chunk:
break
content += chunk
return bytes(content)
def parse_report_file(
input_: bytes | str | os.PathLike[str] | os.PathLike[bytes] | BinaryIO,
*,
nameservers: list[str] | None = None,
dns_timeout: float = DEFAULT_DNS_TIMEOUT,
dns_retries: int = DEFAULT_DNS_MAX_RETRIES,
strip_attachment_payloads: bool = False,
ip_db_path: str | None = None,
always_use_local_files: bool = False,
reverse_dns_map_path: str | None = None,
reverse_dns_map_url: str | None = None,
offline: bool = False,
keep_alive: Callable | None = None,
normalize_timespan_threshold_hours: float = 24.0,
config: ParserConfig | None = None,
) -> ParsedReport:
"""Parses a DMARC aggregate report, DMARC failure report, or SMTP TLS
report from a file at the given path, a file-like object, or bytes
A path is opened and closed by this function. A caller-supplied file
object is read from but never closed, on either the success or the
exception path; it is left open and positioned wherever its own
``read()`` left it, so the caller is free to ``seek(0)`` and reuse it.
When the input is over the size cap, that is one byte past the cap: no
more is read.
Args:
input_ (str | os.PathLike | bytes | BinaryIO): A path to a file,
a file-like object, or bytes
nameservers (list): A list of one or more nameservers to use
(Cloudflare's public DNS resolvers by default)
dns_timeout (float): Sets the DNS timeout in seconds
dns_retries (int): Number of times to retry DNS queries on timeout
or other transient errors
strip_attachment_payloads (bool): Remove attachment payloads from
failure report results
ip_db_path (str): Path to a MMDB file from IPinfo, MaxMind, or DBIP
always_use_local_files (bool): Do not download files
reverse_dns_map_path (str): Path to a reverse DNS map
reverse_dns_map_url (str): URL to a reverse DNS map
offline (bool): Do not make online queries for geolocation or DNS
keep_alive (callable): Keep alive function. Not part of ``config``;
always applies.
normalize_timespan_threshold_hours (float): Normalize timespans beyond this
config (ParserConfig): a single object carrying all parsing and
enrichment options plus the caches; when provided, the
individual option keyword arguments listed above are ignored in
favor of the config's values.
Returns:
dict: The parsed DMARC report
Raises:
ParserError: if the input is not a valid report, or is larger than
``MAX_DECOMPRESSED_REPORT_SIZE`` (refused before it is read into
memory)
"""
cfg = _resolve_config(
config,
offline=offline,
ip_db_path=ip_db_path,
always_use_local_files=always_use_local_files,
reverse_dns_map_path=reverse_dns_map_path,
reverse_dns_map_url=reverse_dns_map_url,
nameservers=nameservers,
dns_timeout=dns_timeout,
dns_retries=dns_retries,
strip_attachment_payloads=strip_attachment_payloads,
normalize_timespan_threshold_hours=normalize_timespan_threshold_hours,
)
file_object: BinaryIO
# An input over the cap is refused before it is held in memory (#949). A
# valid compressed report is smaller than its decompressed form, so the
# one cap serves both; the message is the one the content check gives.
too_large = ParserError(_size_cap_text(b""))
if isinstance(input_, (str, os.PathLike)):
file_path = os.fspath(input_)
logger.debug(f"Parsing {file_path}")
if os.path.getsize(file_path) > MAX_DECOMPRESSED_REPORT_SIZE:
raise too_large
# A path is a handle we opened ourselves, so it must be closed on
# both the success and exception paths.
with open(file_path, "rb") as file_object:
content = _read_up_to(file_object, MAX_DECOMPRESSED_REPORT_SIZE + 1)
else:
if isinstance(input_, (bytes, bytearray, memoryview)):
if memoryview(input_).nbytes > MAX_DECOMPRESSED_REPORT_SIZE:
raise too_large
# The BytesIO wrapper is created here, so this function owns
# it and closes it once the bytes have been read out.
file_object = BytesIO(bytes(input_))
content = file_object.read()
file_object.close()
else:
# A caller-supplied file-like object is never closed by this
# function, on success or failure: the caller may want to
# seek(0) and retry, inspect tell(), or otherwise reuse the
# handle afterward.
file_object = input_
content = _read_up_to(file_object, MAX_DECOMPRESSED_REPORT_SIZE + 1)
if len(content) > MAX_DECOMPRESSED_REPORT_SIZE:
raise too_large
if content.startswith(MAGIC_ZIP) or content.startswith(MAGIC_GZIP):
content = extract_report(content)
# parse_report_file tries the three report formats in turn. When all three
# reject the input, keep each format's error text so the final message
# can explain *why* the file is invalid instead of a bare "Not a valid
# report" (see _describe_parse_failure). Only the text is kept: an
# exception's traceback holds the failed parser's frames, and with them
# its partial tree, while the next parser runs (GHSA-q7gm-8cj9-94hr).
aggregate_message = smtp_tls_message = email_message = ""
try:
report = parse_aggregate_report_file(
content,
config=cfg,
keep_alive=keep_alive,
)
return {"report_type": "aggregate", "report": report}
except InvalidAggregateReport as aggregate_error:
aggregate_message = str(aggregate_error)
leading = _leading_text(content)
if leading.startswith("<"):
# JSON never begins with "<", and parse_smtp_tls_report_json() would
# first decode the whole content.
smtp_tls_message = "not JSON"
else:
try:
report = parse_smtp_tls_report_json(content)
return {"report_type": "smtp_tls", "report": report}
except InvalidSMTPTLSReport as smtp_tls_error:
smtp_tls_message = str(smtp_tls_error)
if leading.startswith(("<", "{")):
# An RFC 822 message begins with a header line, never "<" or "{";
# such a file's failure is reported as the aggregate or SMTP TLS
# error, so no email reason is needed.
pass
elif _exceeds_size_cap(content):
# The email parser has no size cap of its own.
email_message = _size_cap_text(content)
else:
try:
return parse_report_email(content, config=cfg, keep_alive=keep_alive)
except InvalidDMARCReport as email_error:
email_message = str(email_error)
message = _describe_parse_failure(
content, aggregate_message, smtp_tls_message, email_message
)
# The cause is a fresh exception with the relevant format's text, so the
# chain is kept without the failed parse's traceback.
if leading.startswith("<"):
raise ParserError(message) from InvalidAggregateReport(aggregate_message)
if leading.startswith("{"):
raise ParserError(message) from InvalidSMTPTLSReport(smtp_tls_message)
raise ParserError(message) from InvalidDMARCReport(email_message)
_ASCII_UPPER_TO_LOWER = str.maketrans(
"ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz"
)
def _aggregate_report_dedup_key(report: AggregateReport) -> str:
"""Return the ``seen_aggregate_report_ids`` cache key for ``report``.
RFC 9990 §3.5.1 defines a Report-ID as unique only "among reports to the
same domain", so the key is the reporter, the Report-ID, and the policy
domain. ASCII domain letters are compared without case per RFC 4343 §3,
without changing the domain in the report itself. A JSON array encodes
the three fields without the ambiguity of a delimiter that a field could
contain, and the key is the SHA-256 hex digest of that JSON, so it is 64
characters long whatever the sender puts in the three fields
(GHSA-9fw6-mwqr-p43m: the JSON itself was the key, so a report with a
multi-megabyte Report-ID cost that much memory in the cache).
Every writer of the shared cache must build its key here -- the mbox and
mailbox classifier and the CLI's direct-file path -- or a report read
through one path is not recognized as a duplicate of the same report
read through another.
"""
report_org = report["report_metadata"]["org_name"]
report_id = report["report_metadata"]["report_id"]
report_domain = report["policy_published"]["domain"]
# The existing XML parser returns None for an empty <domain/>.
if isinstance(report_domain, str):
report_domain = report_domain.translate(_ASCII_UPPER_TO_LOWER)
encoded = json.dumps((report_org, report_id, report_domain))
return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
def _classify_parsed_email(
parsed_email: ParsedReport,
aggregate_reports: list[AggregateReport],
failure_reports: list[FailureReport],
smtp_tls_reports: list[SMTPTLSReport],
*,
seen_aggregate_report_ids: ExpiringDict,
pending_aggregate_keys: set[str] | None = None,
) -> ReportType:
"""Classify a parsed report email, appending it to the matching list.
Owns the seen-aggregate-report-ID dedup check against
``seen_aggregate_report_ids``: an aggregate report already seen (keyed
on the reporter, the Report-ID, and the reported policy domain) is
logged and dropped instead of appended. Shared, unmodified, by the
sequential and parallel branches of ``get_dmarc_reports_from_mbox`` and
``get_dmarc_reports_from_mailbox`` so both dedup identically -- callers
pass the config's ``seen_aggregate_report_ids`` cache so dedup state
stays scoped to whichever ``ParserConfig`` (explicit or module-default)
is in effect. The key comes from :func:`_aggregate_report_dedup_key`,
which the CLI's direct-file path also uses, so a report seen in a file
and again in an mbox in the same run is recognized as a duplicate.
``pending_aggregate_keys`` lets a caller *defer* the dedup-cache write:
when a set is supplied, a newly seen key is staged in that set instead
of being written to ``seen_aggregate_report_ids``, and the dedup check
consults both. The caller then folds the staged keys into the cache
once the batch is known to have been saved (see
``get_dmarc_reports_from_mailbox``), or drops them so a retry reparses
the same reports rather than silently skipping them as duplicates.
``None`` (the default) keeps the immediate-write behavior.
Returns the report type so mailbox callers know which UID list to
append the source message's UID to.
"""
report_type = parsed_email["report_type"]
# Compare against parsed_email["report_type"] directly (not the
# report_type local above) in each branch so pyright's TypedDict
# discriminated-union narrowing applies to parsed_email["report"].
if parsed_email["report_type"] == "aggregate":
report_org = parsed_email["report"]["report_metadata"]["org_name"]
report_id = parsed_email["report"]["report_metadata"]["report_id"]
report_key = _aggregate_report_dedup_key(parsed_email["report"])
already_seen = report_key in seen_aggregate_report_ids or (
pending_aggregate_keys is not None and report_key in pending_aggregate_keys
)
if not already_seen:
if pending_aggregate_keys is None:
seen_aggregate_report_ids[report_key] = True
else:
pending_aggregate_keys.add(report_key)
aggregate_reports.append(parsed_email["report"])
else:
logger.debug(
"Skipping duplicate aggregate report from "
f"{sanitize_for_log(str(report_org))} "
f"with ID: {sanitize_for_log(str(report_id))}"
)
elif parsed_email["report_type"] == "failure":
failure_reports.append(parsed_email["report"])
elif parsed_email["report_type"] == "smtp_tls":
smtp_tls_reports.append(parsed_email["report"])
return report_type
def _fetch_mailbox_message(
connection: MailboxConnection, msg_uid: Any, test: bool
) -> tuple[int | str, str | ParserError]:
"""Fetch one message from ``connection`` by UID, casting the UID to the
type each backend's ``fetch_message`` expects.
Shared, unmodified, by the sequential and parallel branches of
``get_dmarc_reports_from_mailbox`` so both fetch identically.
Returns ``(message_id, msg_content)``; ``message_id`` is the
backend-appropriate id to use for later move/delete calls.
A backend that re-serializes the stored message while fetching it (the
Maildir backend of mailsuite before 2.3.3 called ``Message.as_string()``;
2.3.3 returns the stored bytes) can fail on the message's own content:
``email.errors.HeaderWriteError`` on a
control character in a header, ``RecursionError`` on deeply nested
multiparts. Those content errors are returned, not raised, as
``msg_content`` -- a ``ParserError`` the caller treats like a parse
failure, disposing of the message as invalid instead of aborting the
batch on every run (GHSA-qxwv-2vwj-v828).
The catch is deliberately narrow (``email.errors.MessageError`` and
``RecursionError``, not ``Exception``): transport and service errors
(``OSError``, IMAP and Microsoft Graph errors, ``httpx`` errors) say
nothing about the message and still propagate, because classifying a
transient network failure as invalid content would move a legitimate
report to the invalid folder.
"""
message_id: int | str
if isinstance(connection, IMAPConnection):
message_id = int(msg_uid)
else:
message_id = str(msg_uid) if not isinstance(msg_uid, str) else msg_uid
try:
if isinstance(connection, (MSGraphConnection, MaildirConnection)):
msg_content = connection.fetch_message(message_id, mark_read=not test)
else:
msg_content = connection.fetch_message(message_id)
except (email.errors.MessageError, RecursionError) as e:
return message_id, ParserError(
f"Unable to fetch message UID {message_id}: {type(e).__name__}: {e}"
)
return message_id, msg_content
def _dispose_invalid_message(
connection: MailboxConnection,
message_id: int | str,
delete: bool,
invalid_reports_folder: str,
) -> bool:
"""Delete or move an unparseable message, per ``delete``.
Callers pass their effective ``delete_invalid`` value as ``delete``.
Returns whether the message left the reports folder.
Shared, unmodified, by the sequential and parallel branches of
``get_dmarc_reports_from_mailbox`` so both dispose of invalid messages
identically.
A delete or move the backend rejects is logged and the message is left
in place, as the archival disposal of parsed reports does, rather than
aborting the rest of the batch. Besides transient backend errors, this
covers messages the backend cannot rewrite: the Maildir backend of
mailsuite before 2.3.3 moved a message by storing it again with
``mailbox.Maildir.add()``, which re-serializes it and raises
``RecursionError`` for deeply nested multiparts (and
``email.errors.HeaderWriteError`` for a control character in a header
where ``BytesGenerator`` verifies headers), so such a message stayed in
the inbox. mailsuite 2.3.3 renames the file instead
(GHSA-qxwv-2vwj-v828, GHSA-9jx3-p8wm-g8vw).
"""
try:
if delete:
logger.debug(f"Deleting message UID {message_id}")
if isinstance(connection, IMAPConnection):
connection.delete_message(int(message_id))
else:
connection.delete_message(str(message_id))
else:
logger.debug(f"Moving message UID {message_id} to {invalid_reports_folder}")
if isinstance(connection, IMAPConnection):
connection.move_message(int(message_id), invalid_reports_folder)
else:
connection.move_message(str(message_id), invalid_reports_folder)
except Exception as e:
action = "deleting" if delete else "moving"
logger.error(
f"Mailbox error: Error {action} invalid message UID {message_id}: "
f"{type(e).__name__}: {e}"
)
return False
return True
def _record_parse_attempt(
attempt_store: AttemptStore,
key: str,
message_id: int | str,
reports_folder: str,
delete_invalid: bool,
invalid_reports_folder: str,
test: bool,
) -> bool:
"""Record that a message is about to be parsed; return ``True`` when it
must be set aside instead.
A message whose earlier parse attempts never finished
``MAX_PARSE_ATTEMPTS`` times is taken to be what killed those runs: the
error is logged here, and the caller disposes of the message as invalid
without parsing it (or, under ``test``, leaves it in place). Such a
message is not recorded again, so a message left in place by a failed
move or by ``test`` keeps the same count on every later run.
"""
count = attempt_store.count(key)
if not attempt_store.enabled or count < MAX_PARSE_ATTEMPTS:
attempt_store.record(key)
return False
if test:
action = "Leaving it in place without parsing it (test mode)"
elif delete_invalid:
action = "Deleting it without parsing it"
else:
action = f"Moving it to {invalid_reports_folder} without parsing it"
logger.error(
f"Message UID {message_id} in {reports_folder} has {count} unfinished "
f"parse attempts; the limit is {MAX_PARSE_ATTEMPTS}. {action}"
)
return True
def get_dmarc_reports_from_mbox(
input_: str,
*,
nameservers: list[str] | None = None,
dns_timeout: float = DEFAULT_DNS_TIMEOUT,
dns_retries: int = DEFAULT_DNS_MAX_RETRIES,
strip_attachment_payloads: bool = False,
ip_db_path: str | None = None,
always_use_local_files: bool = False,
reverse_dns_map_path: str | None = None,
reverse_dns_map_url: str | None = None,
offline: bool = False,
normalize_timespan_threshold_hours: float = 24.0,
n_procs: int = 1,
parse_timeout: float = 0.0,
config: ParserConfig | None = None,
) -> ParsingResults:
"""Parses a mailbox in mbox format containing e-mails with attached
DMARC and SMTP TLS reports
Args:
input_ (str): A path to a mbox file
nameservers (list): A list of one or more nameservers to use
(Cloudflare's public DNS resolvers by default)
dns_timeout (float): Sets the DNS timeout in seconds
dns_retries (int): Number of times to retry DNS queries on timeout
or other transient errors
strip_attachment_payloads (bool): Remove attachment payloads from
failure report results
always_use_local_files (bool): Do not download files
reverse_dns_map_path (str): Path to a reverse DNS map file
reverse_dns_map_url (str): URL to a reverse DNS map file
ip_db_path (str): Path to a MMDB file from IPinfo, MaxMind, or DBIP
offline (bool): Do not make online queries for geolocation or DNS
normalize_timespan_threshold_hours (float): Normalize timespans beyond this
n_procs (int): Number of processes to use for parsing messages in
parallel. Message reading, deduplication, and result assembly
stay in the calling process; only parsing is parallelized. Not
part of ``config``; always applies.
parse_timeout (float): Stop parsing a message after this many
seconds and skip it with a warning, as an invalid message
(``0``, the default, means no limit). When set, messages are
parsed in worker processes even with ``n_procs`` 1. A negative
or non-finite value raises ``ValueError``. See
``parsedmarc.parallel.parallel_map()``. Not part of
``config``; always applies.
config (ParserConfig): a single object carrying all parsing and
enrichment options plus the caches; when provided, the
individual option keyword arguments listed above are ignored in
favor of the config's values.
Returns:
dict: Lists of ``aggregate_reports``, ``failure_reports``, and ``smtp_tls_reports``
"""
cfg = _resolve_config(
config,
offline=offline,
ip_db_path=ip_db_path,
always_use_local_files=always_use_local_files,
reverse_dns_map_path=reverse_dns_map_path,
reverse_dns_map_url=reverse_dns_map_url,
nameservers=nameservers,
dns_timeout=dns_timeout,
dns_retries=dns_retries,
strip_attachment_payloads=strip_attachment_payloads,
normalize_timespan_threshold_hours=normalize_timespan_threshold_hours,
)
aggregate_reports: list[AggregateReport] = []
failure_reports: list[FailureReport] = []
smtp_tls_reports: list[SMTPTLSReport] = []
try:
mbox = mailbox.mbox(input_)
message_keys = mbox.keys()
total_messages = len(message_keys)
logger.debug(f"Found {total_messages} messages in {input_}")
if parse_timeout or (n_procs > 1 and total_messages > 1):
from parsedmarc.parallel import (
_email_timeout_result,
_parse_report_email_job,
parallel_map,
)
func = functools.partial(_parse_report_email_job, config=cfg)
def _jobs():
for i in range(total_messages):
message_key = message_keys[i]
logger.info(f"Processing message {i + 1} of {total_messages}")
yield mbox.get_bytes(message_key)
for result in tqdm(
parallel_map(
func,
_jobs(),
n_procs,
parse_timeout=parse_timeout,
timeout_result=_email_timeout_result,
),
total=total_messages,
disable=None,
):
if isinstance(result, (InvalidDMARCReport, InvalidSMTPTLSReport)):
logger.warning(str(result))
elif isinstance(result, ParserError):
raise result
else:
_classify_parsed_email(
result,
aggregate_reports,
failure_reports,
smtp_tls_reports,
seen_aggregate_report_ids=cfg.seen_aggregate_report_ids,
)
else:
for i in tqdm(range(total_messages), disable=None):
message_key = message_keys[i]
logger.info(f"Processing message {i + 1} of {total_messages}")
msg_content = mbox.get_bytes(message_key)
try:
parsed_email = parse_report_email(msg_content, config=cfg)
_classify_parsed_email(
parsed_email,
aggregate_reports,
failure_reports,
smtp_tls_reports,
seen_aggregate_report_ids=cfg.seen_aggregate_report_ids,
)
except (InvalidDMARCReport, InvalidSMTPTLSReport) as error:
logger.warning(error.__str__())
except mailbox.NoSuchMailboxError:
raise InvalidDMARCReport(f"Mailbox {input_} does not exist")
return {
"aggregate_reports": aggregate_reports,
"failure_reports": failure_reports,
"smtp_tls_reports": smtp_tls_reports,
}
def _migrate_forensic_archive_folder(
connection: MailboxConnection, archive_folder: str
) -> None:
"""Consolidate a pre-rename ``<archive>/Forensic`` subfolder into
``<archive>/Failure``.
Before failure reports were renamed from "forensic" reports, they were
archived under ``<archive_folder>/Forensic``; they now go to
``<archive_folder>/Failure``. This best-effort, run-on-startup migration
moves any pre-existing legacy archive into the new location so reports
filed before and after the rename live in the same folder.
It is a no-op when there is no legacy ``Forensic`` folder (the common
case), and never raises: a mailbox that cannot be reorganized is logged
and skipped, consistent with the rest of parsedmarc's mailbox handling
(warn, don't crash). Uses the folder-management API added in mailsuite
2.1.0 (``folder_exists`` / ``rename_folder`` / ``merge_folders``).
"""
old_folder = f"{archive_folder}/Forensic"
new_folder = f"{archive_folder}/Failure"
try:
if not connection.folder_exists(old_folder):
return
if connection.folder_exists(new_folder):
# Both exist (e.g. a partial earlier migration, or a manually
# created Failure folder): move the legacy folder's messages into
# the new one and drop the now-empty legacy folder.
connection.merge_folders(old_folder, new_folder)
logger.info(f"Merged legacy archive folder {old_folder} into {new_folder}")
else:
connection.rename_folder(old_folder, new_folder)
logger.info(f"Renamed legacy archive folder {old_folder} to {new_folder}")
except Exception as error:
logger.warning(
f"Could not migrate legacy archive folder {old_folder} to {new_folder}: {error}"
)
def _ensure_folder(connection: MailboxConnection, folder: str) -> None:
"""Best-effort create ``folder`` if the backend says it is missing.
``get_dmarc_reports_from_mailbox()`` creates its destination folders up
front, but only when ``create_folders`` is set -- watch mode calls it
with ``create_folders=False``, and the ``Unsaved`` holding folder is
only created up front when a ``save_callback`` was supplied. This is the
defensive check immediately before a message is moved there.
Like ``_migrate_forensic_archive_folder``, it never raises: a backend
that cannot report on or create the folder is logged and skipped, and
the move that follows either succeeds anyway (the folder already
existed) or fails and is logged by its own error handler -- warn, don't
crash.
"""
try:
if not connection.folder_exists(folder):
connection.create_folder(folder)
except Exception as error:
logger.warning(f"Could not create folder {folder}: {error}")
def get_dmarc_reports_from_mailbox(
connection: MailboxConnection,
*,
reports_folder: str = "INBOX",
archive_folder: str = "Archive",
delete: bool = False,
delete_aggregate: bool | None = None,
delete_failure: bool | None = None,
delete_smtp_tls: bool | None = None,
delete_invalid: bool | None = None,
test: bool = False,
ip_db_path: str | None = None,
always_use_local_files: bool = False,
reverse_dns_map_path: str | None = None,
reverse_dns_map_url: str | None = None,
offline: bool = False,
nameservers: list[str] | None = None,
dns_timeout: float = DEFAULT_DNS_TIMEOUT,
dns_retries: int = DEFAULT_DNS_MAX_RETRIES,
strip_attachment_payloads: bool = False,
results: ParsingResults | None = None,
batch_size: int = 10,
since: datetime | date | str | None = None,
create_folders: bool = True,
normalize_timespan_threshold_hours: float = 24.0,
n_procs: int = 1,
parse_timeout: float = 0.0,
save_callback: Callable[[ParsingResults], bool | None] | None = None,
max_unsaved_retries: int = 2,
attempts_file: str | None = None,
config: ParserConfig | None = None,
) -> ParsingResults:
"""
Fetches and parses DMARC and SMTP TLS reports from a mailbox
Args:
connection: A Mailbox connection object
reports_folder (str): The folder where reports can be found
archive_folder (str): The folder to move processed mail to
delete (bool): Delete messages after processing them
delete_aggregate (bool | None): Delete aggregate report messages
after processing them, instead of moving them to the
``Aggregate`` archive subfolder; ``None`` (the default) inherits
the value of ``delete``
delete_failure (bool | None): Delete failure report messages after
processing them, instead of moving them to the ``Failure``
archive subfolder; ``None`` (the default) inherits the value of
``delete``
delete_smtp_tls (bool | None): Delete SMTP TLS report messages after
processing them, instead of moving them to the ``SMTP-TLS``
archive subfolder; ``None`` (the default) inherits the value of
``delete``
delete_invalid (bool | None): Delete unparseable messages, instead
of moving them to the ``Invalid`` archive subfolder where they
can be inspected for debugging; ``None`` (the default) inherits
the value of ``delete``
test (bool): Do not move or delete messages after processing them
ip_db_path (str): Path to a MMDB file from IPinfo, MaxMind, or DBIP
always_use_local_files (bool): Do not download files
reverse_dns_map_path (str): Path to a reverse DNS map file
reverse_dns_map_url (str): URL to a reverse DNS map file
offline (bool): Do not query online for geolocation or DNS
nameservers (list): A list of DNS nameservers to query
dns_timeout (float): Set the DNS query timeout
dns_retries (int): Number of times to retry DNS queries on timeout
or other transient errors
strip_attachment_payloads (bool): Remove attachment payloads from
failure report results
results (dict): Results from the previous run
batch_size (int): Number of messages to read and process before saving
(use 0 for no limit)
since: Search for messages since a certain time
(units - {"m":"minutes", "h":"hours", "d":"days", "w":"weeks"})
create_folders (bool): Whether to create the destination folders
(not used in watch)
normalize_timespan_threshold_hours (float): Normalize timespans beyond this
n_procs (int): Number of processes to use for parsing messages in
parallel. Fetching, archiving, and deduplication remain
sequential in the calling process; only parsing is
parallelized. With ``n_procs > 1``, invalid-message disposition
happens after the parsing phase completes, rather than
interleaved message-by-message as it is when ``n_procs`` is 1.
Not part of ``config``; always applies.
parse_timeout (float): Stop parsing a message after this many
seconds and move it to the ``Invalid`` archive subfolder, as an
invalid message (``0``, the default, means no limit). When set,
messages are parsed in worker processes even with ``n_procs``
1, so disposition of invalid messages happens after the parsing
phase as it does with ``n_procs > 1``. A negative or non-finite
value raises ``ValueError``. See
``parsedmarc.parallel.parallel_map()``. Not part of
``config``; always applies.
save_callback: An optional callable invoked once per fetched batch
with a ``ParsingResults`` dict holding only that batch's newly
parsed reports, after parsing but before any of the batch's
messages are deleted or moved out of ``reports_folder``. It
tells this function whether the batch was actually persisted:
* Returning ``False`` means "not saved": the batch's messages
are left in ``reports_folder`` for retry on the next run (or
moved to ``{archive_folder}/Unsaved`` once they have failed
``max_unsaved_retries`` retries -- see below), and the
aggregate-report dedup cache is not updated for that batch, so
a retry reparses the same reports instead of skipping them as
duplicates.
* Raising counts as "not saved" too: the same bookkeeping runs
(the batch's messages are held back or moved to ``Unsaved`` at
the cap, and the failed attempt counts toward
``max_unsaved_retries``), and the exception is then re-raised
to the caller.
* Any other return value, including ``None``, commits the batch:
the dedup cache is updated and the messages are deleted or
archived exactly as they are with no callback.
``None`` (the default) commits every batch, preserving the prior
behavior. The callback is still invoked when ``test`` is
``True``, so a test run exercises the full pipeline, but neither
the mailbox nor the retry counters are touched regardless of
what it returns.
max_unsaved_retries (int): How many times a message may be *retried*
after ``save_callback`` first reported its batch unsaved, before
it is moved to the ``Unsaved`` archive subfolder instead of
being retried again (default 2, i.e. the initial attempt plus
two retries -- at most three deliveries to any output
destination that does not deduplicate). ``0`` moves a message on
the first failed save; negative values raise ``ValueError``.
Messages moved to ``Unsaved`` are never deleted, whatever the
``delete`` options say; recover them by fixing the output
destination and moving them back into ``reports_folder``.
Counts are kept in memory, per message and
per process, are reset by a successful save, and are only kept
when a ``save_callback`` is supplied -- so the cap applies
across repeated calls within one process (``watch_inbox()``'s
checks), not across separate one-shot processes, each of which
starts a message's count over. Mailbox orchestration, so like
``batch_size`` it is not part of ``config``.
attempts_file (str | None): A JSON file in which to count, per
message, the parse attempts that started but never finished,
because the process died while parsing (the out-of-memory
killer, a container memory cap). An attempt is recorded before
a message is parsed and cleared as soon as the parse returns a
report or raises ``ParserError``, so a failed save is not
counted. A message that has died ``MAX_PARSE_ATTEMPTS`` (2)
times is logged at error level and disposed of as invalid
(moved to ``Invalid``, or deleted under ``delete_invalid``)
without being parsed again. With ``n_procs > 1`` or
``parse_timeout`` set, a message that was being parsed when an
earlier run died is parsed on its own after the others (in this process, or with
``parse_timeout`` in a worker pool of its own, so the timeout
still applies), so the messages parsed beside it are not blamed
for it. ``None`` (the default) keeps
no record. Mailbox orchestration, not part of ``config``.
config (ParserConfig): a single object carrying all parsing and
enrichment options plus the caches; when provided, it replaces
the individual parsing and enrichment option keyword arguments
listed above (DNS, GeoIP, offline mode, attachment payload
stripping, timespan normalization), whose values are then
ignored. The remaining keyword arguments control mailbox
handling and orchestration rather than parsing (the folder
names, the ``delete`` options, ``test``, ``since``,
``batch_size``, ``save_callback``, ``max_unsaved_retries``,
``attempts_file``); they are not part of ``config`` and always
apply.
Returns:
dict: Lists of ``aggregate_reports``, ``failure_reports``, and ``smtp_tls_reports``
"""
# Each per-report-type flag inherits the overall ``delete`` value when it
# is left unset (``None``). Resolve once, up front: every decision below
# reads only these four resolved flags. The raw ``delete`` parameter is
# still forwarded verbatim to the recursive self-call at the end of this
# function, where it is inert because all four resolved flags accompany
# it.
delete_aggregate = delete if delete_aggregate is None else delete_aggregate
delete_failure = delete if delete_failure is None else delete_failure
delete_smtp_tls = delete if delete_smtp_tls is None else delete_smtp_tls
delete_invalid = delete if delete_invalid is None else delete_invalid
if test and (
delete_aggregate or delete_failure or delete_smtp_tls or delete_invalid
):
raise ValueError("delete options and test are mutually exclusive")
if max_unsaved_retries < 0:
raise ValueError("max_unsaved_retries must be >= 0")
from parsedmarc.parallel import _check_parse_timeout
_check_parse_timeout(parse_timeout)
if connection is None:
raise ValueError("Must supply a connection")
attempt_store = AttemptStore(attempts_file) if attempts_file else None
cfg = _resolve_config(
config,
offline=offline,
ip_db_path=ip_db_path,
always_use_local_files=always_use_local_files,
reverse_dns_map_path=reverse_dns_map_path,
reverse_dns_map_url=reverse_dns_map_url,
nameservers=nameservers,
dns_timeout=dns_timeout,
dns_retries=dns_retries,
strip_attachment_payloads=strip_attachment_payloads,
normalize_timespan_threshold_hours=normalize_timespan_threshold_hours,
)
# current_time useful to fetch_messages later in the program
current_time: datetime | date | str | None = None
aggregate_reports: list[AggregateReport] = []
failure_reports: list[FailureReport] = []
smtp_tls_reports: list[SMTPTLSReport] = []
# This call's own reports, kept separate from the accumulated lists
# above (which carry earlier batches' reports in via ``results``) so the
# save callback is handed only what this batch parsed.
batch_aggregate_reports: list[AggregateReport] = []
batch_failure_reports: list[FailureReport] = []
batch_smtp_tls_reports: list[SMTPTLSReport] = []
# Aggregate dedup keys are staged here and only written to the shared
# cache once the batch is known to be saved, so an unsaved batch (or a
# mid-batch crash) leaves the cache clean and the reports are reparsed
# on the retry instead of being dropped as duplicates.
pending_aggregate_keys: set[str] = set()
aggregate_report_msg_uids = []
failure_report_msg_uids = []
smtp_tls_msg_uids = []
aggregate_reports_folder = f"{archive_folder}/Aggregate"
failure_reports_folder = f"{archive_folder}/Failure"
smtp_tls_reports_folder = f"{archive_folder}/SMTP-TLS"
invalid_reports_folder = f"{archive_folder}/Invalid"
unsaved_reports_folder = f"{archive_folder}/Unsaved"
if results:
aggregate_reports = results["aggregate_reports"].copy()
failure_reports = results["failure_reports"].copy()
smtp_tls_reports = results["smtp_tls_reports"].copy()
if not test and create_folders:
_migrate_forensic_archive_folder(connection, archive_folder)
connection.create_folder(archive_folder)
connection.create_folder(aggregate_reports_folder)
connection.create_folder(failure_reports_folder)
connection.create_folder(smtp_tls_reports_folder)
connection.create_folder(invalid_reports_folder)
if save_callback is not None:
# Only reachable when a callback can report a batch unsaved;
# without one no message is ever held back, so the folder would
# sit empty in every mailbox.
connection.create_folder(unsaved_reports_folder)
if since and isinstance(since, str):
_since = 1440 # default one day
if re.match(r"\d+[mhdw]$", since):
s = re.split(r"(\d+)", since)
if s[2] == "m":
_since = int(s[1])
elif s[2] == "h":
_since = int(s[1]) * 60
elif s[2] == "d":
_since = int(s[1]) * 60 * 24
elif s[2] == "w":
_since = int(s[1]) * 60 * 24 * 7
else:
logger.warning(
f"Incorrect format for 'since' option. Provided value: {since}, "
"expected values: (5m|3h|2d|1w). Ignoring option, fetching "
"messages for last 24hrs. SMTP does not support a time or "
"timezone in since. See "
"https://www.rfc-editor.org/rfc/rfc3501#page-52"
)
if isinstance(connection, IMAPConnection):
logger.debug(
"Only days and weeks values in 'since' option are considered "
"for IMAP connections. Examples: 2d or 1w"
)
since = (datetime.now(timezone.utc) - timedelta(minutes=_since)).strftime(
"%d-%b-%Y"
)
current_time = datetime.now(timezone.utc).strftime("%d-%b-%Y")
elif isinstance(connection, MSGraphConnection):
since = (datetime.now(timezone.utc) - timedelta(minutes=_since)).isoformat()
current_time = datetime.now(timezone.utc).isoformat()
elif isinstance(connection, GmailConnection):
since = (datetime.now(timezone.utc) - timedelta(minutes=_since)).strftime(
"%s"
)
current_time = datetime.now(timezone.utc).strftime("%s")
else:
pass
messages = connection.fetch_messages(
reports_folder, batch_size=batch_size, since=since
)
total_messages = len(messages)
logger.debug(f"Found {len(messages)} messages in {reports_folder}")
if batch_size and not since:
message_limit = min(total_messages, batch_size)
else:
message_limit = total_messages
logger.debug(f"Processing {message_limit} messages")
def _parse_finished(key: str | None) -> None:
# A parse that returned a result or raised ParserError did not
# crash the process, so its attempt is cleared before anything that
# can fail for other reasons (the save, the move) runs.
if attempt_store is not None and key is not None:
attempt_store.clear(key)
def _parse_in_process(
message_id: int | str, msg_content: str | ParserError
) -> None:
key: str | None = None
try:
if isinstance(msg_content, ParserError):
raise msg_content
if attempt_store is not None:
key = attempt_key(reports_folder, message_id, msg_content)
if _record_parse_attempt(
attempt_store,
key,
message_id,
reports_folder,
delete_invalid,
invalid_reports_folder,
test,
):
# The entry is kept until the message has left the
# folder, so a failed move is retried on the next run.
if not test and _dispose_invalid_message(
connection, message_id, delete_invalid, invalid_reports_folder
):
attempt_store.clear(key)
return
parsed_email = parse_report_email(
msg_content,
config=cfg,
keep_alive=connection.keepalive,
)
_parse_finished(key)
report_type = _classify_parsed_email(
parsed_email,
batch_aggregate_reports,
batch_failure_reports,
batch_smtp_tls_reports,
seen_aggregate_report_ids=cfg.seen_aggregate_report_ids,
pending_aggregate_keys=pending_aggregate_keys,
)
if report_type == "aggregate":
aggregate_report_msg_uids.append(message_id)
elif report_type == "failure":
failure_report_msg_uids.append(message_id)
elif report_type == "smtp_tls":
smtp_tls_msg_uids.append(message_id)
except ParserError as error:
_parse_finished(key)
logger.warning(error.__str__())
if not test:
_dispose_invalid_message(
connection, message_id, delete_invalid, invalid_reports_folder
)
if parse_timeout or (n_procs > 1 and message_limit > 1):
from parsedmarc.parallel import (
_email_timeout_result,
_parse_report_email_job,
parallel_map,
)
# The config's caches (ip_address_cache, seen_aggregate_report_ids,
# reverse_dns_map) never cross the process boundary --
# ParserConfig.__getstate__ drops them, and each worker accumulates
# its own via the module defaults it rebinds to on unpickling (see
# ParserConfig.__setstate__). keep_alive is a bound method of the
# live connection object and is not a ParserConfig field, so it is
# never submitted to the pool either; the heartbeat passed to
# parallel_map below keeps the connection alive instead.
func = functools.partial(_parse_report_email_job, config=cfg)
# parallel_map yields results in submission order, so the oldest
# queued id always belongs to the next yielded result; popping as
# results arrive keeps this queue no larger than the in-flight
# submission window. Each id travels with its attempts-file key
# (None when there is no attempts file, or no attempt to clear).
fetched_ids: deque[tuple[int | str, str | None]] = deque()
invalid_msg_ids: list[tuple[int | str, str | None]] = []
# Messages that were being parsed when an earlier run died. That
# run had up to a whole submission window in flight (with n_procs
# above 1, or with parse_timeout set, which takes this path even
# with one process), so all of them are suspects; parsing each alone, after the others, lets the next
# crash count against the one message that caused it.
suspects: list[tuple[int | str, str]] = []
def _jobs():
for i in range(message_limit):
msg_uid = messages[i]
logger.debug(
f"Processing message {i + 1} of {message_limit}: UID {msg_uid}"
)
message_id, msg_content = _fetch_mailbox_message(
connection, msg_uid, test
)
if isinstance(msg_content, ParserError):
# Never submitted, so never yielded and never queued in
# fetched_ids; the post-loop disposal handles it.
logger.warning(str(msg_content))
invalid_msg_ids.append((message_id, None))
continue
key: str | None = None
if attempt_store is not None:
key = attempt_key(reports_folder, message_id, msg_content)
if 0 < attempt_store.count(key) < MAX_PARSE_ATTEMPTS:
logger.warning(
f"Message UID {message_id} in {reports_folder} "
"was being parsed when parsedmarc last stopped "
"without finishing; parsing it on its own after "
"the rest of the batch"
)
suspects.append((message_id, msg_content))
continue
if _record_parse_attempt(
attempt_store,
key,
message_id,
reports_folder,
delete_invalid,
invalid_reports_folder,
test,
):
invalid_msg_ids.append((message_id, key))
continue
fetched_ids.append((message_id, key))
yield msg_content
def _take_result(
message_id: int | str,
key: str | None,
result: ParsedReport | ParserError,
) -> None:
_parse_finished(key)
if isinstance(result, ParserError):
logger.warning(str(result))
invalid_msg_ids.append((message_id, None))
else:
report_type = _classify_parsed_email(
result,
batch_aggregate_reports,
batch_failure_reports,
batch_smtp_tls_reports,
seen_aggregate_report_ids=cfg.seen_aggregate_report_ids,
pending_aggregate_keys=pending_aggregate_keys,
)
if report_type == "aggregate":
aggregate_report_msg_uids.append(message_id)
elif report_type == "failure":
failure_report_msg_uids.append(message_id)
elif report_type == "smtp_tls":
smtp_tls_msg_uids.append(message_id)
for result in parallel_map(
func,
_jobs(),
n_procs,
heartbeat=connection.keepalive,
parse_timeout=parse_timeout,
timeout_result=_email_timeout_result,
):
message_id, key = fetched_ids.popleft()
_take_result(message_id, key, result)
for message_id, msg_content in suspects:
if not parse_timeout or attempt_store is None:
_parse_in_process(message_id, msg_content)
continue
# Only a worker process can be stopped when the timeout expires,
# so each suspect gets a pool of its own: the timeout still
# bounds it, and a crash is still charged to it alone. A suspect
# is below MAX_PARSE_ATTEMPTS, so this record cannot reach the
# cap.
key = attempt_key(reports_folder, message_id, msg_content)
attempt_store.record(key)
for result in parallel_map(
func,
[msg_content],
1,
heartbeat=connection.keepalive,
parse_timeout=parse_timeout,
timeout_result=_email_timeout_result,
):
_take_result(message_id, key, result)
if not test:
for invalid_message_id, key in invalid_msg_ids:
# A set-aside message's entry is cleared only once it has
# left the folder, as in _parse_in_process.
if _dispose_invalid_message(
connection,
invalid_message_id,
delete_invalid,
invalid_reports_folder,
):
_parse_finished(key)
else:
for i in range(message_limit):
msg_uid = messages[i]
logger.debug(
f"Processing message {i + 1} of {message_limit}: UID {msg_uid}"
)
message_id, msg_content = _fetch_mailbox_message(connection, msg_uid, test)
_parse_in_process(message_id, msg_content)
# Ask the caller whether this batch actually made it to its output
# destinations before touching a single message. A callback that says
# otherwise (or raises) means the reports exist nowhere else yet, so no
# message may be archived or deleted -- only retained for retry, or moved
# intact to the Unsaved folder once retries run out (#242).
batch_results: ParsingResults = {
"aggregate_reports": batch_aggregate_reports,
"failure_reports": batch_failure_reports,
"smtp_tls_reports": batch_smtp_tls_reports,
}
persisted = True
callback_error: Exception | None = None
if save_callback is not None:
try:
persisted = save_callback(batch_results) is not False
except Exception as error:
# A raising callback is a failed save too: the failure
# bookkeeping below still runs (count the attempt, hold or shelve
# the messages) before the exception is re-raised, so a callback
# that always raises -- e.g. the CLI's under
# ``fail_on_output_error`` -- is still bounded by
# ``max_unsaved_retries`` even when the caller's watch loop
# swallows the exception and keeps checking, as mailsuite's IMAP
# and Maildir backends do.
persisted = False
callback_error = error
batch_msg_uids = (
aggregate_report_msg_uids + failure_report_msg_uids + smtp_tls_msg_uids
)
if persisted:
# Committing: the reports are safely stored elsewhere, so record
# their dedup keys and forget any earlier failures for these
# messages.
for report_key in pending_aggregate_keys:
cfg.seen_aggregate_report_ids[report_key] = True
if not test:
for msg_uid in batch_msg_uids:
_FAILED_SAVE_ATTEMPTS.pop((reports_folder, str(msg_uid)), None)
elif not test:
# Held back for retry. Messages that have now failed the initial
# attempt plus ``max_unsaved_retries`` retries stop being retried and
# move to the Unsaved folder, bounding duplicate delivery to output
# destinations that do not deduplicate; the rest stay put.
retained_uids: list[int | str] = []
over_cap_uids: list[int | str] = []
highest_attempt = 0
for msg_uid in batch_msg_uids:
attempts_key = (reports_folder, str(msg_uid))
attempts = _FAILED_SAVE_ATTEMPTS.get(attempts_key, 0) + 1
_FAILED_SAVE_ATTEMPTS[attempts_key] = attempts
if attempts > max_unsaved_retries:
over_cap_uids.append(msg_uid)
else:
retained_uids.append(msg_uid)
highest_attempt = max(highest_attempt, attempts)
if retained_uids:
logger.error(
f"Reports were not saved: leaving {len(retained_uids)} "
f"message(s) in {reports_folder} to retry on the next run "
f"or check (failed attempt {highest_attempt} of "
f"{max_unsaved_retries + 1})"
)
if over_cap_uids:
logger.error(
f"Reports were not saved after {max_unsaved_retries + 1} "
f"attempt(s): moving {len(over_cap_uids)} message(s) from "
f"{reports_folder} to {unsaved_reports_folder} instead of "
"retrying them further. They are never deleted -- fix the "
f"output destination, then move them back to {reports_folder}"
)
_ensure_folder(connection, unsaved_reports_folder)
for msg_uid in over_cap_uids:
try:
connection.move_message(msg_uid, unsaved_reports_folder)
except Exception as e:
e = f"Error moving message UID {msg_uid}: {e}"
logger.error(f"Mailbox error: {e}")
else:
# Drop the counter only once the message is actually out
# of the retry loop. Clearing it before a failed move
# would hand the still-in-place message a fresh set of
# under-cap retries (and deliveries); keeping it means
# the next failed save classifies the message over-cap
# again and re-attempts the move instead.
_FAILED_SAVE_ATTEMPTS.pop((reports_folder, str(msg_uid)), None)
if callback_error is not None:
raise callback_error
aggregate_reports += batch_aggregate_reports
failure_reports += batch_failure_reports
smtp_tls_reports += batch_smtp_tls_reports
if persisted and not test:
# Each report type is disposed of according to its own effective
# delete flag: deleted outright, or moved to its archive subfolder.
for msg_uids, delete_type, destination_folder, label in (
(
aggregate_report_msg_uids,
delete_aggregate,
aggregate_reports_folder,
"aggregate report",
),
(
failure_report_msg_uids,
delete_failure,
failure_reports_folder,
"failure report",
),
(
smtp_tls_msg_uids,
delete_smtp_tls,
smtp_tls_reports_folder,
"SMTP TLS report",
),
):
number_of_msgs = len(msg_uids)
if number_of_msgs == 0:
continue
if not delete_type:
message = f"Moving {label} messages from"
logger.debug(f"{message} {reports_folder} to {destination_folder}")
for i in range(number_of_msgs):
msg_uid = msg_uids[i]
if delete_type:
logger.debug(
f"Deleting message {i + 1} of {number_of_msgs}: UID {msg_uid}"
)
try:
connection.delete_message(msg_uid)
except Exception as e:
message = "Error deleting message UID"
e = f"{message} {msg_uid}: {e}"
logger.error(f"Mailbox error: {e}")
else:
message = "Moving message"
logger.debug(
f"{message} {i + 1} of {number_of_msgs}: UID {msg_uid}"
)
try:
connection.move_message(msg_uid, destination_folder)
except Exception as e:
e = f"Error moving message UID {msg_uid}: {e}"
logger.error(f"Mailbox error: {e}")
results = {
"aggregate_reports": aggregate_reports,
"failure_reports": failure_reports,
"smtp_tls_reports": smtp_tls_reports,
}
# An unsaved batch left its messages in ``reports_folder``, so the
# re-check below would find them again and immediately reprocess the
# very messages that just failed -- burning through the retry cap in one
# call. Skip it and let the next run retry them.
if persisted and not test and not batch_size:
if current_time:
total_messages = len(
connection.fetch_messages(reports_folder, since=current_time)
)
else:
total_messages = len(connection.fetch_messages(reports_folder))
else:
total_messages = 0
if total_messages > 0:
# Process emails that came in during the last run
results = get_dmarc_reports_from_mailbox(
connection=connection,
reports_folder=reports_folder,
archive_folder=archive_folder,
delete=delete,
delete_aggregate=delete_aggregate,
delete_failure=delete_failure,
delete_smtp_tls=delete_smtp_tls,
delete_invalid=delete_invalid,
test=test,
results=results,
since=current_time,
n_procs=n_procs,
parse_timeout=parse_timeout,
save_callback=save_callback,
max_unsaved_retries=max_unsaved_retries,
attempts_file=attempts_file,
config=cfg,
)
return results
def watch_inbox(
mailbox_connection: MailboxConnection,
callback: Callable,
*,
reports_folder: str = "INBOX",
archive_folder: str = "Archive",
delete: bool = False,
delete_aggregate: bool | None = None,
delete_failure: bool | None = None,
delete_smtp_tls: bool | None = None,
delete_invalid: bool | None = None,
test: bool = False,
check_timeout: int = 30,
ip_db_path: str | None = None,
always_use_local_files: bool = False,
reverse_dns_map_path: str | None = None,
reverse_dns_map_url: str | None = None,
offline: bool = False,
nameservers: list[str] | None = None,
dns_timeout: float = DEFAULT_DNS_TIMEOUT,
dns_retries: int = DEFAULT_DNS_MAX_RETRIES,
strip_attachment_payloads: bool = False,
batch_size: int = 10,
since: datetime | date | str | None = None,
normalize_timespan_threshold_hours: float = 24.0,
config_reloading: Callable | None = None,
n_procs: int = 1,
parse_timeout: float = 0.0,
max_unsaved_retries: int = 2,
attempts_file: str | None = None,
config: ParserConfig | None = None,
):
"""
Watches the mailbox for new messages and sends the results to a
callback function
Args:
mailbox_connection: The mailbox connection object
callback: The callback function to receive the parsing results.
Passed straight through to ``get_dmarc_reports_from_mailbox()``
as its ``save_callback``, so it now runs once per fetched batch,
with only that batch's reports, *before* those messages are
deleted or moved out of ``reports_folder`` -- rather than once
afterward with the whole check's accumulated results. Returning
``False`` reports the batch as unsaved, leaving its messages in
place to be retried on the next check instead of archived or
deleted (see ``save_callback`` and ``max_unsaved_retries`` on
``get_dmarc_reports_from_mailbox()``). Raising counts as an
unsaved batch too -- same retention and retry cap -- before the
exception reaches the mailbox backend's watch loop; what happens
then is backend-specific: the Microsoft Graph and Gmail backends
let it propagate and end the watch, while mailsuite's IMAP and
Maildir watch loops log it and keep checking.
reports_folder (str): The folder where reports can be found
archive_folder (str): The folder to move processed mail to
delete (bool): Delete messages after processing them
delete_aggregate (bool | None): Delete aggregate report messages
after processing them, instead of moving them to the
``Aggregate`` archive subfolder; ``None`` (the default) inherits
the value of ``delete``
delete_failure (bool | None): Delete failure report messages after
processing them, instead of moving them to the ``Failure``
archive subfolder; ``None`` (the default) inherits the value of
``delete``
delete_smtp_tls (bool | None): Delete SMTP TLS report messages after
processing them, instead of moving them to the ``SMTP-TLS``
archive subfolder; ``None`` (the default) inherits the value of
``delete``
delete_invalid (bool | None): Delete unparseable messages, instead
of moving them to the ``Invalid`` archive subfolder where they
can be inspected for debugging; ``None`` (the default) inherits
the value of ``delete``
test (bool): Do not move or delete messages after processing them
check_timeout (int): Number of seconds to wait for an IMAP IDLE
response or the number of seconds until the next mail check
ip_db_path (str): Path to a MMDB file from IPinfo, MaxMind, or DBIP
always_use_local_files (bool): Do not download files
reverse_dns_map_path (str): Path to a reverse DNS map file
reverse_dns_map_url (str): URL to a reverse DNS map file
offline (bool): Do not query online for geolocation or DNS
nameservers (list): A list of one or more nameservers to use
(Cloudflare's public DNS resolvers by default)
dns_timeout (float): Set the DNS query timeout
dns_retries (int): Number of times to retry DNS queries on timeout
or other transient errors
strip_attachment_payloads (bool): Remove attachment payloads from
failure report results
batch_size (int): Number of messages to read and process before saving
since: Search for messages since a certain time
normalize_timespan_threshold_hours (float): Normalize timespans beyond this
config_reloading: Optional callable that returns True when a config
reload (or shutdown) has been requested (e.g. via SIGHUP/SIGTERM).
Polled by the mailbox backend between checks, including the IMAP
IDLE loop, so the watcher exits cleanly at a safe boundary.
n_procs (int): Number of processes to use for parsing messages in
parallel. Passed through to ``get_dmarc_reports_from_mailbox``
on each check. Not part of ``config``; always applies.
parse_timeout (float): Stop parsing a message after this many
seconds and treat it as invalid (``0``, the default, means no
limit). Passed through to ``get_dmarc_reports_from_mailbox``
on each check; a negative or non-finite value raises
``ValueError`` before the watch starts. Not part of ``config``;
always applies.
max_unsaved_retries (int): How many times a message may be retried
after ``callback`` first reported its batch unsaved, before it is
moved to the ``Unsaved`` archive subfolder (default 2). Passed
through to ``get_dmarc_reports_from_mailbox``, where it is
documented in full. Not part of ``config``; always applies.
attempts_file (str | None): A JSON file counting the parse
attempts that never finished, so a message that keeps killing
the process is set aside as invalid (default ``None``: no
record). Passed through to ``get_dmarc_reports_from_mailbox``,
where it is documented in full. Not part of ``config``; always
applies.
config (ParserConfig): a single object carrying all parsing and
enrichment options plus the caches; when provided, it replaces
the individual parsing and enrichment option keyword arguments
listed above (DNS, GeoIP, offline mode, attachment payload
stripping, timespan normalization), whose values are then
ignored. The remaining keyword arguments control mailbox
handling and orchestration rather than parsing (the folder
names, the ``delete`` options, ``test``, ``since``,
``batch_size``, ``max_unsaved_retries``, ``attempts_file``);
they are not part of ``config`` and always apply.
"""
# Validate before the watch loop starts: raised inside a check, this
# would be swallowed and endlessly retried by the IMAP and Maildir
# backends' per-check exception handling instead of surfacing.
if max_unsaved_retries < 0:
raise ValueError("max_unsaved_retries must be >= 0")
from parsedmarc.parallel import _check_parse_timeout
_check_parse_timeout(parse_timeout)
cfg = _resolve_config(
config,
offline=offline,
ip_db_path=ip_db_path,
always_use_local_files=always_use_local_files,
reverse_dns_map_path=reverse_dns_map_path,
reverse_dns_map_url=reverse_dns_map_url,
nameservers=nameservers,
dns_timeout=dns_timeout,
dns_retries=dns_retries,
strip_attachment_payloads=strip_attachment_payloads,
normalize_timespan_threshold_hours=normalize_timespan_threshold_hours,
)
def check_callback(connection):
get_dmarc_reports_from_mailbox(
connection=connection,
reports_folder=reports_folder,
archive_folder=archive_folder,
delete=delete,
delete_aggregate=delete_aggregate,
delete_failure=delete_failure,
delete_smtp_tls=delete_smtp_tls,
delete_invalid=delete_invalid,
test=test,
batch_size=batch_size,
since=since,
create_folders=False,
n_procs=n_procs,
parse_timeout=parse_timeout,
save_callback=callback,
max_unsaved_retries=max_unsaved_retries,
attempts_file=attempts_file,
config=cfg,
)
watch_kwargs: dict = {
"check_callback": check_callback,
"check_timeout": check_timeout,
}
if config_reloading is not None:
watch_kwargs["config_reloading"] = config_reloading
mailbox_connection.watch(**watch_kwargs)
def append_json(
filename: str,
reports: Sequence[AggregateReport]
| Sequence[FailureReport]
| Sequence[SMTPTLSReport],
) -> None:
"""Append ``reports`` to a JSON array on disk, creating the file
if needed.
Reads the existing array (if the file exists and parses cleanly),
merges the new reports onto the end, and rewrites the file as a
single valid JSON array. An earlier version of this used an
``open(..., "a+")`` + ``seek()`` + overwrite pattern, but Python's
documentation is explicit that on POSIX, ``a`` / ``a+`` writes
*always* go to EOF regardless of seek position — so the second
call onto an existing file produced ``[...],\\n[...]``-style
corrupted output. Read-merge-write is the only way to get a valid
JSON array out of repeated appends.
"""
if len(reports) == 0:
# Don't create an empty-array file for an empty input; if a
# file already exists, leave it alone.
return
existing: list = []
if os.path.isfile(filename) and os.path.getsize(filename) > 0:
try:
with open(filename, "r", encoding="utf-8") as f:
loaded = json.loads(f.read())
if isinstance(loaded, list):
existing = loaded
except (json.JSONDecodeError, OSError):
# Corrupted or unreadable: overwrite cleanly rather than
# silently fail to record.
existing = []
merged = existing + list(reports)
with open(filename, "w", newline="\n", encoding="utf-8") as output:
json.dump(merged, output, ensure_ascii=False, indent=2)
def append_csv(filename: str, csv: str) -> None:
with open(filename, "a+", newline="\n", encoding="utf-8") as output:
if output.seek(0, os.SEEK_END) != 0:
# strip the headers from the CSV
_headers, csv = csv.split("\n", 1)
if len(csv) == 0:
# not appending anything, don't do any dance to
# append it correctly
return
output.write(csv)
def save_output(
results: ParsingResults,
*,
output_directory: str = "output",
aggregate_json_filename: str = "aggregate.json",
failure_json_filename: str = "failure.json",
smtp_tls_json_filename: str = "smtp_tls.json",
aggregate_csv_filename: str = "aggregate.csv",
failure_csv_filename: str = "failure.csv",
smtp_tls_csv_filename: str = "smtp_tls.csv",
):
"""
Save report data in the given directory
The message sample of each failure report is written to a ``samples``
subdirectory, named after the sample's own ``subject`` header run
through :func:`parsedmarc.utils.get_filename_safe_string`, falling back
to ``sample`` when sanitizing leaves nothing. Since a subject arrives
from an untrusted sender, sanitizing happens here, at write time; the
``filename_safe_subject`` key a caller may supply alongside it is not
trusted and not used. Names that collide get a ``(1)``, ``(2)``, …
suffix.
Args:
results: Parsing results
output_directory (str): The path to the directory to save in
aggregate_json_filename (str): Filename for the aggregate JSON file
failure_json_filename (str): Filename for the failure JSON file
smtp_tls_json_filename (str): Filename for the SMTP TLS JSON file
aggregate_csv_filename (str): Filename for the aggregate CSV file
failure_csv_filename (str): Filename for the failure CSV file
smtp_tls_csv_filename (str): Filename for the SMTP TLS CSV file
"""
aggregate_reports = results["aggregate_reports"]
failure_reports = results["failure_reports"]
smtp_tls_reports = results["smtp_tls_reports"]
output_directory = os.path.expanduser(output_directory)
if os.path.exists(output_directory):
if not os.path.isdir(output_directory):
raise ValueError(f"{output_directory} is not a directory")
else:
os.makedirs(output_directory)
append_json(
os.path.join(output_directory, aggregate_json_filename), aggregate_reports
)
append_csv(
os.path.join(output_directory, aggregate_csv_filename),
parsed_aggregate_reports_to_csv(aggregate_reports),
)
append_json(os.path.join(output_directory, failure_json_filename), failure_reports)
append_csv(
os.path.join(output_directory, failure_csv_filename),
parsed_failure_reports_to_csv(failure_reports),
)
append_json(
os.path.join(output_directory, smtp_tls_json_filename), smtp_tls_reports
)
append_csv(
os.path.join(output_directory, smtp_tls_csv_filename),
parsed_smtp_tls_reports_to_csv(smtp_tls_reports),
)
samples_directory = os.path.join(output_directory, "samples")
if not os.path.exists(samples_directory):
os.makedirs(samples_directory)
sample_filenames = []
for failure_report in failure_reports:
sample = failure_report["sample"]
message_count = 0
parsed_sample = failure_report["parsed_sample"]
subject = get_filename_safe_string(parsed_sample.get("subject")) or "sample"
filename = subject
while filename in sample_filenames:
message_count += 1
filename = f"{subject} ({message_count})"
sample_filenames.append(filename)
filename = f"{filename}.eml"
path = os.path.join(samples_directory, filename)
with open(path, "w", newline="\n", encoding="utf-8") as sample_file:
sample_file.write(sample)
def get_report_zip(results: ParsingResults) -> bytes:
"""
Creates a zip file of parsed report output
Args:
results: The parsed results
Returns:
bytes: zip file bytes
"""
def add_subdir(root_path, subdir):
subdir_path = os.path.join(root_path, subdir)
for subdir_root, subdir_dirs, subdir_files in os.walk(subdir_path):
for subdir_file in subdir_files:
subdir_file_path = os.path.join(root_path, subdir, subdir_file)
if os.path.isfile(subdir_file_path):
rel_path = os.path.relpath(subdir_root, subdir_file_path)
subdir_arc_name = os.path.join(rel_path, subdir_file)
zip_file.write(subdir_file_path, subdir_arc_name)
for subdir in subdir_dirs:
add_subdir(subdir_path, subdir)
storage = BytesIO()
tmp_dir = tempfile.mkdtemp()
try:
save_output(results, output_directory=tmp_dir)
with zipfile.ZipFile(storage, "w", zipfile.ZIP_DEFLATED) as zip_file:
for root, dirs, files in os.walk(tmp_dir):
for file in files:
file_path = os.path.join(root, file)
if os.path.isfile(file_path):
arcname = os.path.join(os.path.relpath(root, tmp_dir), file)
zip_file.write(file_path, arcname)
for directory in dirs:
dir_path = os.path.join(root, directory)
if os.path.isdir(dir_path):
zip_file.write(dir_path, directory)
add_subdir(root, directory)
finally:
shutil.rmtree(tmp_dir)
return storage.getvalue()
def _build_report_email_content(
results: ParsingResults,
*,
subject: str | None = None,
attachment_filename: str | None = None,
message: str | None = None,
) -> tuple[str, str, list[tuple[str, bytes]]]:
"""Builds the subject, plain-text body, and zip attachment shared by
every report-summary email transport.
Returns:
A ``(subject, plain_message, attachments)`` tuple.
"""
date_string = datetime.now().strftime("%Y-%m-%d")
if attachment_filename:
if not attachment_filename.lower().endswith(".zip"):
attachment_filename += ".zip"
filename = attachment_filename
else:
filename = f"DMARC-{date_string}.zip"
if subject is None:
subject = f"DMARC results for {date_string}"
if message is None:
message = f"DMARC results for {date_string}"
zip_bytes = get_report_zip(results)
attachments = [(filename, zip_bytes)]
return subject, message, attachments
def email_results(
results: ParsingResults,
host: str,
mail_from: str,
mail_to: list[str] | None,
*,
mail_cc: list[str] | None = None,
mail_bcc: list[str] | None = None,
port: int = 0,
require_encryption: bool = False,
verify: bool = True,
username: str | None = None,
password: str | None = None,
subject: str | None = None,
attachment_filename: str | None = None,
message: str | None = None,
starttls: bool | None = None,
):
"""
Emails parsing results as a zip file
Args:
results (dict): Parsing results
host (str): Mail server hostname or IP address
mail_from: The value of the message from header
mail_to (list): A list of addresses to mail to
mail_cc (list): A list of addresses to CC
mail_bcc (list): A list of addresses to BCC
port (int): Port to use
require_encryption (bool): Connect with implicit SSL/TLS from the
start (usually port 465). When ``False``, the connection starts
in plain text and ``starttls`` decides whether it is upgraded.
verify (bool): verify the SSL/TLS certificate
username (str): An optional username
password (str): An optional password
subject (str): Overrides the default message subject
attachment_filename (str): Override the default attachment filename
message (str): Override the default plain text body
starttls (bool | None): Whether to upgrade a plain-text
(``require_encryption=False``) connection with STARTTLS.
``None`` (the default) upgrades when the server advertises
STARTTLS; when it does not, ``mailsuite.smtp.SMTPError`` is
raised before authenticating if ``username`` and ``password``
are given, so credentials are never sent in plain text, and
without credentials the message is sent in plain text and a
warning is logged. ``True`` requires the upgrade and raises
``mailsuite.smtp.SMTPError`` if the server does not advertise
STARTTLS. ``False`` never upgrades, even when the server
advertises STARTTLS, so any credentials and the message are
sent in plain text (a warning is logged when credentials are
sent). Ignored when ``require_encryption=True``.
"""
logger.debug("Emailing report")
assert isinstance(mail_to, list)
subject, message, attachments = _build_report_email_content(
results,
subject=subject,
attachment_filename=attachment_filename,
message=message,
)
send_email(
host,
mail_from,
mail_to,
message_cc=mail_cc,
message_bcc=mail_bcc,
port=port,
require_encryption=require_encryption,
verify=verify,
username=username,
password=password,
subject=subject,
attachments=attachments,
plain_message=message,
starttls=starttls,
)
def email_results_via_msgraph(
results: ParsingResults,
connection: MSGraphConnection,
mail_to: list[str],
*,
mail_cc: list[str] | None = None,
mail_bcc: list[str] | None = None,
subject: str | None = None,
attachment_filename: str | None = None,
message: str | None = None,
) -> None:
"""
Emails parsing results as a zip file via an already-authenticated
Microsoft Graph mailbox connection (``/users/{mailbox}/sendMail``),
saving a copy to Sent Items.
Args:
results (dict): Parsing results
connection (MSGraphConnection): An already-authenticated Microsoft
Graph mailbox connection
mail_to (list): A list of addresses to mail to
mail_cc (list): A list of addresses to CC
mail_bcc (list): A list of addresses to BCC
subject (str): Overrides the default message subject
attachment_filename (str): Override the default attachment filename
message (str): Override the default plain text body
"""
logger.debug("Emailing report via Microsoft Graph")
subject, message, attachments = _build_report_email_content(
results,
subject=subject,
attachment_filename=attachment_filename,
message=message,
)
# Graph derives the From header from the authenticated mailbox and
# ignores message_from; it's still passed for API parity with
# send_message()'s signature.
connection.send_message(
message_from=connection.mailbox_name or "",
message_to=mail_to,
message_cc=mail_cc,
message_bcc=mail_bcc,
subject=subject,
attachments=attachments,
plain_message=message,
)
# Backward-compatible aliases
parse_forensic_report = parse_failure_report
parsed_forensic_reports_to_csv_rows = parsed_failure_reports_to_csv_rows
parsed_forensic_reports_to_csv = parsed_failure_reports_to_csv
Reference in new issue View Git Blame Copy Permalink