mirror of
https://github.com/domainaware/parsedmarc.git
synced 2026-08-02 13:42:19 +00:00
3734 lines
492 KiB
HTML
3734 lines
492 KiB
HTML
|
|
|
|
<!DOCTYPE html>
|
|
<html class="writer-html5" lang="en" data-content_root="../">
|
|
<head>
|
|
<meta charset="utf-8" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
<title>parsedmarc — parsedmarc 10.4.0 documentation</title>
|
|
<link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=b86133f3" />
|
|
<link rel="stylesheet" type="text/css" href="../_static/css/theme.css?v=9edc463e" />
|
|
|
|
|
|
<script src="../_static/jquery.js?v=5d32c60e"></script>
|
|
<script src="../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
|
<script src="../_static/documentation_options.js?v=2f7da7c1"></script>
|
|
<script src="../_static/doctools.js?v=fd6eb6e6"></script>
|
|
<script src="../_static/sphinx_highlight.js?v=6ffebe34"></script>
|
|
<script src="../_static/js/theme.js"></script>
|
|
<link rel="index" title="Index" href="../genindex.html" />
|
|
<link rel="search" title="Search" href="../search.html" />
|
|
</head>
|
|
|
|
<body class="wy-body-for-nav">
|
|
<div class="wy-grid-for-nav">
|
|
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
|
|
<div class="wy-side-scroll">
|
|
<div class="wy-side-nav-search" >
|
|
|
|
|
|
|
|
<a href="../index.html" class="icon icon-home">
|
|
parsedmarc
|
|
</a>
|
|
<div role="search">
|
|
<form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
|
|
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
|
<input type="hidden" name="check_keywords" value="yes" />
|
|
<input type="hidden" name="area" value="default" />
|
|
</form>
|
|
</div>
|
|
</div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
|
|
<p class="caption" role="heading"><span class="caption-text">Contents</span></p>
|
|
<ul>
|
|
<li class="toctree-l1"><a class="reference internal" href="../installation.html">Installation</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../usage.html">Using parsedmarc</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../output.html">Sample outputs</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../elasticsearch.html">Elasticsearch and Kibana</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../opensearch.html">OpenSearch and Grafana</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../kibana.html">Using the Kibana dashboards</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../splunk.html">Splunk</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../davmail.html">Accessing an inbox using OWA/EWS</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../dmarc.html">Understanding DMARC</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../contributing.html">Contributing to parsedmarc</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../api.html">API reference</a></li>
|
|
</ul>
|
|
|
|
</div>
|
|
</div>
|
|
</nav>
|
|
|
|
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu" >
|
|
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
|
|
<a href="../index.html">parsedmarc</a>
|
|
</nav>
|
|
|
|
<div class="wy-nav-content">
|
|
<div class="rst-content">
|
|
<div role="navigation" aria-label="Page navigation">
|
|
<ul class="wy-breadcrumbs">
|
|
<li><a href="../index.html" class="icon icon-home" aria-label="Home"></a></li>
|
|
<li class="breadcrumb-item"><a href="index.html">Module code</a></li>
|
|
<li class="breadcrumb-item active">parsedmarc</li>
|
|
<li class="wy-breadcrumbs-aside">
|
|
</li>
|
|
</ul>
|
|
<hr/>
|
|
</div>
|
|
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
|
|
<div itemprop="articleBody">
|
|
|
|
<h1>Source code for parsedmarc</h1><div class="highlight"><pre>
|
|
<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
|
|
|
|
<span class="sd">"""A Python package for parsing DMARC reports"""</span>
|
|
|
|
<span class="kn">from</span><span class="w"> </span><span class="nn">__future__</span><span class="w"> </span><span class="kn">import</span> <span class="n">annotations</span>
|
|
|
|
<span class="kn">import</span><span class="w"> </span><span class="nn">binascii</span>
|
|
<span class="kn">import</span><span class="w"> </span><span class="nn">email</span>
|
|
<span class="kn">import</span><span class="w"> </span><span class="nn">email.utils</span>
|
|
<span class="kn">import</span><span class="w"> </span><span class="nn">functools</span>
|
|
<span class="kn">import</span><span class="w"> </span><span class="nn">json</span>
|
|
<span class="kn">import</span><span class="w"> </span><span class="nn">logging</span>
|
|
<span class="kn">import</span><span class="w"> </span><span class="nn">mailbox</span>
|
|
<span class="kn">import</span><span class="w"> </span><span class="nn">os</span>
|
|
<span class="kn">import</span><span class="w"> </span><span class="nn">re</span>
|
|
<span class="kn">import</span><span class="w"> </span><span class="nn">shutil</span>
|
|
<span class="kn">import</span><span class="w"> </span><span class="nn">tempfile</span>
|
|
<span class="kn">import</span><span class="w"> </span><span class="nn">traceback</span>
|
|
<span class="kn">import</span><span class="w"> </span><span class="nn">xml.parsers.expat</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">expat</span>
|
|
<span class="kn">import</span><span class="w"> </span><span class="nn">zipfile</span>
|
|
<span class="kn">import</span><span class="w"> </span><span class="nn">zlib</span>
|
|
<span class="kn">from</span><span class="w"> </span><span class="nn">base64</span><span class="w"> </span><span class="kn">import</span> <span class="n">b64decode</span>
|
|
<span class="kn">from</span><span class="w"> </span><span class="nn">collections</span><span class="w"> </span><span class="kn">import</span> <span class="n">deque</span>
|
|
<span class="kn">from</span><span class="w"> </span><span class="nn">collections.abc</span><span class="w"> </span><span class="kn">import</span> <span class="n">Callable</span><span class="p">,</span> <span class="n">Sequence</span>
|
|
<span class="kn">from</span><span class="w"> </span><span class="nn">csv</span><span class="w"> </span><span class="kn">import</span> <span class="n">DictWriter</span>
|
|
<span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">date</span><span class="p">,</span> <span class="n">datetime</span><span class="p">,</span> <span class="n">timedelta</span><span class="p">,</span> <span class="n">timezone</span><span class="p">,</span> <span class="n">tzinfo</span>
|
|
<span class="kn">from</span><span class="w"> </span><span class="nn">io</span><span class="w"> </span><span class="kn">import</span> <span class="n">BytesIO</span><span class="p">,</span> <span class="n">StringIO</span>
|
|
<span class="kn">from</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
|
|
<span class="n">Any</span><span class="p">,</span>
|
|
<span class="n">BinaryIO</span><span class="p">,</span>
|
|
<span class="n">cast</span><span class="p">,</span>
|
|
<span class="p">)</span>
|
|
|
|
<span class="kn">import</span><span class="w"> </span><span class="nn">lxml.etree</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">etree</span>
|
|
<span class="kn">import</span><span class="w"> </span><span class="nn">mailparser</span>
|
|
<span class="kn">import</span><span class="w"> </span><span class="nn">xmltodict</span>
|
|
<span class="kn">from</span><span class="w"> </span><span class="nn">expiringdict</span><span class="w"> </span><span class="kn">import</span> <span class="n">ExpiringDict</span>
|
|
<span class="kn">from</span><span class="w"> </span><span class="nn">mailsuite.smtp</span><span class="w"> </span><span class="kn">import</span> <span class="n">send_email</span>
|
|
<span class="kn">from</span><span class="w"> </span><span class="nn">tqdm</span><span class="w"> </span><span class="kn">import</span> <span class="n">tqdm</span>
|
|
|
|
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.config</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
|
|
<span class="n">IP_ADDRESS_CACHE</span><span class="p">,</span>
|
|
<span class="n">REVERSE_DNS_MAP</span><span class="p">,</span>
|
|
<span class="n">SEEN_AGGREGATE_REPORT_IDS</span><span class="p">,</span>
|
|
<span class="n">ParserConfig</span><span class="p">,</span>
|
|
<span class="p">)</span>
|
|
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.constants</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
|
|
<span class="n">DEFAULT_DNS_MAX_RETRIES</span><span class="p">,</span>
|
|
<span class="n">DEFAULT_DNS_TIMEOUT</span><span class="p">,</span>
|
|
<span class="n">__version__</span><span class="p">,</span>
|
|
<span class="p">)</span>
|
|
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.log</span><span class="w"> </span><span class="kn">import</span> <span class="n">logger</span>
|
|
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.mail</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
|
|
<span class="n">GmailConnection</span><span class="p">,</span>
|
|
<span class="n">IMAPConnection</span><span class="p">,</span>
|
|
<span class="n">MailboxConnection</span><span class="p">,</span>
|
|
<span class="n">MaildirConnection</span><span class="p">,</span>
|
|
<span class="n">MSGraphConnection</span><span class="p">,</span>
|
|
<span class="p">)</span>
|
|
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.types</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
|
|
<span class="n">AggregateReport</span><span class="p">,</span>
|
|
<span class="n">FailureReport</span><span class="p">,</span>
|
|
<span class="n">ForensicReport</span> <span class="k">as</span> <span class="n">ForensicReport</span><span class="p">,</span>
|
|
<span class="n">ParsedReport</span><span class="p">,</span>
|
|
<span class="n">ParsingResults</span><span class="p">,</span>
|
|
<span class="n">ReportType</span><span class="p">,</span>
|
|
<span class="n">SMTPTLSReport</span><span class="p">,</span>
|
|
<span class="p">)</span>
|
|
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.utils</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
|
|
<span class="n">convert_outlook_msg</span><span class="p">,</span>
|
|
<span class="n">get_base_domain</span><span class="p">,</span>
|
|
<span class="n">get_ip_address_info</span><span class="p">,</span>
|
|
<span class="n">human_timestamp_to_datetime</span><span class="p">,</span>
|
|
<span class="n">is_outlook_msg</span><span class="p">,</span>
|
|
<span class="n">parse_email</span><span class="p">,</span>
|
|
<span class="n">timestamp_to_human</span><span class="p">,</span>
|
|
<span class="p">)</span>
|
|
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">"parsedmarc v</span><span class="si">{</span><span class="n">__version__</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
|
|
|
<span class="n">feedback_report_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">"^([\w\-]+): (.+)$"</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">MULTILINE</span><span class="p">)</span>
|
|
<span class="n">xml_header_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">"^<\?xml .*?>"</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">MULTILINE</span><span class="p">)</span>
|
|
<span class="n">xml_schema_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">"</??xs:schema.*>"</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">MULTILINE</span><span class="p">)</span>
|
|
<span class="n">text_report_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">"\s*([a-zA-Z\s]+):\s(.+)"</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">MULTILINE</span><span class="p">)</span>
|
|
<span class="c1"># Captures the value of any xmlns (default or prefixed) declaration so the</span>
|
|
<span class="c1"># RFC 9990 namespace can be detected before xmltodict drops it.</span>
|
|
<span class="n">xml_namespace_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span>
|
|
<span class="w"> </span><span class="sa">r</span><span class="sd">"""xmlns(?::[a-zA-Z_][\w.-]*)?\s*=\s*["']([^"']+)["']"""</span>
|
|
<span class="p">)</span>
|
|
|
|
<span class="c1"># The XML namespace assigned to DMARC aggregate reports by RFC 9990.</span>
|
|
<span class="n">RFC_9990_NAMESPACE</span> <span class="o">=</span> <span class="s2">"urn:ietf:params:xml:ns:dmarc-2.0"</span>
|
|
|
|
<span class="c1"># PolicyOverrideType enumeration from RFC 9990. Compared to RFC 7489,</span>
|
|
<span class="c1"># `policy_test_mode` was added (emitted when t=y suppresses enforcement)</span>
|
|
<span class="c1"># and `forwarded` / `sampled_out` were removed.</span>
|
|
<span class="n">RFC_9990_POLICY_OVERRIDE_TYPES</span> <span class="o">=</span> <span class="nb">frozenset</span><span class="p">(</span>
|
|
<span class="p">{</span>
|
|
<span class="s2">"local_policy"</span><span class="p">,</span>
|
|
<span class="s2">"mailing_list"</span><span class="p">,</span>
|
|
<span class="s2">"other"</span><span class="p">,</span>
|
|
<span class="s2">"policy_test_mode"</span><span class="p">,</span>
|
|
<span class="s2">"trusted_forwarder"</span><span class="p">,</span>
|
|
<span class="p">}</span>
|
|
<span class="p">)</span>
|
|
<span class="n">RFC_7489_REMOVED_POLICY_OVERRIDE_TYPES</span> <span class="o">=</span> <span class="nb">frozenset</span><span class="p">({</span><span class="s2">"forwarded"</span><span class="p">,</span> <span class="s2">"sampled_out"</span><span class="p">})</span>
|
|
|
|
<span class="n">MAGIC_ZIP</span> <span class="o">=</span> <span class="sa">b</span><span class="s2">"</span><span class="se">\x50\x4b\x03\x04</span><span class="s2">"</span>
|
|
<span class="n">MAGIC_GZIP</span> <span class="o">=</span> <span class="sa">b</span><span class="s2">"</span><span class="se">\x1f\x8b</span><span class="s2">"</span>
|
|
<span class="n">MAGIC_XML</span> <span class="o">=</span> <span class="sa">b</span><span class="s2">"</span><span class="se">\x3c\x3f\x78\x6d\x6c\x20</span><span class="s2">"</span>
|
|
<span class="n">MAGIC_XML_TAG</span> <span class="o">=</span> <span class="sa">b</span><span class="s2">"</span><span class="se">\x3c</span><span class="s2">"</span> <span class="c1"># '<' - XML starting with an element tag (no declaration)</span>
|
|
<span class="c1"># 0x7B, "{" -- a JSON text that is an object begins with it (RFC 8259).</span>
|
|
<span class="c1"># Previously written as b"\7b", which Python reads as the octal escape</span>
|
|
<span class="c1"># \7 (BEL) followed by a literal "b", so the branch never matched real</span>
|
|
<span class="c1"># JSON; every in-tree caller happened to pre-guard with its own zip/gzip</span>
|
|
<span class="c1"># or "{" check, which masked it.</span>
|
|
<span class="n">MAGIC_JSON</span> <span class="o">=</span> <span class="sa">b</span><span class="s2">"</span><span class="se">\x7b</span><span class="s2">"</span>
|
|
|
|
<span class="c1"># Per-message count of consecutive failed saves, keyed on</span>
|
|
<span class="c1"># ``(reports_folder, str(message_uid))``. Populated only when</span>
|
|
<span class="c1"># ``get_dmarc_reports_from_mailbox()`` is given a ``save_callback`` that</span>
|
|
<span class="c1"># reports a batch as unsaved; a message whose count exceeds</span>
|
|
<span class="c1"># ``max_unsaved_retries`` is moved to ``{archive_folder}/Unsaved`` instead</span>
|
|
<span class="c1"># of being retried forever, and its entry is dropped. A successful save</span>
|
|
<span class="c1"># clears the entries for that batch's messages.</span>
|
|
<span class="c1">#</span>
|
|
<span class="c1"># Process-local and deliberately not persisted: a restart re-attempts every</span>
|
|
<span class="c1"># message still in the reports folder, which is the safe direction (retry</span>
|
|
<span class="c1"># rather than shelve). The key carries no connection identity, so a library</span>
|
|
<span class="c1"># caller processing two connections that share a folder name (two IMAP</span>
|
|
<span class="c1"># servers, both "INBOX") in one process could collide counters if UIDs</span>
|
|
<span class="c1"># happen to match; the CLI uses one connection per process. It is not a</span>
|
|
<span class="c1"># ``ParserConfig`` field for the same reason ``batch_size`` and the</span>
|
|
<span class="c1"># ``delete`` flags are not -- it governs mailbox orchestration, not</span>
|
|
<span class="c1"># parsing.</span>
|
|
<span class="n">_FAILED_SAVE_ATTEMPTS</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">],</span> <span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
|
|
|
|
<span class="n">EMAIL_SAMPLE_CONTENT_TYPES</span> <span class="o">=</span> <span class="p">(</span>
|
|
<span class="s2">"text/rfc822"</span><span class="p">,</span>
|
|
<span class="s2">"text-rfc-822"</span><span class="p">,</span>
|
|
<span class="s2">"text/rfc822-headers"</span><span class="p">,</span>
|
|
<span class="s2">"text/rfc-822-headers"</span><span class="p">,</span>
|
|
<span class="s2">"message/rfc822"</span><span class="p">,</span>
|
|
<span class="s2">"message/rfc-822"</span><span class="p">,</span>
|
|
<span class="s2">"message/rfc822-headers"</span><span class="p">,</span>
|
|
<span class="s2">"message/rfc-822-headers"</span><span class="p">,</span>
|
|
<span class="p">)</span>
|
|
|
|
|
|
<div class="viewcode-block" id="ParserError">
|
|
<a class="viewcode-back" href="../api.html#parsedmarc.ParserError">[docs]</a>
|
|
<span class="k">class</span><span class="w"> </span><span class="nc">ParserError</span><span class="p">(</span><span class="ne">RuntimeError</span><span class="p">):</span>
|
|
<span class="w"> </span><span class="sd">"""Raised whenever the parser fails for some reason"""</span></div>
|
|
|
|
|
|
|
|
<div class="viewcode-block" id="InvalidDMARCReport">
|
|
<a class="viewcode-back" href="../api.html#parsedmarc.InvalidDMARCReport">[docs]</a>
|
|
<span class="k">class</span><span class="w"> </span><span class="nc">InvalidDMARCReport</span><span class="p">(</span><span class="n">ParserError</span><span class="p">):</span>
|
|
<span class="w"> </span><span class="sd">"""Raised when an invalid DMARC report is encountered"""</span></div>
|
|
|
|
|
|
|
|
<div class="viewcode-block" id="InvalidSMTPTLSReport">
|
|
<a class="viewcode-back" href="../api.html#parsedmarc.InvalidSMTPTLSReport">[docs]</a>
|
|
<span class="k">class</span><span class="w"> </span><span class="nc">InvalidSMTPTLSReport</span><span class="p">(</span><span class="n">ParserError</span><span class="p">):</span>
|
|
<span class="w"> </span><span class="sd">"""Raised when an invalid SMTP TLS report is encountered"""</span></div>
|
|
|
|
|
|
|
|
<div class="viewcode-block" id="InvalidAggregateReport">
|
|
<a class="viewcode-back" href="../api.html#parsedmarc.InvalidAggregateReport">[docs]</a>
|
|
<span class="k">class</span><span class="w"> </span><span class="nc">InvalidAggregateReport</span><span class="p">(</span><span class="n">InvalidDMARCReport</span><span class="p">):</span>
|
|
<span class="w"> </span><span class="sd">"""Raised when an invalid DMARC aggregate report is encountered"""</span></div>
|
|
|
|
|
|
|
|
<div class="viewcode-block" id="InvalidFailureReport">
|
|
<a class="viewcode-back" href="../api.html#parsedmarc.InvalidFailureReport">[docs]</a>
|
|
<span class="k">class</span><span class="w"> </span><span class="nc">InvalidFailureReport</span><span class="p">(</span><span class="n">InvalidDMARCReport</span><span class="p">):</span>
|
|
<span class="w"> </span><span class="sd">"""Raised when an invalid DMARC failure report is encountered"""</span></div>
|
|
|
|
|
|
|
|
<span class="c1"># Backward-compatible alias</span>
|
|
<span class="n">InvalidForensicReport</span> <span class="o">=</span> <span class="n">InvalidFailureReport</span>
|
|
|
|
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">_resolve_config</span><span class="p">(</span>
|
|
<span class="n">config</span><span class="p">:</span> <span class="n">ParserConfig</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="o">*</span><span class="p">,</span>
|
|
<span class="n">offline</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
|
<span class="n">ip_db_path</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">always_use_local_files</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
|
<span class="n">reverse_dns_map_path</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">reverse_dns_map_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">nameservers</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">dns_timeout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="n">DEFAULT_DNS_TIMEOUT</span><span class="p">,</span>
|
|
<span class="n">dns_retries</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="n">DEFAULT_DNS_MAX_RETRIES</span><span class="p">,</span>
|
|
<span class="n">strip_attachment_payloads</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
|
<span class="n">normalize_timespan_threshold_hours</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">24.0</span><span class="p">,</span>
|
|
<span class="p">)</span> <span class="o">-></span> <span class="n">ParserConfig</span><span class="p">:</span>
|
|
<span class="w"> </span><span class="sd">"""Resolve the effective :class:`~parsedmarc.config.ParserConfig` for a</span>
|
|
<span class="sd"> public parsing call, from either an explicit ``config`` or the caller's</span>
|
|
<span class="sd"> individual option keyword arguments.</span>
|
|
|
|
<span class="sd"> If ``config`` is not ``None``, it is returned unchanged: per the</span>
|
|
<span class="sd"> documented ``config=`` contract, the individual option keyword arguments</span>
|
|
<span class="sd"> are ignored in favor of the config's own values, so no merging happens</span>
|
|
<span class="sd"> here.</span>
|
|
|
|
<span class="sd"> Otherwise, a new ``ParserConfig`` is built from the given keyword</span>
|
|
<span class="sd"> arguments. Its three cache fields are deliberately *not* left to their</span>
|
|
<span class="sd"> ``default_factory`` -- doing so would hand back a config with brand new,</span>
|
|
<span class="sd"> empty caches on every call, silently defeating cross-call IP-info</span>
|
|
<span class="sd"> caching and aggregate-report dedup. Instead, the module-default caches</span>
|
|
<span class="sd"> (:data:`IP_ADDRESS_CACHE`, :data:`SEEN_AGGREGATE_REPORT_IDS`,</span>
|
|
<span class="sd"> :data:`REVERSE_DNS_MAP`) are injected explicitly, so kwargs-style calls</span>
|
|
<span class="sd"> keep observing and mutating the same shared caches they always have.</span>
|
|
|
|
<span class="sd"> ``dataclasses.replace`` is deliberately not used here: it would need an</span>
|
|
<span class="sd"> existing ``ParserConfig`` to start from, and there isn't one on the</span>
|
|
<span class="sd"> kwargs path -- this function's job is to build the first one.</span>
|
|
|
|
<span class="sd"> ``psl_overrides_path`` / ``psl_overrides_url`` have no corresponding</span>
|
|
<span class="sd"> keyword arguments on the public functions (see AGENTS.md's guidance on</span>
|
|
<span class="sd"> justifying new config options), so they stay ``None`` on this path; they</span>
|
|
<span class="sd"> are only ever set via an explicitly constructed ``config=``.</span>
|
|
<span class="sd"> """</span>
|
|
<span class="k">if</span> <span class="n">config</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="k">return</span> <span class="n">config</span>
|
|
<span class="k">return</span> <span class="n">ParserConfig</span><span class="p">(</span>
|
|
<span class="n">offline</span><span class="o">=</span><span class="n">offline</span><span class="p">,</span>
|
|
<span class="n">ip_db_path</span><span class="o">=</span><span class="n">ip_db_path</span><span class="p">,</span>
|
|
<span class="n">always_use_local_files</span><span class="o">=</span><span class="n">always_use_local_files</span><span class="p">,</span>
|
|
<span class="n">reverse_dns_map_path</span><span class="o">=</span><span class="n">reverse_dns_map_path</span><span class="p">,</span>
|
|
<span class="n">reverse_dns_map_url</span><span class="o">=</span><span class="n">reverse_dns_map_url</span><span class="p">,</span>
|
|
<span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
|
|
<span class="n">dns_timeout</span><span class="o">=</span><span class="n">dns_timeout</span><span class="p">,</span>
|
|
<span class="n">dns_retries</span><span class="o">=</span><span class="n">dns_retries</span><span class="p">,</span>
|
|
<span class="n">strip_attachment_payloads</span><span class="o">=</span><span class="n">strip_attachment_payloads</span><span class="p">,</span>
|
|
<span class="n">normalize_timespan_threshold_hours</span><span class="o">=</span><span class="nb">float</span><span class="p">(</span><span class="n">normalize_timespan_threshold_hours</span><span class="p">),</span>
|
|
<span class="n">ip_address_cache</span><span class="o">=</span><span class="n">IP_ADDRESS_CACHE</span><span class="p">,</span>
|
|
<span class="n">seen_aggregate_report_ids</span><span class="o">=</span><span class="n">SEEN_AGGREGATE_REPORT_IDS</span><span class="p">,</span>
|
|
<span class="n">reverse_dns_map</span><span class="o">=</span><span class="n">REVERSE_DNS_MAP</span><span class="p">,</span>
|
|
<span class="p">)</span>
|
|
|
|
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">_exc_origin</span><span class="p">(</span><span class="n">error</span><span class="p">:</span> <span class="ne">BaseException</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
|
<span class="w"> </span><span class="sd">"""Returns a ``" (raised at <file>:<line>)"`` suffix pointing at where an</span>
|
|
<span class="sd"> unexpected exception actually originated, but only when the parsedmarc</span>
|
|
<span class="sd"> logger is at ``DEBUG`` level. Returns ``""`` otherwise, so normal output is</span>
|
|
<span class="sd"> unchanged.</span>
|
|
|
|
<span class="sd"> The deepest traceback frame is the line that raised, which is the useful</span>
|
|
<span class="sd"> breadcrumb when a catch-all ``except Exception`` turns an unforeseen error</span>
|
|
<span class="sd"> into a generic ``Invalid*Report`` -- without it, the message says *what*</span>
|
|
<span class="sd"> failed but not *where*.</span>
|
|
<span class="sd"> """</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">logger</span><span class="o">.</span><span class="n">isEnabledFor</span><span class="p">(</span><span class="n">logging</span><span class="o">.</span><span class="n">DEBUG</span><span class="p">):</span>
|
|
<span class="k">return</span> <span class="s2">""</span>
|
|
<span class="n">frames</span> <span class="o">=</span> <span class="n">traceback</span><span class="o">.</span><span class="n">extract_tb</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="n">__traceback__</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">frames</span><span class="p">:</span>
|
|
<span class="k">return</span> <span class="s2">""</span>
|
|
<span class="n">last</span> <span class="o">=</span> <span class="n">frames</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
|
|
<span class="k">return</span> <span class="sa">f</span><span class="s2">" (raised at </span><span class="si">{</span><span class="n">last</span><span class="o">.</span><span class="n">filename</span><span class="si">}</span><span class="s2">:</span><span class="si">{</span><span class="n">last</span><span class="o">.</span><span class="n">lineno</span><span class="si">}</span><span class="s2">)"</span>
|
|
|
|
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">_text</span><span class="p">(</span><span class="n">value</span><span class="p">:</span> <span class="n">Any</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="w"> </span><span class="sd">"""Unwrap a possibly-langAttrString value parsed by xmltodict.</span>
|
|
|
|
<span class="sd"> RFC 9990 changed several aggregate-report elements (extra_contact_info,</span>
|
|
<span class="sd"> error, comment, human_result) to type ``langAttrString`` — an</span>
|
|
<span class="sd"> xs:simpleContent string with an optional ``lang`` attribute. When the</span>
|
|
<span class="sd"> attribute is present, xmltodict parses the element as</span>
|
|
<span class="sd"> ``{"#text": "...", "@lang": "en"}`` instead of a plain string. Returns</span>
|
|
<span class="sd"> the text payload for both shapes, ``None`` for unset values, and leaves</span>
|
|
<span class="sd"> other scalar shapes untouched so callers can preserve whatever the</span>
|
|
<span class="sd"> reporter sent.</span>
|
|
<span class="sd"> """</span>
|
|
<span class="k">if</span> <span class="n">value</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="k">return</span> <span class="kc">None</span>
|
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
|
|
<span class="n">text</span> <span class="o">=</span> <span class="n">value</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"#text"</span><span class="p">)</span>
|
|
<span class="k">return</span> <span class="kc">None</span> <span class="k">if</span> <span class="n">text</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="nb">str</span><span class="p">(</span><span class="n">text</span><span class="p">)</span>
|
|
<span class="k">return</span> <span class="n">value</span>
|
|
|
|
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">_normalize_result_word</span><span class="p">(</span><span class="n">value</span><span class="p">:</span> <span class="n">Any</span><span class="p">)</span> <span class="o">-></span> <span class="n">Any</span><span class="p">:</span>
|
|
<span class="w"> </span><span class="sd">"""Lowercase a reporter-supplied enum word; RFC 7489 Appendix C and</span>
|
|
<span class="sd"> RFC 9990 define result/disposition types as lowercase tokens. Non-string</span>
|
|
<span class="sd"> values (e.g. xmltodict dicts from attribute-bearing elements) pass</span>
|
|
<span class="sd"> through unchanged.</span>
|
|
<span class="sd"> """</span>
|
|
<span class="k">return</span> <span class="n">value</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="nb">str</span><span class="p">)</span> <span class="k">else</span> <span class="n">value</span>
|
|
|
|
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">_bucket_interval_by_day</span><span class="p">(</span>
|
|
<span class="n">begin</span><span class="p">:</span> <span class="n">datetime</span><span class="p">,</span>
|
|
<span class="n">end</span><span class="p">:</span> <span class="n">datetime</span><span class="p">,</span>
|
|
<span class="n">total_count</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
|
|
<span class="p">)</span> <span class="o">-></span> <span class="nb">list</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]]:</span>
|
|
<span class="w"> </span><span class="sd">"""</span>
|
|
<span class="sd"> Split the interval [begin, end) into daily buckets and distribute</span>
|
|
<span class="sd"> `total_count` proportionally across those buckets.</span>
|
|
|
|
<span class="sd"> The function:</span>
|
|
<span class="sd"> 1. Identifies each calendar day touched by [begin, end)</span>
|
|
<span class="sd"> 2. Computes how many seconds of the interval fall into each day</span>
|
|
<span class="sd"> 3. Assigns counts in proportion to those overlaps</span>
|
|
<span class="sd"> 4. Ensures the final counts sum exactly to total_count</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> begin: timezone-aware datetime, inclusive start of interval</span>
|
|
<span class="sd"> end: timezone-aware datetime, exclusive end of interval</span>
|
|
<span class="sd"> total_count: number of messages to distribute</span>
|
|
|
|
<span class="sd"> Returns:</span>
|
|
<span class="sd"> A list of dicts like:</span>
|
|
<span class="sd"> {</span>
|
|
<span class="sd"> "begin": datetime,</span>
|
|
<span class="sd"> "end": datetime,</span>
|
|
<span class="sd"> "count": int</span>
|
|
<span class="sd"> }</span>
|
|
<span class="sd"> """</span>
|
|
<span class="c1"># --- Input validation ----------------------------------------------------</span>
|
|
<span class="k">if</span> <span class="n">begin</span> <span class="o">></span> <span class="n">end</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"begin must be earlier than end"</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">begin</span><span class="o">.</span><span class="n">tzinfo</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">or</span> <span class="n">end</span><span class="o">.</span><span class="n">tzinfo</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"begin and end must be timezone-aware"</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">begin</span><span class="o">.</span><span class="n">tzinfo</span> <span class="ow">is</span> <span class="ow">not</span> <span class="n">end</span><span class="o">.</span><span class="n">tzinfo</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"begin and end must have the same tzinfo"</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">total_count</span> <span class="o"><</span> <span class="mi">0</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"total_count must be non-negative"</span><span class="p">)</span>
|
|
|
|
<span class="c1"># --- Short-circuit trivial cases -----------------------------------------</span>
|
|
<span class="n">interval_seconds</span> <span class="o">=</span> <span class="p">(</span><span class="n">end</span> <span class="o">-</span> <span class="n">begin</span><span class="p">)</span><span class="o">.</span><span class="n">total_seconds</span><span class="p">()</span>
|
|
<span class="k">if</span> <span class="n">interval_seconds</span> <span class="o"><=</span> <span class="mi">0</span> <span class="ow">or</span> <span class="n">total_count</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
|
|
<span class="k">return</span> <span class="p">[]</span>
|
|
|
|
<span class="n">tz</span><span class="p">:</span> <span class="n">tzinfo</span> <span class="o">=</span> <span class="n">begin</span><span class="o">.</span><span class="n">tzinfo</span>
|
|
|
|
<span class="c1"># --- Step 1: Determine all calendar days touched by [begin, end) ----------</span>
|
|
<span class="c1">#</span>
|
|
<span class="c1"># For example:</span>
|
|
<span class="c1"># begin = Jan 1 12:00</span>
|
|
<span class="c1"># end = Jan 3 06:00</span>
|
|
<span class="c1">#</span>
|
|
<span class="c1"># We need buckets for:</span>
|
|
<span class="c1"># Jan 1 12:00 → Jan 2 00:00</span>
|
|
<span class="c1"># Jan 2 00:00 → Jan 3 00:00</span>
|
|
<span class="c1"># Jan 3 00:00 → Jan 3 06:00</span>
|
|
<span class="c1">#</span>
|
|
|
|
<span class="c1"># Start at midnight on the day of `begin`.</span>
|
|
<span class="n">day_cursor</span> <span class="o">=</span> <span class="n">datetime</span><span class="p">(</span><span class="n">begin</span><span class="o">.</span><span class="n">year</span><span class="p">,</span> <span class="n">begin</span><span class="o">.</span><span class="n">month</span><span class="p">,</span> <span class="n">begin</span><span class="o">.</span><span class="n">day</span><span class="p">,</span> <span class="n">tzinfo</span><span class="o">=</span><span class="n">tz</span><span class="p">)</span>
|
|
|
|
<span class="c1"># If `begin` is earlier on that day (e.g. 10:00), we want that midnight.</span>
|
|
<span class="c1"># If `begin` is past that midnight (e.g. 00:30), this is correct.</span>
|
|
<span class="c1"># If `begin` is BEFORE that midnight (rare unless tz shifts), adjust:</span>
|
|
<span class="k">if</span> <span class="n">day_cursor</span> <span class="o">></span> <span class="n">begin</span><span class="p">:</span>
|
|
<span class="n">day_cursor</span> <span class="o">-=</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
|
|
|
|
<span class="n">day_buckets</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]]</span> <span class="o">=</span> <span class="p">[]</span>
|
|
|
|
<span class="k">while</span> <span class="n">day_cursor</span> <span class="o"><</span> <span class="n">end</span><span class="p">:</span>
|
|
<span class="n">day_start</span> <span class="o">=</span> <span class="n">day_cursor</span>
|
|
<span class="n">day_end</span> <span class="o">=</span> <span class="n">day_cursor</span> <span class="o">+</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
|
|
|
|
<span class="c1"># Overlap between [begin, end) and this day</span>
|
|
<span class="n">overlap_start</span> <span class="o">=</span> <span class="nb">max</span><span class="p">(</span><span class="n">begin</span><span class="p">,</span> <span class="n">day_start</span><span class="p">)</span>
|
|
<span class="n">overlap_end</span> <span class="o">=</span> <span class="nb">min</span><span class="p">(</span><span class="n">end</span><span class="p">,</span> <span class="n">day_end</span><span class="p">)</span>
|
|
|
|
<span class="n">overlap_seconds</span> <span class="o">=</span> <span class="p">(</span><span class="n">overlap_end</span> <span class="o">-</span> <span class="n">overlap_start</span><span class="p">)</span><span class="o">.</span><span class="n">total_seconds</span><span class="p">()</span>
|
|
|
|
<span class="k">if</span> <span class="n">overlap_seconds</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
|
<span class="n">day_buckets</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
|
<span class="p">{</span>
|
|
<span class="s2">"begin"</span><span class="p">:</span> <span class="n">overlap_start</span><span class="p">,</span>
|
|
<span class="s2">"end"</span><span class="p">:</span> <span class="n">overlap_end</span><span class="p">,</span>
|
|
<span class="s2">"seconds"</span><span class="p">:</span> <span class="n">overlap_seconds</span><span class="p">,</span>
|
|
<span class="p">}</span>
|
|
<span class="p">)</span>
|
|
|
|
<span class="n">day_cursor</span> <span class="o">=</span> <span class="n">day_end</span>
|
|
|
|
<span class="c1"># --- Step 2: Pro-rate counts across buckets -------------------------------</span>
|
|
<span class="c1">#</span>
|
|
<span class="c1"># Compute the exact fractional count for each bucket:</span>
|
|
<span class="c1"># bucket_fraction = bucket_seconds / interval_seconds</span>
|
|
<span class="c1"># bucket_exact = total_count * bucket_fraction</span>
|
|
<span class="c1">#</span>
|
|
<span class="c1"># Then apply a "largest remainder" rounding strategy to ensure the sum</span>
|
|
<span class="c1"># equals exactly total_count.</span>
|
|
|
|
<span class="n">exact_values</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span>
|
|
<span class="p">(</span><span class="n">b</span><span class="p">[</span><span class="s2">"seconds"</span><span class="p">]</span> <span class="o">/</span> <span class="n">interval_seconds</span><span class="p">)</span> <span class="o">*</span> <span class="n">total_count</span> <span class="k">for</span> <span class="n">b</span> <span class="ow">in</span> <span class="n">day_buckets</span>
|
|
<span class="p">]</span>
|
|
|
|
<span class="n">floor_values</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="nb">int</span><span class="p">(</span><span class="n">x</span><span class="p">)</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">exact_values</span><span class="p">]</span>
|
|
<span class="n">fractional_parts</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="n">x</span> <span class="o">-</span> <span class="nb">int</span><span class="p">(</span><span class="n">x</span><span class="p">)</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">exact_values</span><span class="p">]</span>
|
|
|
|
<span class="c1"># How many counts do we still need to distribute after flooring?</span>
|
|
<span class="n">remainder</span> <span class="o">=</span> <span class="n">total_count</span> <span class="o">-</span> <span class="nb">sum</span><span class="p">(</span><span class="n">floor_values</span><span class="p">)</span>
|
|
|
|
<span class="c1"># Sort buckets by descending fractional remainder</span>
|
|
<span class="n">indices_by_fraction</span> <span class="o">=</span> <span class="nb">sorted</span><span class="p">(</span>
|
|
<span class="nb">range</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">day_buckets</span><span class="p">)),</span>
|
|
<span class="n">key</span><span class="o">=</span><span class="k">lambda</span> <span class="n">i</span><span class="p">:</span> <span class="n">fractional_parts</span><span class="p">[</span><span class="n">i</span><span class="p">],</span>
|
|
<span class="n">reverse</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
|
<span class="p">)</span>
|
|
|
|
<span class="c1"># Start with floor values</span>
|
|
<span class="n">final_counts</span> <span class="o">=</span> <span class="n">floor_values</span><span class="p">[:]</span>
|
|
|
|
<span class="c1"># Add +1 to the buckets with the largest fractional parts</span>
|
|
<span class="k">for</span> <span class="n">idx</span> <span class="ow">in</span> <span class="n">indices_by_fraction</span><span class="p">[:</span><span class="n">remainder</span><span class="p">]:</span>
|
|
<span class="n">final_counts</span><span class="p">[</span><span class="n">idx</span><span class="p">]</span> <span class="o">+=</span> <span class="mi">1</span>
|
|
|
|
<span class="c1"># --- Step 3: Build the final per-day result list -------------------------</span>
|
|
<span class="n">results</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]]</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="k">for</span> <span class="n">bucket</span><span class="p">,</span> <span class="n">count</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">day_buckets</span><span class="p">,</span> <span class="n">final_counts</span><span class="p">):</span>
|
|
<span class="k">if</span> <span class="n">count</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
|
<span class="p">{</span>
|
|
<span class="s2">"begin"</span><span class="p">:</span> <span class="n">bucket</span><span class="p">[</span><span class="s2">"begin"</span><span class="p">],</span>
|
|
<span class="s2">"end"</span><span class="p">:</span> <span class="n">bucket</span><span class="p">[</span><span class="s2">"end"</span><span class="p">],</span>
|
|
<span class="s2">"count"</span><span class="p">:</span> <span class="n">count</span><span class="p">,</span>
|
|
<span class="p">}</span>
|
|
<span class="p">)</span>
|
|
|
|
<span class="k">return</span> <span class="n">results</span>
|
|
|
|
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">_append_parsed_record</span><span class="p">(</span>
|
|
<span class="n">parsed_record</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">],</span>
|
|
<span class="n">records</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]],</span>
|
|
<span class="n">begin_dt</span><span class="p">:</span> <span class="n">datetime</span><span class="p">,</span>
|
|
<span class="n">end_dt</span><span class="p">:</span> <span class="n">datetime</span><span class="p">,</span>
|
|
<span class="n">normalize</span><span class="p">:</span> <span class="nb">bool</span><span class="p">,</span>
|
|
<span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="w"> </span><span class="sd">"""</span>
|
|
<span class="sd"> Append a parsed DMARC record either unchanged or normalized.</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> parsed_record: The record returned by _parse_report_record().</span>
|
|
<span class="sd"> records: Accumulating list of output records.</span>
|
|
<span class="sd"> begin_dt: Report-level begin datetime (UTC).</span>
|
|
<span class="sd"> end_dt: Report-level end datetime (UTC).</span>
|
|
<span class="sd"> normalize: Whether this report exceeded the allowed timespan</span>
|
|
<span class="sd"> and should be normalized per-day.</span>
|
|
<span class="sd"> """</span>
|
|
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">normalize</span><span class="p">:</span>
|
|
<span class="n">parsed_record</span><span class="p">[</span><span class="s2">"normalized_timespan"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span>
|
|
<span class="n">parsed_record</span><span class="p">[</span><span class="s2">"interval_begin"</span><span class="p">]</span> <span class="o">=</span> <span class="n">begin_dt</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%S"</span><span class="p">)</span>
|
|
<span class="n">parsed_record</span><span class="p">[</span><span class="s2">"interval_end"</span><span class="p">]</span> <span class="o">=</span> <span class="n">end_dt</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%S"</span><span class="p">)</span>
|
|
|
|
<span class="n">records</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">parsed_record</span><span class="p">)</span>
|
|
<span class="k">return</span>
|
|
|
|
<span class="c1"># Normalization path: break record into daily buckets</span>
|
|
<span class="n">total_count</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">parsed_record</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"count"</span><span class="p">,</span> <span class="mi">0</span><span class="p">))</span>
|
|
<span class="n">buckets</span> <span class="o">=</span> <span class="n">_bucket_interval_by_day</span><span class="p">(</span><span class="n">begin_dt</span><span class="p">,</span> <span class="n">end_dt</span><span class="p">,</span> <span class="n">total_count</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">buckets</span><span class="p">:</span>
|
|
<span class="k">return</span>
|
|
|
|
<span class="k">for</span> <span class="n">part_index</span><span class="p">,</span> <span class="n">bucket</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">buckets</span><span class="p">):</span>
|
|
<span class="n">new_rec</span> <span class="o">=</span> <span class="n">parsed_record</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
|
<span class="n">new_rec</span><span class="p">[</span><span class="s2">"count"</span><span class="p">]</span> <span class="o">=</span> <span class="n">bucket</span><span class="p">[</span><span class="s2">"count"</span><span class="p">]</span>
|
|
<span class="n">new_rec</span><span class="p">[</span><span class="s2">"normalized_timespan"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
|
|
|
|
<span class="n">new_rec</span><span class="p">[</span><span class="s2">"interval_begin"</span><span class="p">]</span> <span class="o">=</span> <span class="n">bucket</span><span class="p">[</span><span class="s2">"begin"</span><span class="p">]</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%S"</span><span class="p">)</span>
|
|
<span class="n">new_rec</span><span class="p">[</span><span class="s2">"interval_end"</span><span class="p">]</span> <span class="o">=</span> <span class="n">bucket</span><span class="p">[</span><span class="s2">"end"</span><span class="p">]</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%S"</span><span class="p">)</span>
|
|
|
|
<span class="n">records</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">new_rec</span><span class="p">)</span>
|
|
|
|
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">_parse_report_record</span><span class="p">(</span>
|
|
<span class="n">record</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">],</span>
|
|
<span class="o">*</span><span class="p">,</span>
|
|
<span class="n">config</span><span class="p">:</span> <span class="n">ParserConfig</span><span class="p">,</span>
|
|
<span class="n">is_rfc_9990</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
|
<span class="p">)</span> <span class="o">-></span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]:</span>
|
|
<span class="w"> </span><span class="sd">"""</span>
|
|
<span class="sd"> Converts a record from a DMARC aggregate report into a more consistent</span>
|
|
<span class="sd"> format</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> record (dict): The record to convert</span>
|
|
<span class="sd"> config (ParserConfig): Parsing and enrichment options, plus caches</span>
|
|
<span class="sd"> is_rfc_9990 (bool): Whether the enclosing report was detected as</span>
|
|
<span class="sd"> RFC 9990-shaped, for RFC 9990-aware validation warnings</span>
|
|
|
|
<span class="sd"> Returns:</span>
|
|
<span class="sd"> dict: The converted record</span>
|
|
<span class="sd"> """</span>
|
|
<span class="n">record</span> <span class="o">=</span> <span class="n">record</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
|
<span class="n">new_record</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
|
|
<span class="k">if</span> <span class="n">record</span><span class="p">[</span><span class="s2">"row"</span><span class="p">][</span><span class="s2">"source_ip"</span><span class="p">]</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"Source IP address is empty"</span><span class="p">)</span>
|
|
<span class="n">new_record_source</span> <span class="o">=</span> <span class="n">get_ip_address_info</span><span class="p">(</span>
|
|
<span class="n">record</span><span class="p">[</span><span class="s2">"row"</span><span class="p">][</span><span class="s2">"source_ip"</span><span class="p">],</span>
|
|
<span class="n">cache</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">ip_address_cache</span><span class="p">,</span>
|
|
<span class="n">ip_db_path</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">ip_db_path</span><span class="p">,</span>
|
|
<span class="n">always_use_local_files</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">always_use_local_files</span><span class="p">,</span>
|
|
<span class="n">reverse_dns_map_path</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">reverse_dns_map_path</span><span class="p">,</span>
|
|
<span class="n">reverse_dns_map_url</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">reverse_dns_map_url</span><span class="p">,</span>
|
|
<span class="n">reverse_dns_map</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">reverse_dns_map</span><span class="p">,</span>
|
|
<span class="n">offline</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">offline</span><span class="p">,</span>
|
|
<span class="n">nameservers</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">nameservers</span><span class="p">,</span>
|
|
<span class="n">timeout</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">dns_timeout</span><span class="p">,</span>
|
|
<span class="n">retries</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">dns_retries</span><span class="p">,</span>
|
|
<span class="n">psl_overrides_path</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">psl_overrides_path</span><span class="p">,</span>
|
|
<span class="n">psl_overrides_url</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">psl_overrides_url</span><span class="p">,</span>
|
|
<span class="p">)</span>
|
|
<span class="n">new_record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">]</span> <span class="o">=</span> <span class="n">new_record_source</span>
|
|
<span class="n">new_record</span><span class="p">[</span><span class="s2">"count"</span><span class="p">]</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">record</span><span class="p">[</span><span class="s2">"row"</span><span class="p">][</span><span class="s2">"count"</span><span class="p">])</span>
|
|
<span class="n">policy_evaluated</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"row"</span><span class="p">][</span><span class="s2">"policy_evaluated"</span><span class="p">]</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
|
<span class="n">new_policy_evaluated</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
|
|
<span class="s2">"disposition"</span><span class="p">:</span> <span class="s2">"none"</span><span class="p">,</span>
|
|
<span class="s2">"dkim"</span><span class="p">:</span> <span class="s2">"fail"</span><span class="p">,</span>
|
|
<span class="s2">"spf"</span><span class="p">:</span> <span class="s2">"fail"</span><span class="p">,</span>
|
|
<span class="s2">"policy_override_reasons"</span><span class="p">:</span> <span class="p">[],</span>
|
|
<span class="p">}</span>
|
|
<span class="k">if</span> <span class="s2">"disposition"</span> <span class="ow">in</span> <span class="n">policy_evaluated</span><span class="p">:</span>
|
|
<span class="n">new_policy_evaluated</span><span class="p">[</span><span class="s2">"disposition"</span><span class="p">]</span> <span class="o">=</span> <span class="n">_normalize_result_word</span><span class="p">(</span>
|
|
<span class="n">policy_evaluated</span><span class="p">[</span><span class="s2">"disposition"</span><span class="p">]</span>
|
|
<span class="p">)</span>
|
|
<span class="k">for</span> <span class="n">key</span> <span class="ow">in</span> <span class="p">(</span><span class="s2">"dkim"</span><span class="p">,</span> <span class="s2">"spf"</span><span class="p">):</span>
|
|
<span class="k">if</span> <span class="n">key</span> <span class="ow">in</span> <span class="n">policy_evaluated</span><span class="p">:</span>
|
|
<span class="n">new_policy_evaluated</span><span class="p">[</span><span class="n">key</span><span class="p">]</span> <span class="o">=</span> <span class="n">_normalize_result_word</span><span class="p">(</span><span class="n">policy_evaluated</span><span class="p">[</span><span class="n">key</span><span class="p">])</span>
|
|
<span class="n">reasons</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="n">spf_aligned</span> <span class="o">=</span> <span class="p">(</span>
|
|
<span class="n">policy_evaluated</span><span class="p">[</span><span class="s2">"spf"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
|
|
<span class="ow">and</span> <span class="n">policy_evaluated</span><span class="p">[</span><span class="s2">"spf"</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s2">"pass"</span>
|
|
<span class="p">)</span>
|
|
<span class="n">dkim_aligned</span> <span class="o">=</span> <span class="p">(</span>
|
|
<span class="n">policy_evaluated</span><span class="p">[</span><span class="s2">"dkim"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
|
|
<span class="ow">and</span> <span class="n">policy_evaluated</span><span class="p">[</span><span class="s2">"dkim"</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s2">"pass"</span>
|
|
<span class="p">)</span>
|
|
<span class="n">dmarc_aligned</span> <span class="o">=</span> <span class="n">spf_aligned</span> <span class="ow">or</span> <span class="n">dkim_aligned</span>
|
|
<span class="n">new_record</span><span class="p">[</span><span class="s2">"alignment"</span><span class="p">]</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
|
|
<span class="n">new_record</span><span class="p">[</span><span class="s2">"alignment"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">]</span> <span class="o">=</span> <span class="n">spf_aligned</span>
|
|
<span class="n">new_record</span><span class="p">[</span><span class="s2">"alignment"</span><span class="p">][</span><span class="s2">"dkim"</span><span class="p">]</span> <span class="o">=</span> <span class="n">dkim_aligned</span>
|
|
<span class="n">new_record</span><span class="p">[</span><span class="s2">"alignment"</span><span class="p">][</span><span class="s2">"dmarc"</span><span class="p">]</span> <span class="o">=</span> <span class="n">dmarc_aligned</span>
|
|
<span class="k">if</span> <span class="s2">"reason"</span> <span class="ow">in</span> <span class="n">policy_evaluated</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">policy_evaluated</span><span class="p">[</span><span class="s2">"reason"</span><span class="p">])</span> <span class="ow">is</span> <span class="nb">list</span><span class="p">:</span>
|
|
<span class="n">reasons</span> <span class="o">=</span> <span class="n">policy_evaluated</span><span class="p">[</span><span class="s2">"reason"</span><span class="p">]</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">reasons</span> <span class="o">=</span> <span class="p">[</span><span class="n">policy_evaluated</span><span class="p">[</span><span class="s2">"reason"</span><span class="p">]]</span>
|
|
<span class="k">for</span> <span class="n">reason</span> <span class="ow">in</span> <span class="n">reasons</span><span class="p">:</span>
|
|
<span class="c1"># `comment` is langAttrString in RFC 9990 — unwrap {"#text": ..., "@lang": ...}</span>
|
|
<span class="n">reason</span><span class="p">[</span><span class="s2">"comment"</span><span class="p">]</span> <span class="o">=</span> <span class="n">_text</span><span class="p">(</span><span class="n">reason</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"comment"</span><span class="p">))</span>
|
|
<span class="n">reason_type</span> <span class="o">=</span> <span class="n">reason</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"type"</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">is_rfc_9990</span> <span class="ow">and</span> <span class="n">reason_type</span> <span class="ow">in</span> <span class="n">RFC_7489_REMOVED_POLICY_OVERRIDE_TYPES</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
|
<span class="s2">"Policy override reason type </span><span class="si">%r</span><span class="s2"> was removed in RFC 9990; "</span>
|
|
<span class="s2">"expected one of </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span>
|
|
<span class="n">reason_type</span><span class="p">,</span>
|
|
<span class="nb">sorted</span><span class="p">(</span><span class="n">RFC_9990_POLICY_OVERRIDE_TYPES</span><span class="p">),</span>
|
|
<span class="p">)</span>
|
|
<span class="k">elif</span> <span class="p">(</span>
|
|
<span class="n">is_rfc_9990</span>
|
|
<span class="ow">and</span> <span class="n">reason_type</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
|
|
<span class="ow">and</span> <span class="n">reason_type</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">RFC_9990_POLICY_OVERRIDE_TYPES</span>
|
|
<span class="p">):</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
|
<span class="s2">"Unknown policy override reason type </span><span class="si">%r</span><span class="s2"> per RFC 9990; "</span>
|
|
<span class="s2">"expected one of </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span>
|
|
<span class="n">reason_type</span><span class="p">,</span>
|
|
<span class="nb">sorted</span><span class="p">(</span><span class="n">RFC_9990_POLICY_OVERRIDE_TYPES</span><span class="p">),</span>
|
|
<span class="p">)</span>
|
|
<span class="n">new_policy_evaluated</span><span class="p">[</span><span class="s2">"policy_override_reasons"</span><span class="p">]</span> <span class="o">=</span> <span class="n">reasons</span>
|
|
<span class="n">new_record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">]</span> <span class="o">=</span> <span class="n">new_policy_evaluated</span>
|
|
<span class="k">if</span> <span class="s2">"identities"</span> <span class="ow">in</span> <span class="n">record</span><span class="p">:</span>
|
|
<span class="n">new_record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"identities"</span><span class="p">]</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">new_record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">]</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
|
<span class="n">new_record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"dkim"</span><span class="p">:</span> <span class="p">[],</span> <span class="s2">"spf"</span><span class="p">:</span> <span class="p">[]}</span>
|
|
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">new_record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"header_from"</span><span class="p">])</span> <span class="ow">is</span> <span class="nb">str</span><span class="p">:</span>
|
|
<span class="n">lowered_from</span> <span class="o">=</span> <span class="n">new_record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"header_from"</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">lowered_from</span> <span class="o">=</span> <span class="s2">""</span>
|
|
<span class="n">new_record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"header_from"</span><span class="p">]</span> <span class="o">=</span> <span class="n">lowered_from</span>
|
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">],</span> <span class="nb">dict</span><span class="p">):</span>
|
|
<span class="n">auth_results</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">]</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
|
<span class="k">if</span> <span class="s2">"spf"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">auth_results</span><span class="p">:</span>
|
|
<span class="n">auth_results</span><span class="p">[</span><span class="s2">"spf"</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="k">if</span> <span class="s2">"dkim"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">auth_results</span><span class="p">:</span>
|
|
<span class="n">auth_results</span><span class="p">[</span><span class="s2">"dkim"</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">auth_results</span> <span class="o">=</span> <span class="n">new_record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">]</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
|
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">auth_results</span><span class="p">[</span><span class="s2">"dkim"</span><span class="p">],</span> <span class="nb">list</span><span class="p">):</span>
|
|
<span class="n">auth_results</span><span class="p">[</span><span class="s2">"dkim"</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="n">auth_results</span><span class="p">[</span><span class="s2">"dkim"</span><span class="p">]]</span>
|
|
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">auth_results</span><span class="p">[</span><span class="s2">"dkim"</span><span class="p">]:</span>
|
|
<span class="k">if</span> <span class="s2">"domain"</span> <span class="ow">in</span> <span class="n">result</span> <span class="ow">and</span> <span class="n">result</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">new_result</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"domain"</span><span class="p">:</span> <span class="n">result</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">]}</span>
|
|
<span class="k">if</span> <span class="s2">"selector"</span> <span class="ow">in</span> <span class="n">result</span> <span class="ow">and</span> <span class="n">result</span><span class="p">[</span><span class="s2">"selector"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">new_result</span><span class="p">[</span><span class="s2">"selector"</span><span class="p">]</span> <span class="o">=</span> <span class="n">result</span><span class="p">[</span><span class="s2">"selector"</span><span class="p">]</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="n">is_rfc_9990</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
|
<span class="s2">"DKIM auth result for </span><span class="si">%r</span><span class="s2"> is missing the 'selector' "</span>
|
|
<span class="s2">"element, which is REQUIRED by RFC 9990"</span><span class="p">,</span>
|
|
<span class="n">result</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">],</span>
|
|
<span class="p">)</span>
|
|
<span class="n">new_result</span><span class="p">[</span><span class="s2">"selector"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"none"</span>
|
|
<span class="k">if</span> <span class="s2">"result"</span> <span class="ow">in</span> <span class="n">result</span> <span class="ow">and</span> <span class="n">result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">new_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">]</span> <span class="o">=</span> <span class="n">_normalize_result_word</span><span class="p">(</span><span class="n">result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">])</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">new_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"none"</span>
|
|
<span class="n">new_result</span><span class="p">[</span><span class="s2">"human_result"</span><span class="p">]</span> <span class="o">=</span> <span class="n">_text</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"human_result"</span><span class="p">))</span>
|
|
<span class="n">new_record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">][</span><span class="s2">"dkim"</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">new_result</span><span class="p">)</span>
|
|
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">auth_results</span><span class="p">[</span><span class="s2">"spf"</span><span class="p">],</span> <span class="nb">list</span><span class="p">):</span>
|
|
<span class="n">auth_results</span><span class="p">[</span><span class="s2">"spf"</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="n">auth_results</span><span class="p">[</span><span class="s2">"spf"</span><span class="p">]]</span>
|
|
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">auth_results</span><span class="p">[</span><span class="s2">"spf"</span><span class="p">]:</span>
|
|
<span class="k">if</span> <span class="s2">"domain"</span> <span class="ow">in</span> <span class="n">result</span> <span class="ow">and</span> <span class="n">result</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">new_result</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"domain"</span><span class="p">:</span> <span class="n">result</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">]}</span>
|
|
<span class="k">if</span> <span class="s2">"scope"</span> <span class="ow">in</span> <span class="n">result</span> <span class="ow">and</span> <span class="n">result</span><span class="p">[</span><span class="s2">"scope"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">new_result</span><span class="p">[</span><span class="s2">"scope"</span><span class="p">]</span> <span class="o">=</span> <span class="n">result</span><span class="p">[</span><span class="s2">"scope"</span><span class="p">]</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">new_result</span><span class="p">[</span><span class="s2">"scope"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"mfrom"</span>
|
|
<span class="k">if</span> <span class="s2">"result"</span> <span class="ow">in</span> <span class="n">result</span> <span class="ow">and</span> <span class="n">result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">new_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">]</span> <span class="o">=</span> <span class="n">_normalize_result_word</span><span class="p">(</span><span class="n">result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">])</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">new_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"none"</span>
|
|
<span class="n">new_result</span><span class="p">[</span><span class="s2">"human_result"</span><span class="p">]</span> <span class="o">=</span> <span class="n">_text</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"human_result"</span><span class="p">))</span>
|
|
<span class="n">new_record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">new_result</span><span class="p">)</span>
|
|
|
|
<span class="c1"># Backfill envelope_from from the last SPF result's domain when the</span>
|
|
<span class="c1"># reporter omitted the identifier or sent it empty.</span>
|
|
<span class="k">if</span> <span class="n">new_record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"envelope_from"</span><span class="p">)</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">envelope_from</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">auth_results</span><span class="p">[</span><span class="s2">"spf"</span><span class="p">])</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
|
<span class="n">spf_result</span> <span class="o">=</span> <span class="n">auth_results</span><span class="p">[</span><span class="s2">"spf"</span><span class="p">][</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
|
|
<span class="k">if</span> <span class="s2">"domain"</span> <span class="ow">in</span> <span class="n">spf_result</span><span class="p">:</span>
|
|
<span class="n">envelope_from</span> <span class="o">=</span> <span class="n">spf_result</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">]</span>
|
|
<span class="k">if</span> <span class="n">envelope_from</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">envelope_from</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">envelope_from</span><span class="p">)</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
|
|
<span class="n">new_record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"envelope_from"</span><span class="p">]</span> <span class="o">=</span> <span class="n">envelope_from</span>
|
|
|
|
<span class="n">envelope_to</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="k">if</span> <span class="s2">"envelope_to"</span> <span class="ow">in</span> <span class="n">new_record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">]:</span>
|
|
<span class="n">envelope_to</span> <span class="o">=</span> <span class="n">new_record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"envelope_to"</span><span class="p">]</span>
|
|
<span class="k">del</span> <span class="n">new_record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"envelope_to"</span><span class="p">]</span>
|
|
|
|
<span class="n">new_record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"envelope_to"</span><span class="p">]</span> <span class="o">=</span> <span class="n">envelope_to</span>
|
|
|
|
<span class="k">return</span> <span class="n">new_record</span>
|
|
|
|
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">_parse_smtp_tls_failure_details</span><span class="p">(</span><span class="n">failure_details</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]):</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">new_failure_details</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
|
|
<span class="s2">"result_type"</span><span class="p">:</span> <span class="n">failure_details</span><span class="p">[</span><span class="s2">"result-type"</span><span class="p">],</span>
|
|
<span class="s2">"failed_session_count"</span><span class="p">:</span> <span class="n">failure_details</span><span class="p">[</span><span class="s2">"failed-session-count"</span><span class="p">],</span>
|
|
<span class="p">}</span>
|
|
|
|
<span class="k">if</span> <span class="s2">"sending-mta-ip"</span> <span class="ow">in</span> <span class="n">failure_details</span><span class="p">:</span>
|
|
<span class="n">new_failure_details</span><span class="p">[</span><span class="s2">"sending_mta_ip"</span><span class="p">]</span> <span class="o">=</span> <span class="n">failure_details</span><span class="p">[</span><span class="s2">"sending-mta-ip"</span><span class="p">]</span>
|
|
<span class="k">if</span> <span class="s2">"receiving-ip"</span> <span class="ow">in</span> <span class="n">failure_details</span><span class="p">:</span>
|
|
<span class="n">new_failure_details</span><span class="p">[</span><span class="s2">"receiving_ip"</span><span class="p">]</span> <span class="o">=</span> <span class="n">failure_details</span><span class="p">[</span><span class="s2">"receiving-ip"</span><span class="p">]</span>
|
|
<span class="k">if</span> <span class="s2">"receiving-mx-hostname"</span> <span class="ow">in</span> <span class="n">failure_details</span><span class="p">:</span>
|
|
<span class="n">new_failure_details</span><span class="p">[</span><span class="s2">"receiving_mx_hostname"</span><span class="p">]</span> <span class="o">=</span> <span class="n">failure_details</span><span class="p">[</span>
|
|
<span class="s2">"receiving-mx-hostname"</span>
|
|
<span class="p">]</span>
|
|
<span class="k">if</span> <span class="s2">"receiving-mx-helo"</span> <span class="ow">in</span> <span class="n">failure_details</span><span class="p">:</span>
|
|
<span class="n">new_failure_details</span><span class="p">[</span><span class="s2">"receiving_mx_helo"</span><span class="p">]</span> <span class="o">=</span> <span class="n">failure_details</span><span class="p">[</span>
|
|
<span class="s2">"receiving-mx-helo"</span>
|
|
<span class="p">]</span>
|
|
<span class="k">if</span> <span class="s2">"additional-info-uri"</span> <span class="ow">in</span> <span class="n">failure_details</span><span class="p">:</span>
|
|
<span class="n">new_failure_details</span><span class="p">[</span><span class="s2">"additional_info_uri"</span><span class="p">]</span> <span class="o">=</span> <span class="n">failure_details</span><span class="p">[</span>
|
|
<span class="s2">"additional-info-uri"</span>
|
|
<span class="p">]</span>
|
|
<span class="k">if</span> <span class="s2">"failure-reason-code"</span> <span class="ow">in</span> <span class="n">failure_details</span><span class="p">:</span>
|
|
<span class="n">new_failure_details</span><span class="p">[</span><span class="s2">"failure_reason_code"</span><span class="p">]</span> <span class="o">=</span> <span class="n">failure_details</span><span class="p">[</span>
|
|
<span class="s2">"failure-reason-code"</span>
|
|
<span class="p">]</span>
|
|
|
|
<span class="k">return</span> <span class="n">new_failure_details</span>
|
|
|
|
<span class="k">except</span> <span class="ne">KeyError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">InvalidSMTPTLSReport</span><span class="p">(</span>
|
|
<span class="sa">f</span><span class="s2">"Missing required failure details field: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">"</span>
|
|
<span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">e</span>
|
|
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">InvalidSMTPTLSReport</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">)</span> <span class="o">+</span> <span class="n">_exc_origin</span><span class="p">(</span><span class="n">e</span><span class="p">))</span> <span class="kn">from</span><span class="w"> </span><span class="nn">e</span>
|
|
|
|
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">_parse_smtp_tls_report_policy</span><span class="p">(</span><span class="n">policy</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]):</span>
|
|
<span class="n">policy_types</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"tlsa"</span><span class="p">,</span> <span class="s2">"sts"</span><span class="p">,</span> <span class="s2">"no-policy-found"</span><span class="p">]</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">policy_domain</span> <span class="o">=</span> <span class="n">policy</span><span class="p">[</span><span class="s2">"policy"</span><span class="p">][</span><span class="s2">"policy-domain"</span><span class="p">]</span>
|
|
<span class="n">policy_type</span> <span class="o">=</span> <span class="n">policy</span><span class="p">[</span><span class="s2">"policy"</span><span class="p">][</span><span class="s2">"policy-type"</span><span class="p">]</span>
|
|
<span class="n">failure_details</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="k">if</span> <span class="n">policy_type</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">policy_types</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">InvalidSMTPTLSReport</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Invalid policy type </span><span class="si">{</span><span class="n">policy_type</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
|
<span class="n">new_policy</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
|
|
<span class="s2">"policy_domain"</span><span class="p">:</span> <span class="n">policy_domain</span><span class="p">,</span>
|
|
<span class="s2">"policy_type"</span><span class="p">:</span> <span class="n">policy_type</span><span class="p">,</span>
|
|
<span class="p">}</span>
|
|
<span class="k">if</span> <span class="s2">"policy-string"</span> <span class="ow">in</span> <span class="n">policy</span><span class="p">[</span><span class="s2">"policy"</span><span class="p">]:</span>
|
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">policy</span><span class="p">[</span><span class="s2">"policy"</span><span class="p">][</span><span class="s2">"policy-string"</span><span class="p">],</span> <span class="nb">list</span><span class="p">):</span>
|
|
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">policy</span><span class="p">[</span><span class="s2">"policy"</span><span class="p">][</span><span class="s2">"policy-string"</span><span class="p">])</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
|
<span class="n">new_policy</span><span class="p">[</span><span class="s2">"policy_strings"</span><span class="p">]</span> <span class="o">=</span> <span class="n">policy</span><span class="p">[</span><span class="s2">"policy"</span><span class="p">][</span><span class="s2">"policy-string"</span><span class="p">]</span>
|
|
|
|
<span class="k">if</span> <span class="s2">"mx-host-pattern"</span> <span class="ow">in</span> <span class="n">policy</span><span class="p">[</span><span class="s2">"policy"</span><span class="p">]:</span>
|
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">policy</span><span class="p">[</span><span class="s2">"policy"</span><span class="p">][</span><span class="s2">"mx-host-pattern"</span><span class="p">],</span> <span class="nb">list</span><span class="p">):</span>
|
|
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">policy</span><span class="p">[</span><span class="s2">"policy"</span><span class="p">][</span><span class="s2">"mx-host-pattern"</span><span class="p">])</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
|
<span class="n">new_policy</span><span class="p">[</span><span class="s2">"mx_host_patterns"</span><span class="p">]</span> <span class="o">=</span> <span class="n">policy</span><span class="p">[</span><span class="s2">"policy"</span><span class="p">][</span><span class="s2">"mx-host-pattern"</span><span class="p">]</span>
|
|
<span class="n">new_policy</span><span class="p">[</span><span class="s2">"successful_session_count"</span><span class="p">]</span> <span class="o">=</span> <span class="n">policy</span><span class="p">[</span><span class="s2">"summary"</span><span class="p">][</span>
|
|
<span class="s2">"total-successful-session-count"</span>
|
|
<span class="p">]</span>
|
|
<span class="n">new_policy</span><span class="p">[</span><span class="s2">"failed_session_count"</span><span class="p">]</span> <span class="o">=</span> <span class="n">policy</span><span class="p">[</span><span class="s2">"summary"</span><span class="p">][</span>
|
|
<span class="s2">"total-failure-session-count"</span>
|
|
<span class="p">]</span>
|
|
<span class="k">if</span> <span class="s2">"failure-details"</span> <span class="ow">in</span> <span class="n">policy</span><span class="p">:</span>
|
|
<span class="k">for</span> <span class="n">details</span> <span class="ow">in</span> <span class="n">policy</span><span class="p">[</span><span class="s2">"failure-details"</span><span class="p">]:</span>
|
|
<span class="n">failure_details</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_parse_smtp_tls_failure_details</span><span class="p">(</span><span class="n">details</span><span class="p">))</span>
|
|
<span class="n">new_policy</span><span class="p">[</span><span class="s2">"failure_details"</span><span class="p">]</span> <span class="o">=</span> <span class="n">failure_details</span>
|
|
|
|
<span class="k">return</span> <span class="n">new_policy</span>
|
|
|
|
<span class="k">except</span> <span class="ne">KeyError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">InvalidSMTPTLSReport</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Missing required policy field: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">e</span>
|
|
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">InvalidSMTPTLSReport</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">)</span> <span class="o">+</span> <span class="n">_exc_origin</span><span class="p">(</span><span class="n">e</span><span class="p">))</span> <span class="kn">from</span><span class="w"> </span><span class="nn">e</span>
|
|
|
|
|
|
<div class="viewcode-block" id="parse_smtp_tls_report_json">
|
|
<a class="viewcode-back" href="../api.html#parsedmarc.parse_smtp_tls_report_json">[docs]</a>
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">parse_smtp_tls_report_json</span><span class="p">(</span><span class="n">report</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="nb">bytes</span><span class="p">)</span> <span class="o">-></span> <span class="n">SMTPTLSReport</span><span class="p">:</span>
|
|
<span class="w"> </span><span class="sd">"""Parses and validates an SMTP TLS report"""</span>
|
|
<span class="n">required_fields</span> <span class="o">=</span> <span class="p">[</span>
|
|
<span class="s2">"organization-name"</span><span class="p">,</span>
|
|
<span class="s2">"date-range"</span><span class="p">,</span>
|
|
<span class="s2">"contact-info"</span><span class="p">,</span>
|
|
<span class="s2">"report-id"</span><span class="p">,</span>
|
|
<span class="s2">"policies"</span><span class="p">,</span>
|
|
<span class="p">]</span>
|
|
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">report</span><span class="p">,</span> <span class="nb">bytes</span><span class="p">):</span>
|
|
<span class="n">report</span> <span class="o">=</span> <span class="n">report</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s2">"utf-8"</span><span class="p">,</span> <span class="n">errors</span><span class="o">=</span><span class="s2">"replace"</span><span class="p">)</span>
|
|
|
|
<span class="n">policies</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="n">report_dict</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">report</span><span class="p">)</span>
|
|
<span class="k">for</span> <span class="n">required_field</span> <span class="ow">in</span> <span class="n">required_fields</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="n">required_field</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">report_dict</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Missing required field: </span><span class="si">{</span><span class="n">required_field</span><span class="si">}</span><span class="s2">]"</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">report_dict</span><span class="p">[</span><span class="s2">"policies"</span><span class="p">],</span> <span class="nb">list</span><span class="p">):</span>
|
|
<span class="n">policies_type</span> <span class="o">=</span> <span class="nb">type</span><span class="p">(</span><span class="n">report_dict</span><span class="p">[</span><span class="s2">"policies"</span><span class="p">])</span>
|
|
<span class="k">raise</span> <span class="n">InvalidSMTPTLSReport</span><span class="p">(</span><span class="sa">f</span><span class="s2">"policies must be a list, not </span><span class="si">{</span><span class="n">policies_type</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
|
<span class="k">for</span> <span class="n">policy</span> <span class="ow">in</span> <span class="n">report_dict</span><span class="p">[</span><span class="s2">"policies"</span><span class="p">]:</span>
|
|
<span class="n">policies</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_parse_smtp_tls_report_policy</span><span class="p">(</span><span class="n">policy</span><span class="p">))</span>
|
|
|
|
<span class="n">new_report</span><span class="p">:</span> <span class="n">SMTPTLSReport</span> <span class="o">=</span> <span class="p">{</span>
|
|
<span class="s2">"organization_name"</span><span class="p">:</span> <span class="n">report_dict</span><span class="p">[</span><span class="s2">"organization-name"</span><span class="p">],</span>
|
|
<span class="s2">"begin_date"</span><span class="p">:</span> <span class="n">report_dict</span><span class="p">[</span><span class="s2">"date-range"</span><span class="p">][</span><span class="s2">"start-datetime"</span><span class="p">],</span>
|
|
<span class="s2">"end_date"</span><span class="p">:</span> <span class="n">report_dict</span><span class="p">[</span><span class="s2">"date-range"</span><span class="p">][</span><span class="s2">"end-datetime"</span><span class="p">],</span>
|
|
<span class="s2">"contact_info"</span><span class="p">:</span> <span class="n">report_dict</span><span class="p">[</span><span class="s2">"contact-info"</span><span class="p">],</span>
|
|
<span class="s2">"report_id"</span><span class="p">:</span> <span class="n">report_dict</span><span class="p">[</span><span class="s2">"report-id"</span><span class="p">],</span>
|
|
<span class="s2">"policies"</span><span class="p">:</span> <span class="n">policies</span><span class="p">,</span>
|
|
<span class="p">}</span>
|
|
|
|
<span class="k">return</span> <span class="n">new_report</span>
|
|
|
|
<span class="k">except</span> <span class="ne">KeyError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">InvalidSMTPTLSReport</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Missing required field: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">e</span>
|
|
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">InvalidSMTPTLSReport</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">)</span> <span class="o">+</span> <span class="n">_exc_origin</span><span class="p">(</span><span class="n">e</span><span class="p">))</span> <span class="kn">from</span><span class="w"> </span><span class="nn">e</span></div>
|
|
|
|
|
|
|
|
<div class="viewcode-block" id="parsed_smtp_tls_reports_to_csv_rows">
|
|
<a class="viewcode-back" href="../api.html#parsedmarc.parsed_smtp_tls_reports_to_csv_rows">[docs]</a>
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">parsed_smtp_tls_reports_to_csv_rows</span><span class="p">(</span>
|
|
<span class="n">reports</span><span class="p">:</span> <span class="n">SMTPTLSReport</span> <span class="o">|</span> <span class="nb">list</span><span class="p">[</span><span class="n">SMTPTLSReport</span><span class="p">],</span>
|
|
<span class="p">)</span> <span class="o">-></span> <span class="nb">list</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]]:</span>
|
|
<span class="w"> </span><span class="sd">"""Converts one oor more parsed SMTP TLS reports into a list of single</span>
|
|
<span class="sd"> layer dict objects suitable for use in a CSV"""</span>
|
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">reports</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
|
|
<span class="n">reports</span> <span class="o">=</span> <span class="p">[</span><span class="n">reports</span><span class="p">]</span>
|
|
|
|
<span class="n">rows</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="k">for</span> <span class="n">report</span> <span class="ow">in</span> <span class="n">reports</span><span class="p">:</span>
|
|
<span class="n">common_fields</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
|
|
<span class="s2">"organization_name"</span><span class="p">:</span> <span class="n">report</span><span class="p">[</span><span class="s2">"organization_name"</span><span class="p">],</span>
|
|
<span class="s2">"begin_date"</span><span class="p">:</span> <span class="n">report</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">],</span>
|
|
<span class="s2">"end_date"</span><span class="p">:</span> <span class="n">report</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">],</span>
|
|
<span class="s2">"report_id"</span><span class="p">:</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_id"</span><span class="p">],</span>
|
|
<span class="p">}</span>
|
|
<span class="n">record</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="n">common_fields</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
|
<span class="k">for</span> <span class="n">policy</span> <span class="ow">in</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policies"</span><span class="p">]:</span>
|
|
<span class="k">if</span> <span class="s2">"policy_strings"</span> <span class="ow">in</span> <span class="n">policy</span><span class="p">:</span>
|
|
<span class="n">record</span><span class="p">[</span><span class="s2">"policy_strings"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"|"</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">policy</span><span class="p">[</span><span class="s2">"policy_strings"</span><span class="p">])</span>
|
|
<span class="k">if</span> <span class="s2">"mx_host_patterns"</span> <span class="ow">in</span> <span class="n">policy</span><span class="p">:</span>
|
|
<span class="n">record</span><span class="p">[</span><span class="s2">"mx_host_patterns"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"|"</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">policy</span><span class="p">[</span><span class="s2">"mx_host_patterns"</span><span class="p">])</span>
|
|
<span class="n">successful_record</span> <span class="o">=</span> <span class="n">record</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
|
<span class="n">successful_record</span><span class="p">[</span><span class="s2">"policy_domain"</span><span class="p">]</span> <span class="o">=</span> <span class="n">policy</span><span class="p">[</span><span class="s2">"policy_domain"</span><span class="p">]</span>
|
|
<span class="n">successful_record</span><span class="p">[</span><span class="s2">"policy_type"</span><span class="p">]</span> <span class="o">=</span> <span class="n">policy</span><span class="p">[</span><span class="s2">"policy_type"</span><span class="p">]</span>
|
|
<span class="n">successful_record</span><span class="p">[</span><span class="s2">"successful_session_count"</span><span class="p">]</span> <span class="o">=</span> <span class="n">policy</span><span class="p">[</span>
|
|
<span class="s2">"successful_session_count"</span>
|
|
<span class="p">]</span>
|
|
<span class="n">successful_record</span><span class="p">[</span><span class="s2">"failed_session_count"</span><span class="p">]</span> <span class="o">=</span> <span class="n">policy</span><span class="p">[</span><span class="s2">"failed_session_count"</span><span class="p">]</span>
|
|
<span class="n">rows</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">successful_record</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="s2">"failure_details"</span> <span class="ow">in</span> <span class="n">policy</span><span class="p">:</span>
|
|
<span class="k">for</span> <span class="n">failure_details</span> <span class="ow">in</span> <span class="n">policy</span><span class="p">[</span><span class="s2">"failure_details"</span><span class="p">]:</span>
|
|
<span class="n">failure_record</span> <span class="o">=</span> <span class="n">record</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
|
<span class="k">for</span> <span class="n">key</span> <span class="ow">in</span> <span class="n">failure_details</span><span class="o">.</span><span class="n">keys</span><span class="p">():</span>
|
|
<span class="n">failure_record</span><span class="p">[</span><span class="n">key</span><span class="p">]</span> <span class="o">=</span> <span class="n">failure_details</span><span class="p">[</span><span class="n">key</span><span class="p">]</span>
|
|
<span class="n">rows</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">failure_record</span><span class="p">)</span>
|
|
|
|
<span class="k">return</span> <span class="n">rows</span></div>
|
|
|
|
|
|
|
|
<div class="viewcode-block" id="parsed_smtp_tls_reports_to_csv">
|
|
<a class="viewcode-back" href="../api.html#parsedmarc.parsed_smtp_tls_reports_to_csv">[docs]</a>
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">parsed_smtp_tls_reports_to_csv</span><span class="p">(</span>
|
|
<span class="n">reports</span><span class="p">:</span> <span class="n">SMTPTLSReport</span> <span class="o">|</span> <span class="nb">list</span><span class="p">[</span><span class="n">SMTPTLSReport</span><span class="p">],</span>
|
|
<span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
|
<span class="w"> </span><span class="sd">"""</span>
|
|
<span class="sd"> Converts one or more parsed SMTP TLS reports to flat CSV format, including</span>
|
|
<span class="sd"> headers</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> reports: A parsed aggregate report or list of parsed aggregate reports</span>
|
|
|
|
<span class="sd"> Returns:</span>
|
|
<span class="sd"> str: Parsed aggregate report data in flat CSV format, including headers</span>
|
|
<span class="sd"> """</span>
|
|
|
|
<span class="n">fields</span> <span class="o">=</span> <span class="p">[</span>
|
|
<span class="s2">"organization_name"</span><span class="p">,</span>
|
|
<span class="s2">"begin_date"</span><span class="p">,</span>
|
|
<span class="s2">"end_date"</span><span class="p">,</span>
|
|
<span class="s2">"report_id"</span><span class="p">,</span>
|
|
<span class="s2">"result_type"</span><span class="p">,</span>
|
|
<span class="s2">"successful_session_count"</span><span class="p">,</span>
|
|
<span class="s2">"failed_session_count"</span><span class="p">,</span>
|
|
<span class="s2">"policy_domain"</span><span class="p">,</span>
|
|
<span class="s2">"policy_type"</span><span class="p">,</span>
|
|
<span class="s2">"policy_strings"</span><span class="p">,</span>
|
|
<span class="s2">"mx_host_patterns"</span><span class="p">,</span>
|
|
<span class="s2">"sending_mta_ip"</span><span class="p">,</span>
|
|
<span class="s2">"receiving_ip"</span><span class="p">,</span>
|
|
<span class="s2">"receiving_mx_hostname"</span><span class="p">,</span>
|
|
<span class="s2">"receiving_mx_helo"</span><span class="p">,</span>
|
|
<span class="s2">"additional_info_uri"</span><span class="p">,</span>
|
|
<span class="s2">"failure_reason_code"</span><span class="p">,</span>
|
|
<span class="p">]</span>
|
|
|
|
<span class="n">csv_file_object</span> <span class="o">=</span> <span class="n">StringIO</span><span class="p">(</span><span class="n">newline</span><span class="o">=</span><span class="s2">"</span><span class="se">\n</span><span class="s2">"</span><span class="p">)</span>
|
|
<span class="n">writer</span> <span class="o">=</span> <span class="n">DictWriter</span><span class="p">(</span><span class="n">csv_file_object</span><span class="p">,</span> <span class="n">fields</span><span class="p">)</span>
|
|
<span class="n">writer</span><span class="o">.</span><span class="n">writeheader</span><span class="p">()</span>
|
|
|
|
<span class="n">rows</span> <span class="o">=</span> <span class="n">parsed_smtp_tls_reports_to_csv_rows</span><span class="p">(</span><span class="n">reports</span><span class="p">)</span>
|
|
|
|
<span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="n">rows</span><span class="p">:</span>
|
|
<span class="n">writer</span><span class="o">.</span><span class="n">writerow</span><span class="p">(</span><span class="n">row</span><span class="p">)</span>
|
|
<span class="n">csv_file_object</span><span class="o">.</span><span class="n">flush</span><span class="p">()</span>
|
|
|
|
<span class="k">return</span> <span class="n">csv_file_object</span><span class="o">.</span><span class="n">getvalue</span><span class="p">()</span></div>
|
|
|
|
|
|
|
|
<div class="viewcode-block" id="parse_aggregate_report_xml">
|
|
<a class="viewcode-back" href="../api.html#parsedmarc.parse_aggregate_report_xml">[docs]</a>
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">parse_aggregate_report_xml</span><span class="p">(</span>
|
|
<span class="n">xml</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="nb">bytes</span><span class="p">,</span>
|
|
<span class="o">*</span><span class="p">,</span>
|
|
<span class="n">ip_db_path</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">always_use_local_files</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
|
<span class="n">reverse_dns_map_path</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">reverse_dns_map_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">offline</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
|
<span class="n">nameservers</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">timeout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="n">DEFAULT_DNS_TIMEOUT</span><span class="p">,</span>
|
|
<span class="n">retries</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="n">DEFAULT_DNS_MAX_RETRIES</span><span class="p">,</span>
|
|
<span class="n">keep_alive</span><span class="p">:</span> <span class="n">Callable</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">normalize_timespan_threshold_hours</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">24.0</span><span class="p">,</span>
|
|
<span class="n">config</span><span class="p">:</span> <span class="n">ParserConfig</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="p">)</span> <span class="o">-></span> <span class="n">AggregateReport</span><span class="p">:</span>
|
|
<span class="w"> </span><span class="sd">"""Parses a DMARC XML report string and returns a consistent dict</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> xml (str | bytes): DMARC aggregate report XML (bytes are decoded</span>
|
|
<span class="sd"> with errors ignored)</span>
|
|
<span class="sd"> ip_db_path (str): Path to a MMDB file from IPinfo, MaxMind, or DBIP</span>
|
|
<span class="sd"> always_use_local_files (bool): Do not download files</span>
|
|
<span class="sd"> reverse_dns_map_path (str): Path to a reverse DNS map file</span>
|
|
<span class="sd"> reverse_dns_map_url (str): URL to a reverse DNS map file</span>
|
|
<span class="sd"> offline (bool): Do not query online for geolocation or DNS</span>
|
|
<span class="sd"> nameservers (list): A list of one or more nameservers to use</span>
|
|
<span class="sd"> (Cloudflare's public DNS resolvers by default)</span>
|
|
<span class="sd"> timeout (float): Sets the DNS timeout in seconds</span>
|
|
<span class="sd"> retries (int): Number of times to retry DNS queries on timeout or</span>
|
|
<span class="sd"> other transient errors</span>
|
|
<span class="sd"> keep_alive (callable): Keep alive function. Not part of ``config``;</span>
|
|
<span class="sd"> always applies.</span>
|
|
<span class="sd"> normalize_timespan_threshold_hours (float): Normalize timespans beyond this</span>
|
|
<span class="sd"> config (ParserConfig): a single object carrying all parsing and</span>
|
|
<span class="sd"> enrichment options plus the caches; when provided, the</span>
|
|
<span class="sd"> individual option keyword arguments listed above are ignored in</span>
|
|
<span class="sd"> favor of the config's values.</span>
|
|
|
|
<span class="sd"> Returns:</span>
|
|
<span class="sd"> dict: The parsed aggregate DMARC report</span>
|
|
<span class="sd"> """</span>
|
|
<span class="n">cfg</span> <span class="o">=</span> <span class="n">_resolve_config</span><span class="p">(</span>
|
|
<span class="n">config</span><span class="p">,</span>
|
|
<span class="n">offline</span><span class="o">=</span><span class="n">offline</span><span class="p">,</span>
|
|
<span class="n">ip_db_path</span><span class="o">=</span><span class="n">ip_db_path</span><span class="p">,</span>
|
|
<span class="n">always_use_local_files</span><span class="o">=</span><span class="n">always_use_local_files</span><span class="p">,</span>
|
|
<span class="n">reverse_dns_map_path</span><span class="o">=</span><span class="n">reverse_dns_map_path</span><span class="p">,</span>
|
|
<span class="n">reverse_dns_map_url</span><span class="o">=</span><span class="n">reverse_dns_map_url</span><span class="p">,</span>
|
|
<span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
|
|
<span class="n">dns_timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">,</span>
|
|
<span class="n">dns_retries</span><span class="o">=</span><span class="n">retries</span><span class="p">,</span>
|
|
<span class="n">normalize_timespan_threshold_hours</span><span class="o">=</span><span class="n">normalize_timespan_threshold_hours</span><span class="p">,</span>
|
|
<span class="p">)</span>
|
|
<span class="n">errors</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="c1"># Parse XML and recover from errors</span>
|
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">xml</span><span class="p">,</span> <span class="nb">bytes</span><span class="p">):</span>
|
|
<span class="n">xml</span> <span class="o">=</span> <span class="n">xml</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="n">errors</span><span class="o">=</span><span class="s2">"ignore"</span><span class="p">)</span>
|
|
|
|
<span class="c1"># Detect the XML namespace before any rewriting strips it. The dmarc-2.0</span>
|
|
<span class="c1"># namespace is one of the indicators for an RFC 9990 report but it is</span>
|
|
<span class="c1"># NOT a reliable sole discriminator: the <version> element value is</span>
|
|
<span class="c1"># ambiguous (RFC 9990's appendix sample uses <version>1.0</version></span>
|
|
<span class="c1"># inside the dmarc-2.0 namespace), and real-world reporters frequently</span>
|
|
<span class="c1"># emit RFC 9990-shaped reports without declaring the namespace at all.</span>
|
|
<span class="c1"># The final `is_rfc_9990` decision is made post-parse so that</span>
|
|
<span class="c1"># RFC 9990-only fields (np, testing, discovery_method, generator,</span>
|
|
<span class="c1"># human_result) can also vote it in.</span>
|
|
<span class="n">xml_namespace</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="n">namespace_match</span> <span class="o">=</span> <span class="n">xml_namespace_regex</span><span class="o">.</span><span class="n">search</span><span class="p">(</span><span class="n">xml</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">namespace_match</span><span class="p">:</span>
|
|
<span class="n">xml_namespace</span> <span class="o">=</span> <span class="n">namespace_match</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
|
|
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">xmltodict</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">xml</span><span class="p">)[</span><span class="s2">"feedback"</span><span class="p">]</span>
|
|
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
|
<span class="n">errors</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Invalid XML: </span><span class="si">{</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">tree</span> <span class="o">=</span> <span class="n">etree</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span>
|
|
<span class="n">BytesIO</span><span class="p">(</span><span class="n">xml</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s2">"utf-8"</span><span class="p">)),</span>
|
|
<span class="n">etree</span><span class="o">.</span><span class="n">XMLParser</span><span class="p">(</span><span class="n">recover</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">resolve_entities</span><span class="o">=</span><span class="kc">False</span><span class="p">),</span>
|
|
<span class="p">)</span>
|
|
<span class="n">s</span> <span class="o">=</span> <span class="n">etree</span><span class="o">.</span><span class="n">tostring</span><span class="p">(</span><span class="n">tree</span><span class="p">)</span>
|
|
<span class="n">xml</span> <span class="o">=</span> <span class="s2">""</span> <span class="k">if</span> <span class="n">s</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">s</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s2">"utf-8"</span><span class="p">)</span>
|
|
<span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span>
|
|
<span class="n">xml</span> <span class="o">=</span> <span class="s2">"<a/>"</span>
|
|
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="c1"># Replace XML header (sometimes they are invalid)</span>
|
|
<span class="n">xml</span> <span class="o">=</span> <span class="n">xml_header_regex</span><span class="o">.</span><span class="n">sub</span><span class="p">(</span><span class="s1">'<?xml version="1.0"?>'</span><span class="p">,</span> <span class="n">xml</span><span class="p">)</span>
|
|
|
|
<span class="c1"># Remove invalid schema tags</span>
|
|
<span class="n">xml</span> <span class="o">=</span> <span class="n">xml_schema_regex</span><span class="o">.</span><span class="n">sub</span><span class="p">(</span><span class="s2">""</span><span class="p">,</span> <span class="n">xml</span><span class="p">)</span>
|
|
|
|
<span class="n">report</span> <span class="o">=</span> <span class="n">xmltodict</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">xml</span><span class="p">)[</span><span class="s2">"feedback"</span><span class="p">]</span>
|
|
<span class="n">report_metadata</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">]</span>
|
|
<span class="c1"># <email> is xs:string in both RFC 7489 and RFC 9990, but defensive</span>
|
|
<span class="c1"># parsing in the wild: some reporters emit it with an xml:lang or</span>
|
|
<span class="c1"># similar attribute, which xmltodict turns into a dict.</span>
|
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">report_metadata</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"email"</span><span class="p">),</span> <span class="nb">dict</span><span class="p">):</span>
|
|
<span class="n">unwrapped</span> <span class="o">=</span> <span class="n">_text</span><span class="p">(</span><span class="n">report_metadata</span><span class="p">[</span><span class="s2">"email"</span><span class="p">])</span>
|
|
<span class="k">if</span> <span class="n">unwrapped</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
|
<span class="s2">"Discarding malformed <email> in report_metadata: </span><span class="si">%r</span><span class="s2">"</span><span class="p">,</span>
|
|
<span class="n">report_metadata</span><span class="p">[</span><span class="s2">"email"</span><span class="p">],</span>
|
|
<span class="p">)</span>
|
|
<span class="n">report_metadata</span><span class="p">[</span><span class="s2">"email"</span><span class="p">]</span> <span class="o">=</span> <span class="n">unwrapped</span>
|
|
<span class="n">schema</span> <span class="o">=</span> <span class="s2">"draft"</span>
|
|
<span class="k">if</span> <span class="s2">"version"</span> <span class="ow">in</span> <span class="n">report</span><span class="p">:</span>
|
|
<span class="n">schema</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"version"</span><span class="p">]</span>
|
|
<span class="n">new_report</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
|
|
<span class="s2">"xml_schema"</span><span class="p">:</span> <span class="n">schema</span><span class="p">,</span>
|
|
<span class="s2">"xml_namespace"</span><span class="p">:</span> <span class="n">xml_namespace</span><span class="p">,</span>
|
|
<span class="p">}</span>
|
|
<span class="n">new_report_metadata</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
|
|
<span class="k">if</span> <span class="n">report_metadata</span><span class="p">[</span><span class="s2">"org_name"</span><span class="p">]</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="n">report_metadata</span><span class="p">[</span><span class="s2">"email"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">report_metadata</span><span class="p">[</span><span class="s2">"org_name"</span><span class="p">]</span> <span class="o">=</span> <span class="n">report_metadata</span><span class="p">[</span><span class="s2">"email"</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">"@"</span><span class="p">)[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
|
|
<span class="n">org_name</span> <span class="o">=</span> <span class="n">report_metadata</span><span class="p">[</span><span class="s2">"org_name"</span><span class="p">]</span>
|
|
<span class="k">if</span> <span class="n">org_name</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="s2">" "</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">org_name</span><span class="p">:</span>
|
|
<span class="n">new_org_name</span> <span class="o">=</span> <span class="n">get_base_domain</span><span class="p">(</span><span class="n">org_name</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">new_org_name</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">org_name</span> <span class="o">=</span> <span class="n">new_org_name</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">org_name</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Could not parse org_name from XML.</span><span class="se">\r\n</span><span class="si">{</span><span class="n">report</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
|
<span class="k">raise</span> <span class="ne">KeyError</span><span class="p">(</span>
|
|
<span class="s2">"Organization name is missing. This field is a requirement "</span>
|
|
<span class="s2">"for saving the report"</span>
|
|
<span class="p">)</span>
|
|
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"org_name"</span><span class="p">]</span> <span class="o">=</span> <span class="n">org_name</span>
|
|
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"org_email"</span><span class="p">]</span> <span class="o">=</span> <span class="n">report_metadata</span><span class="p">[</span><span class="s2">"email"</span><span class="p">]</span>
|
|
<span class="c1"># extra_contact_info is langAttrString in RFC 9990 (xs:string in</span>
|
|
<span class="c1"># RFC 7489) — unwrap {"#text": ..., "@lang": ...} if present.</span>
|
|
<span class="n">extra</span> <span class="o">=</span> <span class="n">_text</span><span class="p">(</span><span class="n">report_metadata</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"extra_contact_info"</span><span class="p">))</span>
|
|
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"org_extra_contact_info"</span><span class="p">]</span> <span class="o">=</span> <span class="n">extra</span>
|
|
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"report_id"</span><span class="p">]</span> <span class="o">=</span> <span class="n">report_metadata</span><span class="p">[</span><span class="s2">"report_id"</span><span class="p">]</span>
|
|
<span class="n">report_id</span> <span class="o">=</span> <span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"report_id"</span><span class="p">]</span>
|
|
<span class="n">report_id</span> <span class="o">=</span> <span class="n">report_id</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"<"</span><span class="p">,</span> <span class="s2">""</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">">"</span><span class="p">,</span> <span class="s2">""</span><span class="p">)</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">"@"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
|
|
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"report_id"</span><span class="p">]</span> <span class="o">=</span> <span class="n">report_id</span>
|
|
<span class="n">date_range</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">][</span><span class="s2">"date_range"</span><span class="p">]</span>
|
|
|
|
<span class="n">begin_ts</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">date_range</span><span class="p">[</span><span class="s2">"begin"</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">"."</span><span class="p">)[</span><span class="mi">0</span><span class="p">])</span>
|
|
<span class="n">end_ts</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">date_range</span><span class="p">[</span><span class="s2">"end"</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">"."</span><span class="p">)[</span><span class="mi">0</span><span class="p">])</span>
|
|
<span class="n">span_seconds</span> <span class="o">=</span> <span class="n">end_ts</span> <span class="o">-</span> <span class="n">begin_ts</span>
|
|
|
|
<span class="n">normalize_timespan</span> <span class="o">=</span> <span class="p">(</span>
|
|
<span class="n">span_seconds</span> <span class="o">></span> <span class="n">cfg</span><span class="o">.</span><span class="n">normalize_timespan_threshold_hours</span> <span class="o">*</span> <span class="mi">3600</span>
|
|
<span class="p">)</span>
|
|
|
|
<span class="n">date_range</span><span class="p">[</span><span class="s2">"begin"</span><span class="p">]</span> <span class="o">=</span> <span class="n">timestamp_to_human</span><span class="p">(</span><span class="n">begin_ts</span><span class="p">)</span>
|
|
<span class="n">date_range</span><span class="p">[</span><span class="s2">"end"</span><span class="p">]</span> <span class="o">=</span> <span class="n">timestamp_to_human</span><span class="p">(</span><span class="n">end_ts</span><span class="p">)</span>
|
|
|
|
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">]</span> <span class="o">=</span> <span class="n">date_range</span><span class="p">[</span><span class="s2">"begin"</span><span class="p">]</span>
|
|
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">]</span> <span class="o">=</span> <span class="n">date_range</span><span class="p">[</span><span class="s2">"end"</span><span class="p">]</span>
|
|
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"timespan_requires_normalization"</span><span class="p">]</span> <span class="o">=</span> <span class="n">normalize_timespan</span>
|
|
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"original_timespan_seconds"</span><span class="p">]</span> <span class="o">=</span> <span class="n">span_seconds</span>
|
|
<span class="n">begin_dt</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span>
|
|
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span>
|
|
<span class="p">)</span>
|
|
<span class="n">end_dt</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span>
|
|
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span>
|
|
<span class="p">)</span>
|
|
|
|
<span class="c1"># <error> is langAttrString in RFC 9990 (xs:string in RFC 7489) and</span>
|
|
<span class="c1"># was cardinality-narrowed from "unbounded" to "1" in RFC 9990, but</span>
|
|
<span class="c1"># the parser still accepts a list for backward compatibility with</span>
|
|
<span class="c1"># RFC 7489 reports that carry multiple errors.</span>
|
|
<span class="k">if</span> <span class="s2">"error"</span> <span class="ow">in</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">]:</span>
|
|
<span class="n">raw_errors</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">][</span><span class="s2">"error"</span><span class="p">]</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">raw_errors</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
|
|
<span class="n">raw_errors</span> <span class="o">=</span> <span class="p">[</span><span class="n">raw_errors</span><span class="p">]</span>
|
|
<span class="n">errors</span> <span class="o">=</span> <span class="p">[</span><span class="n">text</span> <span class="k">for</span> <span class="n">text</span> <span class="ow">in</span> <span class="p">(</span><span class="n">_text</span><span class="p">(</span><span class="n">e</span><span class="p">)</span> <span class="k">for</span> <span class="n">e</span> <span class="ow">in</span> <span class="n">raw_errors</span><span class="p">)</span> <span class="k">if</span> <span class="n">text</span><span class="p">]</span>
|
|
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"errors"</span><span class="p">]</span> <span class="o">=</span> <span class="n">errors</span>
|
|
<span class="c1"># <generator> is a plain xs:string in RFC 9990 but apply _text() so</span>
|
|
<span class="c1"># a malformed reporter that decorates it with attributes still</span>
|
|
<span class="c1"># yields a string instead of breaking downstream consumers.</span>
|
|
<span class="n">generator</span> <span class="o">=</span> <span class="n">_text</span><span class="p">(</span><span class="n">report_metadata</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"generator"</span><span class="p">))</span>
|
|
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"generator"</span><span class="p">]</span> <span class="o">=</span> <span class="n">generator</span>
|
|
<span class="n">new_report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">]</span> <span class="o">=</span> <span class="n">new_report_metadata</span>
|
|
<span class="n">records</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="n">policy_published</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">]</span>
|
|
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">policy_published</span><span class="p">)</span> <span class="ow">is</span> <span class="nb">list</span><span class="p">:</span>
|
|
<span class="n">policy_published</span> <span class="o">=</span> <span class="n">policy_published</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
|
|
|
|
<span class="c1"># Final RFC 9990 detection: the dmarc-2.0 XML namespace OR any</span>
|
|
<span class="c1"># RFC 9990-only field. Real-world reporters that follow the schema</span>
|
|
<span class="c1"># without declaring the namespace still get RFC 9990-aware</span>
|
|
<span class="c1"># warnings (missing DKIM selector, removed override-reason types,</span>
|
|
<span class="c1"># etc.) and a truthful audit trail in `xml_namespace`.</span>
|
|
<span class="n">rfc_9990_only_policy_fields</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"np"</span><span class="p">,</span> <span class="s2">"testing"</span><span class="p">,</span> <span class="s2">"discovery_method"</span><span class="p">}</span>
|
|
<span class="n">is_rfc_9990</span> <span class="o">=</span> <span class="p">(</span>
|
|
<span class="n">xml_namespace</span> <span class="o">==</span> <span class="n">RFC_9990_NAMESPACE</span>
|
|
<span class="ow">or</span> <span class="s2">"generator"</span> <span class="ow">in</span> <span class="n">report_metadata</span>
|
|
<span class="ow">or</span> <span class="nb">any</span><span class="p">(</span><span class="n">f</span> <span class="ow">in</span> <span class="n">policy_published</span> <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="n">rfc_9990_only_policy_fields</span><span class="p">)</span>
|
|
<span class="p">)</span>
|
|
<span class="n">new_policy_published</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
|
|
<span class="n">new_policy_published</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">]</span> <span class="o">=</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">]</span>
|
|
<span class="n">adkim</span> <span class="o">=</span> <span class="s2">"r"</span>
|
|
<span class="k">if</span> <span class="s2">"adkim"</span> <span class="ow">in</span> <span class="n">policy_published</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"adkim"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">adkim</span> <span class="o">=</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"adkim"</span><span class="p">]</span>
|
|
<span class="n">new_policy_published</span><span class="p">[</span><span class="s2">"adkim"</span><span class="p">]</span> <span class="o">=</span> <span class="n">adkim</span>
|
|
<span class="n">aspf</span> <span class="o">=</span> <span class="s2">"r"</span>
|
|
<span class="k">if</span> <span class="s2">"aspf"</span> <span class="ow">in</span> <span class="n">policy_published</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"aspf"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">aspf</span> <span class="o">=</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"aspf"</span><span class="p">]</span>
|
|
<span class="n">new_policy_published</span><span class="p">[</span><span class="s2">"aspf"</span><span class="p">]</span> <span class="o">=</span> <span class="n">aspf</span>
|
|
<span class="n">new_policy_published</span><span class="p">[</span><span class="s2">"p"</span><span class="p">]</span> <span class="o">=</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"p"</span><span class="p">]</span>
|
|
<span class="n">sp</span> <span class="o">=</span> <span class="n">new_policy_published</span><span class="p">[</span><span class="s2">"p"</span><span class="p">]</span>
|
|
<span class="k">if</span> <span class="s2">"sp"</span> <span class="ow">in</span> <span class="n">policy_published</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"sp"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">sp</span> <span class="o">=</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"sp"</span><span class="p">]</span>
|
|
<span class="n">new_policy_published</span><span class="p">[</span><span class="s2">"sp"</span><span class="p">]</span> <span class="o">=</span> <span class="n">sp</span>
|
|
<span class="n">pct</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="k">if</span> <span class="s2">"pct"</span> <span class="ow">in</span> <span class="n">policy_published</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"pct"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">pct</span> <span class="o">=</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"pct"</span><span class="p">]</span>
|
|
<span class="n">new_policy_published</span><span class="p">[</span><span class="s2">"pct"</span><span class="p">]</span> <span class="o">=</span> <span class="n">pct</span>
|
|
<span class="n">fo</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="k">if</span> <span class="s2">"fo"</span> <span class="ow">in</span> <span class="n">policy_published</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"fo"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">fo</span> <span class="o">=</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"fo"</span><span class="p">]</span>
|
|
<span class="n">new_policy_published</span><span class="p">[</span><span class="s2">"fo"</span><span class="p">]</span> <span class="o">=</span> <span class="n">fo</span>
|
|
<span class="n">np_</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="k">if</span> <span class="s2">"np"</span> <span class="ow">in</span> <span class="n">policy_published</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"np"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">np_</span> <span class="o">=</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"np"</span><span class="p">]</span>
|
|
<span class="k">if</span> <span class="n">np_</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">(</span><span class="s2">"none"</span><span class="p">,</span> <span class="s2">"quarantine"</span><span class="p">,</span> <span class="s2">"reject"</span><span class="p">):</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Invalid np value: </span><span class="si">{</span><span class="n">np_</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
|
<span class="n">new_policy_published</span><span class="p">[</span><span class="s2">"np"</span><span class="p">]</span> <span class="o">=</span> <span class="n">np_</span>
|
|
<span class="n">testing</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="k">if</span> <span class="s2">"testing"</span> <span class="ow">in</span> <span class="n">policy_published</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"testing"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">testing</span> <span class="o">=</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"testing"</span><span class="p">]</span>
|
|
<span class="k">if</span> <span class="n">testing</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">(</span><span class="s2">"n"</span><span class="p">,</span> <span class="s2">"y"</span><span class="p">):</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Invalid testing value: </span><span class="si">{</span><span class="n">testing</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
|
<span class="n">new_policy_published</span><span class="p">[</span><span class="s2">"testing"</span><span class="p">]</span> <span class="o">=</span> <span class="n">testing</span>
|
|
<span class="n">discovery_method</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="k">if</span> <span class="s2">"discovery_method"</span> <span class="ow">in</span> <span class="n">policy_published</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"discovery_method"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">discovery_method</span> <span class="o">=</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"discovery_method"</span><span class="p">]</span>
|
|
<span class="k">if</span> <span class="n">discovery_method</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">(</span><span class="s2">"psl"</span><span class="p">,</span> <span class="s2">"treewalk"</span><span class="p">):</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
|
<span class="sa">f</span><span class="s2">"Invalid discovery_method value: </span><span class="si">{</span><span class="n">discovery_method</span><span class="si">}</span><span class="s2">"</span>
|
|
<span class="p">)</span>
|
|
<span class="n">new_policy_published</span><span class="p">[</span><span class="s2">"discovery_method"</span><span class="p">]</span> <span class="o">=</span> <span class="n">discovery_method</span>
|
|
<span class="n">new_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">]</span> <span class="o">=</span> <span class="n">new_policy_published</span>
|
|
|
|
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">"record"</span><span class="p">])</span> <span class="ow">is</span> <span class="nb">list</span><span class="p">:</span>
|
|
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">"record"</span><span class="p">])):</span>
|
|
<span class="k">if</span> <span class="n">keep_alive</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">i</span> <span class="o">></span> <span class="mi">0</span> <span class="ow">and</span> <span class="n">i</span> <span class="o">%</span> <span class="mi">20</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Sending keepalive cmd"</span><span class="p">)</span>
|
|
<span class="n">keep_alive</span><span class="p">()</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Processed </span><span class="si">{}</span><span class="s2">/</span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">i</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">"record"</span><span class="p">])))</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">report_record</span> <span class="o">=</span> <span class="n">_parse_report_record</span><span class="p">(</span>
|
|
<span class="n">report</span><span class="p">[</span><span class="s2">"record"</span><span class="p">][</span><span class="n">i</span><span class="p">],</span>
|
|
<span class="n">config</span><span class="o">=</span><span class="n">cfg</span><span class="p">,</span>
|
|
<span class="n">is_rfc_9990</span><span class="o">=</span><span class="n">is_rfc_9990</span><span class="p">,</span>
|
|
<span class="p">)</span>
|
|
<span class="n">_append_parsed_record</span><span class="p">(</span>
|
|
<span class="n">parsed_record</span><span class="o">=</span><span class="n">report_record</span><span class="p">,</span>
|
|
<span class="n">records</span><span class="o">=</span><span class="n">records</span><span class="p">,</span>
|
|
<span class="n">begin_dt</span><span class="o">=</span><span class="n">begin_dt</span><span class="p">,</span>
|
|
<span class="n">end_dt</span><span class="o">=</span><span class="n">end_dt</span><span class="p">,</span>
|
|
<span class="n">normalize</span><span class="o">=</span><span class="n">normalize_timespan</span><span class="p">,</span>
|
|
<span class="p">)</span>
|
|
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Could not parse record: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
|
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">report_record</span> <span class="o">=</span> <span class="n">_parse_report_record</span><span class="p">(</span>
|
|
<span class="n">report</span><span class="p">[</span><span class="s2">"record"</span><span class="p">],</span>
|
|
<span class="n">config</span><span class="o">=</span><span class="n">cfg</span><span class="p">,</span>
|
|
<span class="n">is_rfc_9990</span><span class="o">=</span><span class="n">is_rfc_9990</span><span class="p">,</span>
|
|
<span class="p">)</span>
|
|
<span class="n">_append_parsed_record</span><span class="p">(</span>
|
|
<span class="n">parsed_record</span><span class="o">=</span><span class="n">report_record</span><span class="p">,</span>
|
|
<span class="n">records</span><span class="o">=</span><span class="n">records</span><span class="p">,</span>
|
|
<span class="n">begin_dt</span><span class="o">=</span><span class="n">begin_dt</span><span class="p">,</span>
|
|
<span class="n">end_dt</span><span class="o">=</span><span class="n">end_dt</span><span class="p">,</span>
|
|
<span class="n">normalize</span><span class="o">=</span><span class="n">normalize_timespan</span><span class="p">,</span>
|
|
<span class="p">)</span>
|
|
|
|
<span class="n">new_report</span><span class="p">[</span><span class="s2">"records"</span><span class="p">]</span> <span class="o">=</span> <span class="n">records</span>
|
|
|
|
<span class="k">return</span> <span class="n">cast</span><span class="p">(</span><span class="n">AggregateReport</span><span class="p">,</span> <span class="n">new_report</span><span class="p">)</span>
|
|
|
|
<span class="k">except</span> <span class="n">expat</span><span class="o">.</span><span class="n">ExpatError</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">InvalidAggregateReport</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Invalid XML: </span><span class="si">{</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">error</span>
|
|
|
|
<span class="k">except</span> <span class="ne">KeyError</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">InvalidAggregateReport</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Missing field: </span><span class="si">{</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">error</span>
|
|
<span class="k">except</span> <span class="ne">AttributeError</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">InvalidAggregateReport</span><span class="p">(</span><span class="s2">"Report missing required section"</span><span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">error</span>
|
|
|
|
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">InvalidAggregateReport</span><span class="p">(</span>
|
|
<span class="sa">f</span><span class="s2">"Unexpected error: </span><span class="si">{</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}{</span><span class="n">_exc_origin</span><span class="p">(</span><span class="n">error</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span>
|
|
<span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">error</span></div>
|
|
|
|
|
|
|
|
<div class="viewcode-block" id="extract_report">
|
|
<a class="viewcode-back" href="../api.html#parsedmarc.extract_report">[docs]</a>
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">extract_report</span><span class="p">(</span><span class="n">content</span><span class="p">:</span> <span class="nb">bytes</span> <span class="o">|</span> <span class="nb">str</span> <span class="o">|</span> <span class="n">BinaryIO</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
|
<span class="w"> </span><span class="sd">"""</span>
|
|
<span class="sd"> Extracts text from a zip or gzip file, as a base64-encoded string,</span>
|
|
<span class="sd"> file-like object, or bytes.</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> content: report file as a base64-encoded string, file-like object or</span>
|
|
<span class="sd"> bytes.</span>
|
|
|
|
<span class="sd"> Returns:</span>
|
|
<span class="sd"> str: The extracted text</span>
|
|
|
|
<span class="sd"> """</span>
|
|
<span class="n">file_object</span><span class="p">:</span> <span class="n">BinaryIO</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="n">header</span><span class="p">:</span> <span class="nb">bytes</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">content</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">file_object</span> <span class="o">=</span> <span class="n">BytesIO</span><span class="p">(</span>
|
|
<span class="n">b64decode</span><span class="p">(</span>
|
|
<span class="n">content</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"</span><span class="se">\n</span><span class="s2">"</span><span class="p">,</span> <span class="s2">""</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"</span><span class="se">\r</span><span class="s2">"</span><span class="p">,</span> <span class="s2">""</span><span class="p">),</span> <span class="n">validate</span><span class="o">=</span><span class="kc">True</span>
|
|
<span class="p">)</span>
|
|
<span class="p">)</span>
|
|
<span class="k">except</span> <span class="n">binascii</span><span class="o">.</span><span class="n">Error</span><span class="p">:</span>
|
|
<span class="k">return</span> <span class="n">content</span>
|
|
<span class="n">header</span> <span class="o">=</span> <span class="n">file_object</span><span class="o">.</span><span class="n">read</span><span class="p">(</span><span class="mi">6</span><span class="p">)</span>
|
|
<span class="n">file_object</span><span class="o">.</span><span class="n">seek</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
|
|
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">content</span><span class="p">,</span> <span class="p">(</span><span class="nb">bytes</span><span class="p">)):</span>
|
|
<span class="n">file_object</span> <span class="o">=</span> <span class="n">BytesIO</span><span class="p">(</span><span class="nb">bytes</span><span class="p">(</span><span class="n">content</span><span class="p">))</span>
|
|
<span class="n">header</span> <span class="o">=</span> <span class="n">file_object</span><span class="o">.</span><span class="n">read</span><span class="p">(</span><span class="mi">6</span><span class="p">)</span>
|
|
<span class="n">file_object</span><span class="o">.</span><span class="n">seek</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">stream</span> <span class="o">=</span> <span class="n">cast</span><span class="p">(</span><span class="n">BinaryIO</span><span class="p">,</span> <span class="n">content</span><span class="p">)</span>
|
|
<span class="n">seekable</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">stream</span><span class="p">,</span> <span class="s2">"seekable"</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
|
<span class="n">can_seek</span> <span class="o">=</span> <span class="kc">False</span>
|
|
<span class="k">if</span> <span class="nb">callable</span><span class="p">(</span><span class="n">seekable</span><span class="p">):</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">can_seek</span> <span class="o">=</span> <span class="nb">bool</span><span class="p">(</span><span class="n">seekable</span><span class="p">())</span>
|
|
<span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span>
|
|
<span class="n">can_seek</span> <span class="o">=</span> <span class="kc">False</span>
|
|
|
|
<span class="k">if</span> <span class="n">can_seek</span><span class="p">:</span>
|
|
<span class="n">header_raw</span> <span class="o">=</span> <span class="n">stream</span><span class="o">.</span><span class="n">read</span><span class="p">(</span><span class="mi">6</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">header_raw</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
|
|
<span class="k">raise</span> <span class="n">ParserError</span><span class="p">(</span><span class="s2">"File objects must be opened in binary (rb) mode"</span><span class="p">)</span>
|
|
<span class="n">header</span> <span class="o">=</span> <span class="nb">bytes</span><span class="p">(</span><span class="n">header_raw</span><span class="p">)</span>
|
|
<span class="n">stream</span><span class="o">.</span><span class="n">seek</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
|
|
<span class="n">file_object</span> <span class="o">=</span> <span class="n">stream</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">header_raw</span> <span class="o">=</span> <span class="n">stream</span><span class="o">.</span><span class="n">read</span><span class="p">(</span><span class="mi">6</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">header_raw</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
|
|
<span class="k">raise</span> <span class="n">ParserError</span><span class="p">(</span><span class="s2">"File objects must be opened in binary (rb) mode"</span><span class="p">)</span>
|
|
<span class="n">header</span> <span class="o">=</span> <span class="nb">bytes</span><span class="p">(</span><span class="n">header_raw</span><span class="p">)</span>
|
|
<span class="n">remainder</span> <span class="o">=</span> <span class="n">stream</span><span class="o">.</span><span class="n">read</span><span class="p">()</span>
|
|
<span class="n">file_object</span> <span class="o">=</span> <span class="n">BytesIO</span><span class="p">(</span><span class="n">header</span> <span class="o">+</span> <span class="nb">bytes</span><span class="p">(</span><span class="n">remainder</span><span class="p">))</span>
|
|
|
|
<span class="k">if</span> <span class="n">header</span><span class="p">[:</span> <span class="nb">len</span><span class="p">(</span><span class="n">MAGIC_ZIP</span><span class="p">)]</span> <span class="o">==</span> <span class="n">MAGIC_ZIP</span><span class="p">:</span>
|
|
<span class="n">_zip</span> <span class="o">=</span> <span class="n">zipfile</span><span class="o">.</span><span class="n">ZipFile</span><span class="p">(</span><span class="n">file_object</span><span class="p">)</span>
|
|
<span class="n">report</span> <span class="o">=</span> <span class="n">_zip</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">_zip</span><span class="o">.</span><span class="n">namelist</span><span class="p">()[</span><span class="mi">0</span><span class="p">])</span><span class="o">.</span><span class="n">read</span><span class="p">()</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="n">errors</span><span class="o">=</span><span class="s2">"ignore"</span><span class="p">)</span>
|
|
<span class="k">elif</span> <span class="n">header</span><span class="p">[:</span> <span class="nb">len</span><span class="p">(</span><span class="n">MAGIC_GZIP</span><span class="p">)]</span> <span class="o">==</span> <span class="n">MAGIC_GZIP</span><span class="p">:</span>
|
|
<span class="n">report</span> <span class="o">=</span> <span class="n">zlib</span><span class="o">.</span><span class="n">decompress</span><span class="p">(</span><span class="n">file_object</span><span class="o">.</span><span class="n">read</span><span class="p">(),</span> <span class="n">zlib</span><span class="o">.</span><span class="n">MAX_WBITS</span> <span class="o">|</span> <span class="mi">16</span><span class="p">)</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span>
|
|
<span class="n">errors</span><span class="o">=</span><span class="s2">"ignore"</span>
|
|
<span class="p">)</span>
|
|
<span class="k">elif</span> <span class="p">(</span>
|
|
<span class="n">header</span><span class="p">[:</span> <span class="nb">len</span><span class="p">(</span><span class="n">MAGIC_XML</span><span class="p">)]</span> <span class="o">==</span> <span class="n">MAGIC_XML</span>
|
|
<span class="ow">or</span> <span class="n">header</span><span class="p">[:</span> <span class="nb">len</span><span class="p">(</span><span class="n">MAGIC_XML_TAG</span><span class="p">)]</span> <span class="o">==</span> <span class="n">MAGIC_XML_TAG</span>
|
|
<span class="ow">or</span> <span class="n">header</span><span class="p">[:</span> <span class="nb">len</span><span class="p">(</span><span class="n">MAGIC_JSON</span><span class="p">)]</span> <span class="o">==</span> <span class="n">MAGIC_JSON</span>
|
|
<span class="p">):</span>
|
|
<span class="n">report</span> <span class="o">=</span> <span class="n">file_object</span><span class="o">.</span><span class="n">read</span><span class="p">()</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="n">errors</span><span class="o">=</span><span class="s2">"ignore"</span><span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">ParserError</span><span class="p">(</span><span class="s2">"Not a valid zip, gzip, json, or xml file"</span><span class="p">)</span>
|
|
|
|
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">ParserError</span><span class="p">(</span>
|
|
<span class="sa">f</span><span class="s2">"Invalid archive file: </span><span class="si">{</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}{</span><span class="n">_exc_origin</span><span class="p">(</span><span class="n">error</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span>
|
|
<span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">error</span>
|
|
<span class="k">finally</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="n">file_object</span><span class="p">:</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">file_object</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
|
|
<span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span>
|
|
<span class="k">pass</span>
|
|
|
|
<span class="k">return</span> <span class="n">report</span></div>
|
|
|
|
|
|
|
|
<div class="viewcode-block" id="extract_report_from_file_path">
|
|
<a class="viewcode-back" href="../api.html#parsedmarc.extract_report_from_file_path">[docs]</a>
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">extract_report_from_file_path</span><span class="p">(</span>
|
|
<span class="n">file_path</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="nb">bytes</span> <span class="o">|</span> <span class="n">os</span><span class="o">.</span><span class="n">PathLike</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">|</span> <span class="n">os</span><span class="o">.</span><span class="n">PathLike</span><span class="p">[</span><span class="nb">bytes</span><span class="p">],</span>
|
|
<span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
|
<span class="w"> </span><span class="sd">"""Extracts report from a file at the given file_path"""</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">fspath</span><span class="p">(</span><span class="n">file_path</span><span class="p">),</span> <span class="s2">"rb"</span><span class="p">)</span> <span class="k">as</span> <span class="n">report_file</span><span class="p">:</span>
|
|
<span class="k">return</span> <span class="n">extract_report</span><span class="p">(</span><span class="n">report_file</span><span class="o">.</span><span class="n">read</span><span class="p">())</span>
|
|
<span class="k">except</span> <span class="ne">FileNotFoundError</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">ParserError</span><span class="p">(</span><span class="s2">"File was not found"</span><span class="p">)</span></div>
|
|
|
|
|
|
|
|
<div class="viewcode-block" id="parse_aggregate_report_file">
|
|
<a class="viewcode-back" href="../api.html#parsedmarc.parse_aggregate_report_file">[docs]</a>
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">parse_aggregate_report_file</span><span class="p">(</span>
|
|
<span class="n">_input</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="nb">bytes</span> <span class="o">|</span> <span class="n">BinaryIO</span><span class="p">,</span>
|
|
<span class="o">*</span><span class="p">,</span>
|
|
<span class="n">offline</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
|
<span class="n">always_use_local_files</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
|
<span class="n">reverse_dns_map_path</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">reverse_dns_map_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">ip_db_path</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">nameservers</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">dns_timeout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="n">DEFAULT_DNS_TIMEOUT</span><span class="p">,</span>
|
|
<span class="n">dns_retries</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="n">DEFAULT_DNS_MAX_RETRIES</span><span class="p">,</span>
|
|
<span class="n">keep_alive</span><span class="p">:</span> <span class="n">Callable</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">normalize_timespan_threshold_hours</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">24.0</span><span class="p">,</span>
|
|
<span class="n">config</span><span class="p">:</span> <span class="n">ParserConfig</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="p">)</span> <span class="o">-></span> <span class="n">AggregateReport</span><span class="p">:</span>
|
|
<span class="w"> </span><span class="sd">"""Parses a file at the given path, a file-like object. or bytes as an</span>
|
|
<span class="sd"> aggregate DMARC report</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> _input (str | bytes | IO): A path to a file, a file like object, or bytes</span>
|
|
<span class="sd"> offline (bool): Do not query online for geolocation or DNS</span>
|
|
<span class="sd"> always_use_local_files (bool): Do not download files</span>
|
|
<span class="sd"> reverse_dns_map_path (str): Path to a reverse DNS map file</span>
|
|
<span class="sd"> reverse_dns_map_url (str): URL to a reverse DNS map file</span>
|
|
<span class="sd"> ip_db_path (str): Path to a MMDB file from IPinfo, MaxMind, or DBIP</span>
|
|
<span class="sd"> nameservers (list): A list of one or more nameservers to use</span>
|
|
<span class="sd"> (Cloudflare's public DNS resolvers by default)</span>
|
|
<span class="sd"> dns_timeout (float): Sets the DNS timeout in seconds</span>
|
|
<span class="sd"> dns_retries (int): Number of times to retry DNS queries on timeout</span>
|
|
<span class="sd"> or other transient errors</span>
|
|
<span class="sd"> keep_alive (callable): Keep alive function. Not part of ``config``;</span>
|
|
<span class="sd"> always applies.</span>
|
|
<span class="sd"> normalize_timespan_threshold_hours (float): Normalize timespans beyond this</span>
|
|
<span class="sd"> config (ParserConfig): a single object carrying all parsing and</span>
|
|
<span class="sd"> enrichment options plus the caches; when provided, the</span>
|
|
<span class="sd"> individual option keyword arguments listed above are ignored in</span>
|
|
<span class="sd"> favor of the config's values.</span>
|
|
|
|
<span class="sd"> Returns:</span>
|
|
<span class="sd"> dict: The parsed DMARC aggregate report</span>
|
|
<span class="sd"> """</span>
|
|
<span class="n">cfg</span> <span class="o">=</span> <span class="n">_resolve_config</span><span class="p">(</span>
|
|
<span class="n">config</span><span class="p">,</span>
|
|
<span class="n">offline</span><span class="o">=</span><span class="n">offline</span><span class="p">,</span>
|
|
<span class="n">ip_db_path</span><span class="o">=</span><span class="n">ip_db_path</span><span class="p">,</span>
|
|
<span class="n">always_use_local_files</span><span class="o">=</span><span class="n">always_use_local_files</span><span class="p">,</span>
|
|
<span class="n">reverse_dns_map_path</span><span class="o">=</span><span class="n">reverse_dns_map_path</span><span class="p">,</span>
|
|
<span class="n">reverse_dns_map_url</span><span class="o">=</span><span class="n">reverse_dns_map_url</span><span class="p">,</span>
|
|
<span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
|
|
<span class="n">dns_timeout</span><span class="o">=</span><span class="n">dns_timeout</span><span class="p">,</span>
|
|
<span class="n">dns_retries</span><span class="o">=</span><span class="n">dns_retries</span><span class="p">,</span>
|
|
<span class="n">normalize_timespan_threshold_hours</span><span class="o">=</span><span class="n">normalize_timespan_threshold_hours</span><span class="p">,</span>
|
|
<span class="p">)</span>
|
|
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">xml</span> <span class="o">=</span> <span class="n">extract_report</span><span class="p">(</span><span class="n">_input</span><span class="p">)</span>
|
|
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">InvalidAggregateReport</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">)</span> <span class="o">+</span> <span class="n">_exc_origin</span><span class="p">(</span><span class="n">e</span><span class="p">))</span> <span class="kn">from</span><span class="w"> </span><span class="nn">e</span>
|
|
|
|
<span class="k">return</span> <span class="n">parse_aggregate_report_xml</span><span class="p">(</span>
|
|
<span class="n">xml</span><span class="p">,</span>
|
|
<span class="n">config</span><span class="o">=</span><span class="n">cfg</span><span class="p">,</span>
|
|
<span class="n">keep_alive</span><span class="o">=</span><span class="n">keep_alive</span><span class="p">,</span>
|
|
<span class="p">)</span></div>
|
|
|
|
|
|
|
|
<div class="viewcode-block" id="parsed_aggregate_reports_to_csv_rows">
|
|
<a class="viewcode-back" href="../api.html#parsedmarc.parsed_aggregate_reports_to_csv_rows">[docs]</a>
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">parsed_aggregate_reports_to_csv_rows</span><span class="p">(</span>
|
|
<span class="n">reports</span><span class="p">:</span> <span class="n">AggregateReport</span> <span class="o">|</span> <span class="nb">list</span><span class="p">[</span><span class="n">AggregateReport</span><span class="p">],</span>
|
|
<span class="p">)</span> <span class="o">-></span> <span class="nb">list</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]]:</span>
|
|
<span class="w"> </span><span class="sd">"""</span>
|
|
<span class="sd"> Converts one or more parsed aggregate reports to list of dicts in flat CSV</span>
|
|
<span class="sd"> format</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> reports: A parsed aggregate report or list of parsed aggregate reports</span>
|
|
|
|
<span class="sd"> Returns:</span>
|
|
<span class="sd"> list: Parsed aggregate report data as a list of dicts in flat CSV</span>
|
|
<span class="sd"> format</span>
|
|
<span class="sd"> """</span>
|
|
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">to_str</span><span class="p">(</span><span class="n">obj</span><span class="p">):</span>
|
|
<span class="k">return</span> <span class="nb">str</span><span class="p">(</span><span class="n">obj</span><span class="p">)</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
|
|
|
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">reports</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
|
|
<span class="n">reports</span> <span class="o">=</span> <span class="p">[</span><span class="n">reports</span><span class="p">]</span>
|
|
|
|
<span class="n">rows</span> <span class="o">=</span> <span class="p">[]</span>
|
|
|
|
<span class="k">for</span> <span class="n">report</span> <span class="ow">in</span> <span class="n">reports</span><span class="p">:</span>
|
|
<span class="n">xml_schema</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"xml_schema"</span><span class="p">]</span>
|
|
<span class="n">org_name</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">][</span><span class="s2">"org_name"</span><span class="p">]</span>
|
|
<span class="n">org_email</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">][</span><span class="s2">"org_email"</span><span class="p">]</span>
|
|
<span class="n">org_extra_contact</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">][</span><span class="s2">"org_extra_contact_info"</span><span class="p">]</span>
|
|
<span class="n">report_id</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">][</span><span class="s2">"report_id"</span><span class="p">]</span>
|
|
<span class="n">begin_date</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">][</span><span class="s2">"begin_date"</span><span class="p">]</span>
|
|
<span class="n">end_date</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">][</span><span class="s2">"end_date"</span><span class="p">]</span>
|
|
<span class="n">normalized_timespan</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">][</span>
|
|
<span class="s2">"timespan_requires_normalization"</span>
|
|
<span class="p">]</span>
|
|
<span class="n">errors</span> <span class="o">=</span> <span class="s2">"|"</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">][</span><span class="s2">"errors"</span><span class="p">])</span>
|
|
<span class="n">domain</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"domain"</span><span class="p">]</span>
|
|
<span class="n">adkim</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"adkim"</span><span class="p">]</span>
|
|
<span class="n">aspf</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"aspf"</span><span class="p">]</span>
|
|
<span class="n">p</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"p"</span><span class="p">]</span>
|
|
<span class="n">sp</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"sp"</span><span class="p">]</span>
|
|
<span class="n">pct</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"pct"</span><span class="p">]</span>
|
|
<span class="n">fo</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"fo"</span><span class="p">]</span>
|
|
<span class="n">np_</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"np"</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
|
<span class="n">testing</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"testing"</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
|
<span class="n">discovery_method</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"discovery_method"</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
|
|
|
<span class="n">report_dict</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
|
|
<span class="n">xml_schema</span><span class="o">=</span><span class="n">xml_schema</span><span class="p">,</span>
|
|
<span class="n">org_name</span><span class="o">=</span><span class="n">org_name</span><span class="p">,</span>
|
|
<span class="n">org_email</span><span class="o">=</span><span class="n">org_email</span><span class="p">,</span>
|
|
<span class="n">org_extra_contact_info</span><span class="o">=</span><span class="n">org_extra_contact</span><span class="p">,</span>
|
|
<span class="n">report_id</span><span class="o">=</span><span class="n">report_id</span><span class="p">,</span>
|
|
<span class="n">begin_date</span><span class="o">=</span><span class="n">begin_date</span><span class="p">,</span>
|
|
<span class="n">end_date</span><span class="o">=</span><span class="n">end_date</span><span class="p">,</span>
|
|
<span class="n">normalized_timespan</span><span class="o">=</span><span class="n">normalized_timespan</span><span class="p">,</span>
|
|
<span class="n">errors</span><span class="o">=</span><span class="n">errors</span><span class="p">,</span>
|
|
<span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span>
|
|
<span class="n">adkim</span><span class="o">=</span><span class="n">adkim</span><span class="p">,</span>
|
|
<span class="n">aspf</span><span class="o">=</span><span class="n">aspf</span><span class="p">,</span>
|
|
<span class="n">p</span><span class="o">=</span><span class="n">p</span><span class="p">,</span>
|
|
<span class="n">sp</span><span class="o">=</span><span class="n">sp</span><span class="p">,</span>
|
|
<span class="n">np</span><span class="o">=</span><span class="n">np_</span><span class="p">,</span>
|
|
<span class="n">pct</span><span class="o">=</span><span class="n">pct</span><span class="p">,</span>
|
|
<span class="n">fo</span><span class="o">=</span><span class="n">fo</span><span class="p">,</span>
|
|
<span class="n">testing</span><span class="o">=</span><span class="n">testing</span><span class="p">,</span>
|
|
<span class="n">discovery_method</span><span class="o">=</span><span class="n">discovery_method</span><span class="p">,</span>
|
|
<span class="p">)</span>
|
|
|
|
<span class="k">for</span> <span class="n">record</span> <span class="ow">in</span> <span class="n">report</span><span class="p">[</span><span class="s2">"records"</span><span class="p">]:</span>
|
|
<span class="n">row</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="n">report_dict</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"interval_begin"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"interval_end"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"source_ip_address"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"ip_address"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"source_country"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"country"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"source_reverse_dns"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"reverse_dns"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"source_base_domain"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"base_domain"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"source_name"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"name"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"source_type"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"type"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"source_asn"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"asn"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"source_as_name"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"as_name"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"source_as_domain"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"as_domain"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"count"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"count"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"spf_aligned"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"alignment"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"dkim_aligned"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"alignment"</span><span class="p">][</span><span class="s2">"dkim"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"dmarc_aligned"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"alignment"</span><span class="p">][</span><span class="s2">"dmarc"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"disposition"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"disposition"</span><span class="p">]</span>
|
|
<span class="n">policy_override_reasons</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span>
|
|
<span class="nb">map</span><span class="p">(</span>
|
|
<span class="k">lambda</span> <span class="n">r_</span><span class="p">:</span> <span class="n">r_</span><span class="p">[</span><span class="s2">"type"</span><span class="p">]</span> <span class="ow">or</span> <span class="s2">"none"</span><span class="p">,</span>
|
|
<span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"policy_override_reasons"</span><span class="p">],</span>
|
|
<span class="p">)</span>
|
|
<span class="p">)</span>
|
|
<span class="n">policy_override_comments</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span>
|
|
<span class="nb">map</span><span class="p">(</span>
|
|
<span class="k">lambda</span> <span class="n">r_</span><span class="p">:</span> <span class="n">r_</span><span class="p">[</span><span class="s2">"comment"</span><span class="p">]</span> <span class="ow">or</span> <span class="s2">"none"</span><span class="p">,</span>
|
|
<span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"policy_override_reasons"</span><span class="p">],</span>
|
|
<span class="p">)</span>
|
|
<span class="p">)</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"policy_override_reasons"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">","</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">policy_override_reasons</span><span class="p">)</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"policy_override_comments"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"|"</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">policy_override_comments</span><span class="p">)</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"envelope_from"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"envelope_from"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"header_from"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"header_from"</span><span class="p">]</span>
|
|
<span class="n">envelope_to</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"envelope_to"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"envelope_to"</span><span class="p">]</span> <span class="o">=</span> <span class="n">envelope_to</span>
|
|
<span class="n">dkim_domains</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="n">dkim_selectors</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="n">dkim_results</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="k">for</span> <span class="n">dkim_result</span> <span class="ow">in</span> <span class="n">record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">][</span><span class="s2">"dkim"</span><span class="p">]:</span>
|
|
<span class="n">dkim_domains</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">])</span>
|
|
<span class="k">if</span> <span class="s2">"selector"</span> <span class="ow">in</span> <span class="n">dkim_result</span><span class="p">:</span>
|
|
<span class="n">dkim_selectors</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">"selector"</span><span class="p">])</span>
|
|
<span class="n">dkim_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">])</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"dkim_domains"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">","</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="n">to_str</span><span class="p">,</span> <span class="n">dkim_domains</span><span class="p">))</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"dkim_selectors"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">","</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="n">to_str</span><span class="p">,</span> <span class="n">dkim_selectors</span><span class="p">))</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"dkim_results"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">","</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="n">to_str</span><span class="p">,</span> <span class="n">dkim_results</span><span class="p">))</span>
|
|
<span class="n">spf_domains</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="n">spf_scopes</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="n">spf_results</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="k">for</span> <span class="n">spf_result</span> <span class="ow">in</span> <span class="n">record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">]:</span>
|
|
<span class="n">spf_domains</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">])</span>
|
|
<span class="n">spf_scopes</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">"scope"</span><span class="p">])</span>
|
|
<span class="n">spf_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">])</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"spf_domains"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">","</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="n">to_str</span><span class="p">,</span> <span class="n">spf_domains</span><span class="p">))</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"spf_scopes"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">","</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="n">to_str</span><span class="p">,</span> <span class="n">spf_scopes</span><span class="p">))</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"spf_results"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">","</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="n">to_str</span><span class="p">,</span> <span class="n">spf_results</span><span class="p">))</span>
|
|
<span class="n">rows</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">row</span><span class="p">)</span>
|
|
|
|
<span class="k">for</span> <span class="n">r</span> <span class="ow">in</span> <span class="n">rows</span><span class="p">:</span>
|
|
<span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">r</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
|
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">v</span><span class="p">)</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="nb">bool</span><span class="p">]:</span>
|
|
<span class="n">r</span><span class="p">[</span><span class="n">k</span><span class="p">]</span> <span class="o">=</span> <span class="s2">""</span>
|
|
|
|
<span class="k">return</span> <span class="n">rows</span></div>
|
|
|
|
|
|
|
|
<div class="viewcode-block" id="parsed_aggregate_reports_to_csv">
|
|
<a class="viewcode-back" href="../api.html#parsedmarc.parsed_aggregate_reports_to_csv">[docs]</a>
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">parsed_aggregate_reports_to_csv</span><span class="p">(</span>
|
|
<span class="n">reports</span><span class="p">:</span> <span class="n">AggregateReport</span> <span class="o">|</span> <span class="nb">list</span><span class="p">[</span><span class="n">AggregateReport</span><span class="p">],</span>
|
|
<span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
|
<span class="w"> </span><span class="sd">"""</span>
|
|
<span class="sd"> Converts one or more parsed aggregate reports to flat CSV format, including</span>
|
|
<span class="sd"> headers</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> reports: A parsed aggregate report or list of parsed aggregate reports</span>
|
|
|
|
<span class="sd"> Returns:</span>
|
|
<span class="sd"> str: Parsed aggregate report data in flat CSV format, including headers</span>
|
|
<span class="sd"> """</span>
|
|
|
|
<span class="n">fields</span> <span class="o">=</span> <span class="p">[</span>
|
|
<span class="s2">"xml_schema"</span><span class="p">,</span>
|
|
<span class="s2">"org_name"</span><span class="p">,</span>
|
|
<span class="s2">"org_email"</span><span class="p">,</span>
|
|
<span class="s2">"org_extra_contact_info"</span><span class="p">,</span>
|
|
<span class="s2">"report_id"</span><span class="p">,</span>
|
|
<span class="s2">"begin_date"</span><span class="p">,</span>
|
|
<span class="s2">"end_date"</span><span class="p">,</span>
|
|
<span class="s2">"normalized_timespan"</span><span class="p">,</span>
|
|
<span class="s2">"errors"</span><span class="p">,</span>
|
|
<span class="s2">"domain"</span><span class="p">,</span>
|
|
<span class="s2">"adkim"</span><span class="p">,</span>
|
|
<span class="s2">"aspf"</span><span class="p">,</span>
|
|
<span class="s2">"p"</span><span class="p">,</span>
|
|
<span class="s2">"sp"</span><span class="p">,</span>
|
|
<span class="s2">"np"</span><span class="p">,</span>
|
|
<span class="s2">"pct"</span><span class="p">,</span>
|
|
<span class="s2">"fo"</span><span class="p">,</span>
|
|
<span class="s2">"testing"</span><span class="p">,</span>
|
|
<span class="s2">"discovery_method"</span><span class="p">,</span>
|
|
<span class="s2">"source_ip_address"</span><span class="p">,</span>
|
|
<span class="s2">"source_country"</span><span class="p">,</span>
|
|
<span class="s2">"source_reverse_dns"</span><span class="p">,</span>
|
|
<span class="s2">"source_base_domain"</span><span class="p">,</span>
|
|
<span class="s2">"source_name"</span><span class="p">,</span>
|
|
<span class="s2">"source_type"</span><span class="p">,</span>
|
|
<span class="s2">"source_asn"</span><span class="p">,</span>
|
|
<span class="s2">"source_as_name"</span><span class="p">,</span>
|
|
<span class="s2">"source_as_domain"</span><span class="p">,</span>
|
|
<span class="s2">"count"</span><span class="p">,</span>
|
|
<span class="s2">"spf_aligned"</span><span class="p">,</span>
|
|
<span class="s2">"dkim_aligned"</span><span class="p">,</span>
|
|
<span class="s2">"dmarc_aligned"</span><span class="p">,</span>
|
|
<span class="s2">"disposition"</span><span class="p">,</span>
|
|
<span class="s2">"policy_override_reasons"</span><span class="p">,</span>
|
|
<span class="s2">"policy_override_comments"</span><span class="p">,</span>
|
|
<span class="s2">"envelope_from"</span><span class="p">,</span>
|
|
<span class="s2">"header_from"</span><span class="p">,</span>
|
|
<span class="s2">"envelope_to"</span><span class="p">,</span>
|
|
<span class="s2">"dkim_domains"</span><span class="p">,</span>
|
|
<span class="s2">"dkim_selectors"</span><span class="p">,</span>
|
|
<span class="s2">"dkim_results"</span><span class="p">,</span>
|
|
<span class="s2">"spf_domains"</span><span class="p">,</span>
|
|
<span class="s2">"spf_scopes"</span><span class="p">,</span>
|
|
<span class="s2">"spf_results"</span><span class="p">,</span>
|
|
<span class="p">]</span>
|
|
|
|
<span class="n">csv_file_object</span> <span class="o">=</span> <span class="n">StringIO</span><span class="p">(</span><span class="n">newline</span><span class="o">=</span><span class="s2">"</span><span class="se">\n</span><span class="s2">"</span><span class="p">)</span>
|
|
<span class="n">writer</span> <span class="o">=</span> <span class="n">DictWriter</span><span class="p">(</span><span class="n">csv_file_object</span><span class="p">,</span> <span class="n">fields</span><span class="p">)</span>
|
|
<span class="n">writer</span><span class="o">.</span><span class="n">writeheader</span><span class="p">()</span>
|
|
|
|
<span class="n">rows</span> <span class="o">=</span> <span class="n">parsed_aggregate_reports_to_csv_rows</span><span class="p">(</span><span class="n">reports</span><span class="p">)</span>
|
|
|
|
<span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="n">rows</span><span class="p">:</span>
|
|
<span class="n">writer</span><span class="o">.</span><span class="n">writerow</span><span class="p">(</span><span class="n">row</span><span class="p">)</span>
|
|
<span class="n">csv_file_object</span><span class="o">.</span><span class="n">flush</span><span class="p">()</span>
|
|
|
|
<span class="k">return</span> <span class="n">csv_file_object</span><span class="o">.</span><span class="n">getvalue</span><span class="p">()</span></div>
|
|
|
|
|
|
|
|
<div class="viewcode-block" id="parse_failure_report">
|
|
<a class="viewcode-back" href="../api.html#parsedmarc.parse_failure_report">[docs]</a>
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">parse_failure_report</span><span class="p">(</span>
|
|
<span class="n">feedback_report</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
|
<span class="n">sample</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
|
<span class="n">msg_date</span><span class="p">:</span> <span class="n">datetime</span><span class="p">,</span>
|
|
<span class="o">*</span><span class="p">,</span>
|
|
<span class="n">always_use_local_files</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
|
<span class="n">reverse_dns_map_path</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">reverse_dns_map_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">offline</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
|
<span class="n">ip_db_path</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">nameservers</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">dns_timeout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="n">DEFAULT_DNS_TIMEOUT</span><span class="p">,</span>
|
|
<span class="n">dns_retries</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="n">DEFAULT_DNS_MAX_RETRIES</span><span class="p">,</span>
|
|
<span class="n">strip_attachment_payloads</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
|
<span class="n">config</span><span class="p">:</span> <span class="n">ParserConfig</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="p">)</span> <span class="o">-></span> <span class="n">FailureReport</span><span class="p">:</span>
|
|
<span class="w"> </span><span class="sd">"""</span>
|
|
<span class="sd"> Converts a DMARC failure report and sample to a dict</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> feedback_report (str): A message's feedback report as a string</span>
|
|
<span class="sd"> sample (str): The RFC 822 headers or RFC 822 message sample</span>
|
|
<span class="sd"> ip_db_path (str): Path to a MMDB file from IPinfo, MaxMind, or DBIP</span>
|
|
<span class="sd"> always_use_local_files (bool): Do not download files</span>
|
|
<span class="sd"> reverse_dns_map_path (str): Path to a reverse DNS map file</span>
|
|
<span class="sd"> reverse_dns_map_url (str): URL to a reverse DNS map file</span>
|
|
<span class="sd"> offline (bool): Do not query online for geolocation or DNS</span>
|
|
<span class="sd"> msg_date (str): The message's date header</span>
|
|
<span class="sd"> nameservers (list): A list of one or more nameservers to use</span>
|
|
<span class="sd"> (Cloudflare's public DNS resolvers by default)</span>
|
|
<span class="sd"> dns_timeout (float): Sets the DNS timeout in seconds</span>
|
|
<span class="sd"> dns_retries (int): Number of times to retry DNS queries on timeout</span>
|
|
<span class="sd"> or other transient errors</span>
|
|
<span class="sd"> strip_attachment_payloads (bool): Remove attachment payloads from</span>
|
|
<span class="sd"> failure report results</span>
|
|
<span class="sd"> config (ParserConfig): a single object carrying all parsing and</span>
|
|
<span class="sd"> enrichment options plus the caches; when provided, the</span>
|
|
<span class="sd"> individual option keyword arguments listed above are ignored in</span>
|
|
<span class="sd"> favor of the config's values.</span>
|
|
|
|
<span class="sd"> Returns:</span>
|
|
<span class="sd"> dict: A parsed report and sample</span>
|
|
<span class="sd"> """</span>
|
|
<span class="n">cfg</span> <span class="o">=</span> <span class="n">_resolve_config</span><span class="p">(</span>
|
|
<span class="n">config</span><span class="p">,</span>
|
|
<span class="n">offline</span><span class="o">=</span><span class="n">offline</span><span class="p">,</span>
|
|
<span class="n">ip_db_path</span><span class="o">=</span><span class="n">ip_db_path</span><span class="p">,</span>
|
|
<span class="n">always_use_local_files</span><span class="o">=</span><span class="n">always_use_local_files</span><span class="p">,</span>
|
|
<span class="n">reverse_dns_map_path</span><span class="o">=</span><span class="n">reverse_dns_map_path</span><span class="p">,</span>
|
|
<span class="n">reverse_dns_map_url</span><span class="o">=</span><span class="n">reverse_dns_map_url</span><span class="p">,</span>
|
|
<span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
|
|
<span class="n">dns_timeout</span><span class="o">=</span><span class="n">dns_timeout</span><span class="p">,</span>
|
|
<span class="n">dns_retries</span><span class="o">=</span><span class="n">dns_retries</span><span class="p">,</span>
|
|
<span class="n">strip_attachment_payloads</span><span class="o">=</span><span class="n">strip_attachment_payloads</span><span class="p">,</span>
|
|
<span class="p">)</span>
|
|
<span class="n">delivery_results</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"delivered"</span><span class="p">,</span> <span class="s2">"spam"</span><span class="p">,</span> <span class="s2">"policy"</span><span class="p">,</span> <span class="s2">"reject"</span><span class="p">,</span> <span class="s2">"other"</span><span class="p">]</span>
|
|
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">parsed_report</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
|
|
<span class="n">report_values</span> <span class="o">=</span> <span class="n">feedback_report_regex</span><span class="o">.</span><span class="n">findall</span><span class="p">(</span><span class="n">feedback_report</span><span class="p">)</span>
|
|
<span class="k">for</span> <span class="n">report_value</span> <span class="ow">in</span> <span class="n">report_values</span><span class="p">:</span>
|
|
<span class="n">key</span> <span class="o">=</span> <span class="n">report_value</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"-"</span><span class="p">,</span> <span class="s2">"_"</span><span class="p">)</span>
|
|
<span class="n">parsed_report</span><span class="p">[</span><span class="n">key</span><span class="p">]</span> <span class="o">=</span> <span class="n">report_value</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
|
|
|
|
<span class="k">if</span> <span class="s2">"arrival_date"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_report</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="n">msg_date</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">InvalidFailureReport</span><span class="p">(</span><span class="s2">"Failure sample is not a valid email"</span><span class="p">)</span>
|
|
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"arrival_date"</span><span class="p">]</span> <span class="o">=</span> <span class="n">msg_date</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span>
|
|
|
|
<span class="k">if</span> <span class="s2">"version"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_report</span><span class="p">:</span>
|
|
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"version"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
|
|
|
<span class="k">if</span> <span class="s2">"user_agent"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_report</span><span class="p">:</span>
|
|
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"user_agent"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
|
|
|
<span class="k">if</span> <span class="s2">"delivery_result"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_report</span><span class="p">:</span>
|
|
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"delivery_result"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="k">for</span> <span class="n">delivery_result</span> <span class="ow">in</span> <span class="n">delivery_results</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="n">delivery_result</span> <span class="ow">in</span> <span class="n">parsed_report</span><span class="p">[</span><span class="s2">"delivery_result"</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">():</span>
|
|
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"delivery_result"</span><span class="p">]</span> <span class="o">=</span> <span class="n">delivery_result</span>
|
|
<span class="k">break</span>
|
|
<span class="k">if</span> <span class="n">parsed_report</span><span class="p">[</span><span class="s2">"delivery_result"</span><span class="p">]</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">delivery_results</span><span class="p">:</span>
|
|
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"delivery_result"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"other"</span>
|
|
|
|
<span class="n">arrival_utc</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span>
|
|
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"arrival_date"</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span>
|
|
<span class="p">)</span>
|
|
<span class="n">arrival_utc</span> <span class="o">=</span> <span class="n">arrival_utc</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%S"</span><span class="p">)</span>
|
|
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"arrival_date_utc"</span><span class="p">]</span> <span class="o">=</span> <span class="n">arrival_utc</span>
|
|
|
|
<span class="n">ip_address</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="sa">r</span><span class="s2">"\s"</span><span class="p">,</span> <span class="n">parsed_report</span><span class="p">[</span><span class="s2">"source_ip"</span><span class="p">])</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
|
|
<span class="n">parsed_report_source</span> <span class="o">=</span> <span class="n">get_ip_address_info</span><span class="p">(</span>
|
|
<span class="n">ip_address</span><span class="p">,</span>
|
|
<span class="n">cache</span><span class="o">=</span><span class="n">cfg</span><span class="o">.</span><span class="n">ip_address_cache</span><span class="p">,</span>
|
|
<span class="n">ip_db_path</span><span class="o">=</span><span class="n">cfg</span><span class="o">.</span><span class="n">ip_db_path</span><span class="p">,</span>
|
|
<span class="n">always_use_local_files</span><span class="o">=</span><span class="n">cfg</span><span class="o">.</span><span class="n">always_use_local_files</span><span class="p">,</span>
|
|
<span class="n">reverse_dns_map_path</span><span class="o">=</span><span class="n">cfg</span><span class="o">.</span><span class="n">reverse_dns_map_path</span><span class="p">,</span>
|
|
<span class="n">reverse_dns_map_url</span><span class="o">=</span><span class="n">cfg</span><span class="o">.</span><span class="n">reverse_dns_map_url</span><span class="p">,</span>
|
|
<span class="n">reverse_dns_map</span><span class="o">=</span><span class="n">cfg</span><span class="o">.</span><span class="n">reverse_dns_map</span><span class="p">,</span>
|
|
<span class="n">offline</span><span class="o">=</span><span class="n">cfg</span><span class="o">.</span><span class="n">offline</span><span class="p">,</span>
|
|
<span class="n">nameservers</span><span class="o">=</span><span class="n">cfg</span><span class="o">.</span><span class="n">nameservers</span><span class="p">,</span>
|
|
<span class="n">timeout</span><span class="o">=</span><span class="n">cfg</span><span class="o">.</span><span class="n">dns_timeout</span><span class="p">,</span>
|
|
<span class="n">retries</span><span class="o">=</span><span class="n">cfg</span><span class="o">.</span><span class="n">dns_retries</span><span class="p">,</span>
|
|
<span class="n">psl_overrides_path</span><span class="o">=</span><span class="n">cfg</span><span class="o">.</span><span class="n">psl_overrides_path</span><span class="p">,</span>
|
|
<span class="n">psl_overrides_url</span><span class="o">=</span><span class="n">cfg</span><span class="o">.</span><span class="n">psl_overrides_url</span><span class="p">,</span>
|
|
<span class="p">)</span>
|
|
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">]</span> <span class="o">=</span> <span class="n">parsed_report_source</span>
|
|
<span class="k">del</span> <span class="n">parsed_report</span><span class="p">[</span><span class="s2">"source_ip"</span><span class="p">]</span>
|
|
|
|
<span class="c1"># Identity-Alignment is REQUIRED per RFC 9991 §4. Default silently for</span>
|
|
<span class="c1"># backward compatibility with pre-9991 reporters, but log so the</span>
|
|
<span class="c1"># offending reporter is visible. Values are CFWS-separated per the</span>
|
|
<span class="c1"># ABNF, so each mechanism is stripped after splitting.</span>
|
|
<span class="k">if</span> <span class="s2">"identity_alignment"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_report</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
|
<span class="s2">"Failure report missing required 'Identity-Alignment' "</span>
|
|
<span class="s2">"field (RFC 9991 §4); defaulting to no aligned mechanisms"</span>
|
|
<span class="p">)</span>
|
|
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"authentication_mechanisms"</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">raw_alignment</span> <span class="o">=</span> <span class="n">parsed_report</span><span class="p">[</span><span class="s2">"identity_alignment"</span><span class="p">]</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
|
|
<span class="k">if</span> <span class="n">raw_alignment</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s2">"none"</span><span class="p">:</span>
|
|
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"authentication_mechanisms"</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"authentication_mechanisms"</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span>
|
|
<span class="n">m</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span> <span class="k">for</span> <span class="n">m</span> <span class="ow">in</span> <span class="n">raw_alignment</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">","</span><span class="p">)</span> <span class="k">if</span> <span class="n">m</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
|
|
<span class="p">]</span>
|
|
<span class="k">del</span> <span class="n">parsed_report</span><span class="p">[</span><span class="s2">"identity_alignment"</span><span class="p">]</span>
|
|
|
|
<span class="c1"># Auth-Failure is REQUIRED per RFC 9991 §4. Comma-separated per ABNF</span>
|
|
<span class="c1"># so strip each token.</span>
|
|
<span class="k">if</span> <span class="s2">"auth_failure"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_report</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
|
<span class="s2">"Failure report missing required 'Auth-Failure' field "</span>
|
|
<span class="s2">"(RFC 9991 §4); defaulting to 'dmarc'"</span>
|
|
<span class="p">)</span>
|
|
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"auth_failure"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"dmarc"</span>
|
|
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"auth_failure"</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span>
|
|
<span class="n">f</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span> <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="n">parsed_report</span><span class="p">[</span><span class="s2">"auth_failure"</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">","</span><span class="p">)</span> <span class="k">if</span> <span class="n">f</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
|
|
<span class="p">]</span>
|
|
|
|
<span class="c1"># Feedback-Type is REQUIRED per RFC 5965 §3.1, but some gateways</span>
|
|
<span class="c1"># (e.g. Exim/cPanel-based ones that send a plain-text summary without</span>
|
|
<span class="c1"># a machine-readable message/feedback-report part) omit it. The</span>
|
|
<span class="c1"># Elasticsearch/OpenSearch outputs require the key, so default it</span>
|
|
<span class="c1"># instead of dropping the report there (see issue #332).</span>
|
|
<span class="k">if</span> <span class="s2">"feedback_type"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_report</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
|
<span class="s2">"Failure report missing required 'Feedback-Type' field "</span>
|
|
<span class="s2">"(RFC 5965 §3.1); defaulting to 'auth-failure'"</span>
|
|
<span class="p">)</span>
|
|
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"feedback_type"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"auth-failure"</span>
|
|
|
|
<span class="c1"># Authentication-Results is likewise REQUIRED per RFC 6591 §3.1 and</span>
|
|
<span class="c1"># required by the Elasticsearch/OpenSearch outputs.</span>
|
|
<span class="k">if</span> <span class="s2">"authentication_results"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_report</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
|
<span class="s2">"Failure report missing required 'Authentication-Results' "</span>
|
|
<span class="s2">"field (RFC 6591 §3.1); defaulting to None"</span>
|
|
<span class="p">)</span>
|
|
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"authentication_results"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
|
|
|
<span class="n">optional_fields</span> <span class="o">=</span> <span class="p">[</span>
|
|
<span class="s2">"original_envelope_id"</span><span class="p">,</span>
|
|
<span class="s2">"dkim_domain"</span><span class="p">,</span>
|
|
<span class="s2">"original_mail_from"</span><span class="p">,</span>
|
|
<span class="s2">"original_rcpt_to"</span><span class="p">,</span>
|
|
<span class="p">]</span>
|
|
<span class="k">for</span> <span class="n">optional_field</span> <span class="ow">in</span> <span class="n">optional_fields</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="n">optional_field</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_report</span><span class="p">:</span>
|
|
<span class="n">parsed_report</span><span class="p">[</span><span class="n">optional_field</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
|
|
|
<span class="n">parsed_sample</span> <span class="o">=</span> <span class="n">parse_email</span><span class="p">(</span>
|
|
<span class="n">sample</span><span class="p">,</span> <span class="n">strip_attachment_payloads</span><span class="o">=</span><span class="n">cfg</span><span class="o">.</span><span class="n">strip_attachment_payloads</span>
|
|
<span class="p">)</span>
|
|
|
|
<span class="k">if</span> <span class="s2">"reported_domain"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_report</span><span class="p">:</span>
|
|
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"reported_domain"</span><span class="p">]</span> <span class="o">=</span> <span class="n">parsed_sample</span><span class="p">[</span><span class="s2">"from"</span><span class="p">][</span><span class="s2">"domain"</span><span class="p">]</span>
|
|
|
|
<span class="n">sample_headers_only</span> <span class="o">=</span> <span class="kc">False</span>
|
|
<span class="n">number_of_attachments</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">parsed_sample</span><span class="p">[</span><span class="s2">"attachments"</span><span class="p">])</span>
|
|
<span class="k">if</span> <span class="n">number_of_attachments</span> <span class="o"><</span> <span class="mi">1</span> <span class="ow">and</span> <span class="n">parsed_sample</span><span class="p">[</span><span class="s2">"body"</span><span class="p">]</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">sample_headers_only</span> <span class="o">=</span> <span class="kc">True</span>
|
|
<span class="k">if</span> <span class="n">sample_headers_only</span> <span class="ow">and</span> <span class="n">parsed_sample</span><span class="p">[</span><span class="s2">"has_defects"</span><span class="p">]:</span>
|
|
<span class="k">del</span> <span class="n">parsed_sample</span><span class="p">[</span><span class="s2">"defects"</span><span class="p">]</span>
|
|
<span class="k">del</span> <span class="n">parsed_sample</span><span class="p">[</span><span class="s2">"defects_categories"</span><span class="p">]</span>
|
|
<span class="k">del</span> <span class="n">parsed_sample</span><span class="p">[</span><span class="s2">"has_defects"</span><span class="p">]</span>
|
|
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"sample_headers_only"</span><span class="p">]</span> <span class="o">=</span> <span class="n">sample_headers_only</span>
|
|
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"sample"</span><span class="p">]</span> <span class="o">=</span> <span class="n">sample</span>
|
|
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">]</span> <span class="o">=</span> <span class="n">parsed_sample</span>
|
|
|
|
<span class="k">return</span> <span class="n">cast</span><span class="p">(</span><span class="n">FailureReport</span><span class="p">,</span> <span class="n">parsed_report</span><span class="p">)</span>
|
|
|
|
<span class="k">except</span> <span class="ne">KeyError</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">InvalidFailureReport</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Missing value: </span><span class="si">{</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">error</span>
|
|
|
|
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">InvalidFailureReport</span><span class="p">(</span>
|
|
<span class="sa">f</span><span class="s2">"Unexpected error: </span><span class="si">{</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}{</span><span class="n">_exc_origin</span><span class="p">(</span><span class="n">error</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span>
|
|
<span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">error</span></div>
|
|
|
|
|
|
|
|
<div class="viewcode-block" id="parsed_failure_reports_to_csv_rows">
|
|
<a class="viewcode-back" href="../api.html#parsedmarc.parsed_failure_reports_to_csv_rows">[docs]</a>
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">parsed_failure_reports_to_csv_rows</span><span class="p">(</span>
|
|
<span class="n">reports</span><span class="p">:</span> <span class="n">FailureReport</span> <span class="o">|</span> <span class="nb">list</span><span class="p">[</span><span class="n">FailureReport</span><span class="p">],</span>
|
|
<span class="p">)</span> <span class="o">-></span> <span class="nb">list</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]]:</span>
|
|
<span class="w"> </span><span class="sd">"""</span>
|
|
<span class="sd"> Converts one or more parsed failure reports to a list of dicts in flat CSV</span>
|
|
<span class="sd"> format</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> reports: A parsed failure report or list of parsed failure reports</span>
|
|
|
|
<span class="sd"> Returns:</span>
|
|
<span class="sd"> list: Parsed failure report data as a list of dicts in flat CSV format</span>
|
|
<span class="sd"> """</span>
|
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">reports</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
|
|
<span class="n">reports</span> <span class="o">=</span> <span class="p">[</span><span class="n">reports</span><span class="p">]</span>
|
|
|
|
<span class="n">rows</span> <span class="o">=</span> <span class="p">[]</span>
|
|
|
|
<span class="k">for</span> <span class="n">report</span> <span class="ow">in</span> <span class="n">reports</span><span class="p">:</span>
|
|
<span class="n">row</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="n">report</span><span class="p">)</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"source_ip_address"</span><span class="p">]</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"ip_address"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"source_reverse_dns"</span><span class="p">]</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"reverse_dns"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"source_base_domain"</span><span class="p">]</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"base_domain"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"source_name"</span><span class="p">]</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"name"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"source_type"</span><span class="p">]</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"type"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"source_asn"</span><span class="p">]</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"asn"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"source_as_name"</span><span class="p">]</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"as_name"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"source_as_domain"</span><span class="p">]</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"as_domain"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"source_country"</span><span class="p">]</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"country"</span><span class="p">]</span>
|
|
<span class="k">del</span> <span class="n">row</span><span class="p">[</span><span class="s2">"source"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"subject"</span><span class="p">]</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"subject"</span><span class="p">)</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"auth_failure"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">","</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">"auth_failure"</span><span class="p">])</span>
|
|
<span class="n">authentication_mechanisms</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"authentication_mechanisms"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"authentication_mechanisms"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">","</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">authentication_mechanisms</span><span class="p">)</span>
|
|
<span class="k">del</span> <span class="n">row</span><span class="p">[</span><span class="s2">"sample"</span><span class="p">]</span>
|
|
<span class="k">del</span> <span class="n">row</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">]</span>
|
|
<span class="n">rows</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">row</span><span class="p">)</span>
|
|
|
|
<span class="k">return</span> <span class="n">rows</span></div>
|
|
|
|
|
|
|
|
<div class="viewcode-block" id="parsed_failure_reports_to_csv">
|
|
<a class="viewcode-back" href="../api.html#parsedmarc.parsed_failure_reports_to_csv">[docs]</a>
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">parsed_failure_reports_to_csv</span><span class="p">(</span>
|
|
<span class="n">reports</span><span class="p">:</span> <span class="n">FailureReport</span> <span class="o">|</span> <span class="nb">list</span><span class="p">[</span><span class="n">FailureReport</span><span class="p">],</span>
|
|
<span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
|
<span class="w"> </span><span class="sd">"""</span>
|
|
<span class="sd"> Converts one or more parsed failure reports to flat CSV format, including</span>
|
|
<span class="sd"> headers</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> reports: A parsed failure report or list of parsed failure reports</span>
|
|
|
|
<span class="sd"> Returns:</span>
|
|
<span class="sd"> str: Parsed failure report data in flat CSV format, including headers</span>
|
|
<span class="sd"> """</span>
|
|
<span class="n">fields</span> <span class="o">=</span> <span class="p">[</span>
|
|
<span class="s2">"feedback_type"</span><span class="p">,</span>
|
|
<span class="s2">"user_agent"</span><span class="p">,</span>
|
|
<span class="s2">"version"</span><span class="p">,</span>
|
|
<span class="s2">"original_envelope_id"</span><span class="p">,</span>
|
|
<span class="s2">"original_mail_from"</span><span class="p">,</span>
|
|
<span class="s2">"original_rcpt_to"</span><span class="p">,</span>
|
|
<span class="s2">"arrival_date"</span><span class="p">,</span>
|
|
<span class="s2">"arrival_date_utc"</span><span class="p">,</span>
|
|
<span class="s2">"subject"</span><span class="p">,</span>
|
|
<span class="s2">"message_id"</span><span class="p">,</span>
|
|
<span class="s2">"authentication_results"</span><span class="p">,</span>
|
|
<span class="s2">"dkim_domain"</span><span class="p">,</span>
|
|
<span class="s2">"source_ip_address"</span><span class="p">,</span>
|
|
<span class="s2">"source_country"</span><span class="p">,</span>
|
|
<span class="s2">"source_reverse_dns"</span><span class="p">,</span>
|
|
<span class="s2">"source_base_domain"</span><span class="p">,</span>
|
|
<span class="s2">"source_name"</span><span class="p">,</span>
|
|
<span class="s2">"source_type"</span><span class="p">,</span>
|
|
<span class="s2">"source_asn"</span><span class="p">,</span>
|
|
<span class="s2">"source_as_name"</span><span class="p">,</span>
|
|
<span class="s2">"source_as_domain"</span><span class="p">,</span>
|
|
<span class="s2">"delivery_result"</span><span class="p">,</span>
|
|
<span class="s2">"auth_failure"</span><span class="p">,</span>
|
|
<span class="s2">"reported_domain"</span><span class="p">,</span>
|
|
<span class="s2">"authentication_mechanisms"</span><span class="p">,</span>
|
|
<span class="s2">"sample_headers_only"</span><span class="p">,</span>
|
|
<span class="p">]</span>
|
|
|
|
<span class="n">csv_file</span> <span class="o">=</span> <span class="n">StringIO</span><span class="p">()</span>
|
|
<span class="n">csv_writer</span> <span class="o">=</span> <span class="n">DictWriter</span><span class="p">(</span><span class="n">csv_file</span><span class="p">,</span> <span class="n">fieldnames</span><span class="o">=</span><span class="n">fields</span><span class="p">)</span>
|
|
<span class="n">csv_writer</span><span class="o">.</span><span class="n">writeheader</span><span class="p">()</span>
|
|
|
|
<span class="n">rows</span> <span class="o">=</span> <span class="n">parsed_failure_reports_to_csv_rows</span><span class="p">(</span><span class="n">reports</span><span class="p">)</span>
|
|
|
|
<span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="n">rows</span><span class="p">:</span>
|
|
<span class="n">new_row</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
|
|
<span class="k">for</span> <span class="n">key</span> <span class="ow">in</span> <span class="n">fields</span><span class="p">:</span>
|
|
<span class="n">new_row</span><span class="p">[</span><span class="n">key</span><span class="p">]</span> <span class="o">=</span> <span class="n">row</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">key</span><span class="p">)</span>
|
|
<span class="n">csv_writer</span><span class="o">.</span><span class="n">writerow</span><span class="p">(</span><span class="n">new_row</span><span class="p">)</span>
|
|
|
|
<span class="k">return</span> <span class="n">csv_file</span><span class="o">.</span><span class="n">getvalue</span><span class="p">()</span></div>
|
|
|
|
|
|
|
|
<div class="viewcode-block" id="parse_report_email">
|
|
<a class="viewcode-back" href="../api.html#parsedmarc.parse_report_email">[docs]</a>
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">parse_report_email</span><span class="p">(</span>
|
|
<span class="n">input_</span><span class="p">:</span> <span class="nb">bytes</span> <span class="o">|</span> <span class="nb">str</span><span class="p">,</span>
|
|
<span class="o">*</span><span class="p">,</span>
|
|
<span class="n">offline</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
|
<span class="n">ip_db_path</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">always_use_local_files</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
|
<span class="n">reverse_dns_map_path</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">reverse_dns_map_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">nameservers</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">dns_timeout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="n">DEFAULT_DNS_TIMEOUT</span><span class="p">,</span>
|
|
<span class="n">dns_retries</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="n">DEFAULT_DNS_MAX_RETRIES</span><span class="p">,</span>
|
|
<span class="n">strip_attachment_payloads</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
|
<span class="n">keep_alive</span><span class="p">:</span> <span class="n">Callable</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">normalize_timespan_threshold_hours</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">24.0</span><span class="p">,</span>
|
|
<span class="n">config</span><span class="p">:</span> <span class="n">ParserConfig</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="p">)</span> <span class="o">-></span> <span class="n">ParsedReport</span><span class="p">:</span>
|
|
<span class="w"> </span><span class="sd">"""</span>
|
|
<span class="sd"> Parses a DMARC report from an email</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> input_: An emailed DMARC report in RFC 822 format, as bytes or a string</span>
|
|
<span class="sd"> ip_db_path (str): Path to a MMDB file from IPinfo, MaxMind, or DBIP</span>
|
|
<span class="sd"> always_use_local_files (bool): Do not download files</span>
|
|
<span class="sd"> reverse_dns_map_path (str): Path to a reverse DNS map</span>
|
|
<span class="sd"> reverse_dns_map_url (str): URL to a reverse DNS map</span>
|
|
<span class="sd"> offline (bool): Do not query online for geolocation on DNS</span>
|
|
<span class="sd"> nameservers (list): A list of one or more nameservers to use</span>
|
|
<span class="sd"> dns_timeout (float): Sets the DNS timeout in seconds</span>
|
|
<span class="sd"> dns_retries (int): Number of times to retry DNS queries on timeout</span>
|
|
<span class="sd"> or other transient errors</span>
|
|
<span class="sd"> strip_attachment_payloads (bool): Remove attachment payloads from</span>
|
|
<span class="sd"> failure report results</span>
|
|
<span class="sd"> keep_alive (callable): keep alive function. Not part of ``config``;</span>
|
|
<span class="sd"> always applies.</span>
|
|
<span class="sd"> normalize_timespan_threshold_hours (float): Normalize timespans beyond this</span>
|
|
<span class="sd"> config (ParserConfig): a single object carrying all parsing and</span>
|
|
<span class="sd"> enrichment options plus the caches; when provided, the</span>
|
|
<span class="sd"> individual option keyword arguments listed above are ignored in</span>
|
|
<span class="sd"> favor of the config's values.</span>
|
|
|
|
<span class="sd"> Returns:</span>
|
|
<span class="sd"> dict:</span>
|
|
<span class="sd"> * ``report_type``: ``aggregate`` or ``failure``</span>
|
|
<span class="sd"> * ``report``: The parsed report</span>
|
|
<span class="sd"> """</span>
|
|
<span class="n">cfg</span> <span class="o">=</span> <span class="n">_resolve_config</span><span class="p">(</span>
|
|
<span class="n">config</span><span class="p">,</span>
|
|
<span class="n">offline</span><span class="o">=</span><span class="n">offline</span><span class="p">,</span>
|
|
<span class="n">ip_db_path</span><span class="o">=</span><span class="n">ip_db_path</span><span class="p">,</span>
|
|
<span class="n">always_use_local_files</span><span class="o">=</span><span class="n">always_use_local_files</span><span class="p">,</span>
|
|
<span class="n">reverse_dns_map_path</span><span class="o">=</span><span class="n">reverse_dns_map_path</span><span class="p">,</span>
|
|
<span class="n">reverse_dns_map_url</span><span class="o">=</span><span class="n">reverse_dns_map_url</span><span class="p">,</span>
|
|
<span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
|
|
<span class="n">dns_timeout</span><span class="o">=</span><span class="n">dns_timeout</span><span class="p">,</span>
|
|
<span class="n">dns_retries</span><span class="o">=</span><span class="n">dns_retries</span><span class="p">,</span>
|
|
<span class="n">strip_attachment_payloads</span><span class="o">=</span><span class="n">strip_attachment_payloads</span><span class="p">,</span>
|
|
<span class="n">normalize_timespan_threshold_hours</span><span class="o">=</span><span class="n">normalize_timespan_threshold_hours</span><span class="p">,</span>
|
|
<span class="p">)</span>
|
|
<span class="n">result</span><span class="p">:</span> <span class="n">ParsedReport</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="n">msg_date</span><span class="p">:</span> <span class="n">datetime</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">(</span><span class="n">timezone</span><span class="o">.</span><span class="n">utc</span><span class="p">)</span>
|
|
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">input_data</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="nb">bytes</span> <span class="o">|</span> <span class="nb">bytearray</span> <span class="o">|</span> <span class="nb">memoryview</span> <span class="o">=</span> <span class="n">input_</span>
|
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">input_data</span><span class="p">,</span> <span class="p">(</span><span class="nb">bytes</span><span class="p">,</span> <span class="nb">bytearray</span><span class="p">,</span> <span class="nb">memoryview</span><span class="p">)):</span>
|
|
<span class="n">input_bytes</span> <span class="o">=</span> <span class="nb">bytes</span><span class="p">(</span><span class="n">input_data</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">is_outlook_msg</span><span class="p">(</span><span class="n">input_bytes</span><span class="p">):</span>
|
|
<span class="n">converted</span> <span class="o">=</span> <span class="n">convert_outlook_msg</span><span class="p">(</span><span class="n">input_bytes</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">converted</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
|
|
<span class="n">input_str</span> <span class="o">=</span> <span class="n">converted</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">input_str</span> <span class="o">=</span> <span class="nb">bytes</span><span class="p">(</span><span class="n">converted</span><span class="p">)</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span>
|
|
<span class="n">encoding</span><span class="o">=</span><span class="s2">"utf8"</span><span class="p">,</span> <span class="n">errors</span><span class="o">=</span><span class="s2">"replace"</span>
|
|
<span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">input_str</span> <span class="o">=</span> <span class="n">input_bytes</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="n">encoding</span><span class="o">=</span><span class="s2">"utf8"</span><span class="p">,</span> <span class="n">errors</span><span class="o">=</span><span class="s2">"replace"</span><span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">input_str</span> <span class="o">=</span> <span class="n">input_data</span>
|
|
|
|
<span class="n">msg</span> <span class="o">=</span> <span class="n">mailparser</span><span class="o">.</span><span class="n">parse_from_string</span><span class="p">(</span><span class="n">input_str</span><span class="p">)</span>
|
|
<span class="n">msg_headers</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">msg</span><span class="o">.</span><span class="n">headers_json</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="s2">"Date"</span> <span class="ow">in</span> <span class="n">msg_headers</span><span class="p">:</span>
|
|
<span class="n">msg_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">msg_headers</span><span class="p">[</span><span class="s2">"Date"</span><span class="p">])</span>
|
|
<span class="n">date</span> <span class="o">=</span> <span class="n">email</span><span class="o">.</span><span class="n">utils</span><span class="o">.</span><span class="n">format_datetime</span><span class="p">(</span><span class="n">msg_date</span><span class="p">)</span>
|
|
<span class="n">msg</span> <span class="o">=</span> <span class="n">email</span><span class="o">.</span><span class="n">message_from_string</span><span class="p">(</span><span class="n">input_str</span><span class="p">)</span>
|
|
|
|
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">ParserError</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span> <span class="o">+</span> <span class="n">_exc_origin</span><span class="p">(</span><span class="n">e</span><span class="p">))</span> <span class="kn">from</span><span class="w"> </span><span class="nn">e</span>
|
|
<span class="n">subject</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="n">feedback_report</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="n">smtp_tls_report</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="n">sample</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="n">is_feedback_report</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span>
|
|
<span class="k">if</span> <span class="s2">"From"</span> <span class="ow">in</span> <span class="n">msg_headers</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Parsing mail from </span><span class="si">{}</span><span class="s2"> on </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">msg_headers</span><span class="p">[</span><span class="s2">"From"</span><span class="p">],</span> <span class="n">date</span><span class="p">))</span>
|
|
<span class="k">if</span> <span class="s2">"Subject"</span> <span class="ow">in</span> <span class="n">msg_headers</span><span class="p">:</span>
|
|
<span class="n">subject</span> <span class="o">=</span> <span class="n">msg_headers</span><span class="p">[</span><span class="s2">"Subject"</span><span class="p">]</span>
|
|
<span class="k">for</span> <span class="n">part</span> <span class="ow">in</span> <span class="n">msg</span><span class="o">.</span><span class="n">walk</span><span class="p">():</span>
|
|
<span class="n">content_type</span> <span class="o">=</span> <span class="n">part</span><span class="o">.</span><span class="n">get_content_type</span><span class="p">()</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
|
|
<span class="n">payload_obj</span> <span class="o">=</span> <span class="n">part</span><span class="o">.</span><span class="n">get_payload</span><span class="p">()</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">payload_obj</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
|
|
<span class="n">payload_obj</span> <span class="o">=</span> <span class="p">[</span><span class="n">payload_obj</span><span class="p">]</span>
|
|
<span class="n">payload</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">payload_obj</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span>
|
|
<span class="k">if</span> <span class="n">content_type</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">"multipart/"</span><span class="p">):</span>
|
|
<span class="k">continue</span>
|
|
<span class="k">if</span> <span class="n">content_type</span> <span class="o">==</span> <span class="s2">"text/html"</span><span class="p">:</span>
|
|
<span class="k">continue</span>
|
|
<span class="k">elif</span> <span class="n">content_type</span> <span class="o">==</span> <span class="s2">"message/feedback-report"</span><span class="p">:</span>
|
|
<span class="n">is_feedback_report</span> <span class="o">=</span> <span class="kc">True</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="s2">"Feedback-Type"</span> <span class="ow">in</span> <span class="n">payload</span><span class="p">:</span>
|
|
<span class="n">feedback_report</span> <span class="o">=</span> <span class="n">payload</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">feedback_report</span> <span class="o">=</span> <span class="n">b64decode</span><span class="p">(</span><span class="n">payload</span><span class="p">)</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span>
|
|
<span class="n">feedback_report</span> <span class="o">=</span> <span class="n">feedback_report</span><span class="o">.</span><span class="n">lstrip</span><span class="p">(</span><span class="s2">"b'"</span><span class="p">)</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s2">"'"</span><span class="p">)</span>
|
|
<span class="n">feedback_report</span> <span class="o">=</span> <span class="n">feedback_report</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"</span><span class="se">\\</span><span class="s2">r"</span><span class="p">,</span> <span class="s2">""</span><span class="p">)</span>
|
|
<span class="n">feedback_report</span> <span class="o">=</span> <span class="n">feedback_report</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"</span><span class="se">\\</span><span class="s2">n"</span><span class="p">,</span> <span class="s2">"</span><span class="se">\n</span><span class="s2">"</span><span class="p">)</span>
|
|
<span class="k">except</span> <span class="p">(</span><span class="ne">ValueError</span><span class="p">,</span> <span class="ne">TypeError</span><span class="p">,</span> <span class="n">binascii</span><span class="o">.</span><span class="n">Error</span><span class="p">):</span>
|
|
<span class="n">feedback_report</span> <span class="o">=</span> <span class="n">payload</span>
|
|
<span class="k">elif</span> <span class="n">is_feedback_report</span> <span class="ow">and</span> <span class="n">content_type</span> <span class="ow">in</span> <span class="n">EMAIL_SAMPLE_CONTENT_TYPES</span><span class="p">:</span>
|
|
<span class="n">sample</span> <span class="o">=</span> <span class="n">payload</span>
|
|
<span class="k">elif</span> <span class="n">content_type</span> <span class="o">==</span> <span class="s2">"application/tlsrpt+json"</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">payload</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">"{"</span><span class="p">):</span>
|
|
<span class="n">payload</span> <span class="o">=</span> <span class="n">b64decode</span><span class="p">(</span><span class="n">payload</span><span class="p">)</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s2">"utf-8"</span><span class="p">,</span> <span class="n">errors</span><span class="o">=</span><span class="s2">"replace"</span><span class="p">)</span>
|
|
<span class="n">smtp_tls_report</span> <span class="o">=</span> <span class="n">parse_smtp_tls_report_json</span><span class="p">(</span><span class="n">payload</span><span class="p">)</span>
|
|
<span class="k">return</span> <span class="p">{</span><span class="s2">"report_type"</span><span class="p">:</span> <span class="s2">"smtp_tls"</span><span class="p">,</span> <span class="s2">"report"</span><span class="p">:</span> <span class="n">smtp_tls_report</span><span class="p">}</span>
|
|
<span class="k">elif</span> <span class="n">content_type</span> <span class="o">==</span> <span class="s2">"application/tlsrpt+gzip"</span><span class="p">:</span>
|
|
<span class="n">payload</span> <span class="o">=</span> <span class="n">extract_report</span><span class="p">(</span><span class="n">payload</span><span class="p">)</span>
|
|
<span class="n">smtp_tls_report</span> <span class="o">=</span> <span class="n">parse_smtp_tls_report_json</span><span class="p">(</span><span class="n">payload</span><span class="p">)</span>
|
|
<span class="k">return</span> <span class="p">{</span><span class="s2">"report_type"</span><span class="p">:</span> <span class="s2">"smtp_tls"</span><span class="p">,</span> <span class="s2">"report"</span><span class="p">:</span> <span class="n">smtp_tls_report</span><span class="p">}</span>
|
|
<span class="k">elif</span> <span class="n">content_type</span> <span class="o">==</span> <span class="s2">"text/plain"</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="s2">"A message claiming to be from you has failed"</span> <span class="ow">in</span> <span class="n">payload</span><span class="p">:</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">parts</span> <span class="o">=</span> <span class="n">payload</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">"detected."</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
|
|
<span class="n">field_matches</span> <span class="o">=</span> <span class="n">text_report_regex</span><span class="o">.</span><span class="n">findall</span><span class="p">(</span><span class="n">parts</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span>
|
|
<span class="n">fields</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
|
|
<span class="k">for</span> <span class="n">match</span> <span class="ow">in</span> <span class="n">field_matches</span><span class="p">:</span>
|
|
<span class="n">field_name</span> <span class="o">=</span> <span class="n">match</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">" "</span><span class="p">,</span> <span class="s2">"-"</span><span class="p">)</span>
|
|
<span class="n">fields</span><span class="p">[</span><span class="n">field_name</span><span class="p">]</span> <span class="o">=</span> <span class="n">match</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
|
|
|
|
<span class="n">feedback_report</span> <span class="o">=</span> <span class="s2">"Arrival-Date: </span><span class="si">{}</span><span class="se">\n</span><span class="s2">Source-IP: </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
|
<span class="n">fields</span><span class="p">[</span><span class="s2">"received-date"</span><span class="p">],</span> <span class="n">fields</span><span class="p">[</span><span class="s2">"sender-ip-address"</span><span class="p">]</span>
|
|
<span class="p">)</span>
|
|
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
|
<span class="n">error</span> <span class="o">=</span> <span class="sa">f</span><span class="s1">'Unable to parse message with subject "</span><span class="si">{</span><span class="n">subject</span><span class="si">}</span><span class="s1">": </span><span class="si">{</span><span class="n">e</span><span class="si">}{</span><span class="n">_exc_origin</span><span class="p">(</span><span class="n">e</span><span class="p">)</span><span class="si">}</span><span class="s1">'</span>
|
|
<span class="k">raise</span> <span class="n">InvalidDMARCReport</span><span class="p">(</span><span class="n">error</span><span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">e</span>
|
|
|
|
<span class="n">sample</span> <span class="o">=</span> <span class="n">parts</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">lstrip</span><span class="p">()</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="n">sample</span><span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">payload_bytes</span> <span class="o">=</span> <span class="n">b64decode</span><span class="p">(</span><span class="n">payload</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">payload_bytes</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="n">MAGIC_ZIP</span><span class="p">)</span> <span class="ow">or</span> <span class="n">payload_bytes</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span>
|
|
<span class="n">MAGIC_GZIP</span>
|
|
<span class="p">):</span>
|
|
<span class="n">payload_text</span> <span class="o">=</span> <span class="n">extract_report</span><span class="p">(</span><span class="n">payload_bytes</span><span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">payload_text</span> <span class="o">=</span> <span class="n">payload_bytes</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s2">"utf-8"</span><span class="p">,</span> <span class="n">errors</span><span class="o">=</span><span class="s2">"replace"</span><span class="p">)</span>
|
|
|
|
<span class="k">if</span> <span class="n">payload_text</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">"{"</span><span class="p">):</span>
|
|
<span class="n">smtp_tls_report</span> <span class="o">=</span> <span class="n">parse_smtp_tls_report_json</span><span class="p">(</span><span class="n">payload_text</span><span class="p">)</span>
|
|
<span class="n">result</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"report_type"</span><span class="p">:</span> <span class="s2">"smtp_tls"</span><span class="p">,</span> <span class="s2">"report"</span><span class="p">:</span> <span class="n">smtp_tls_report</span><span class="p">}</span>
|
|
<span class="k">return</span> <span class="n">result</span>
|
|
<span class="k">elif</span> <span class="n">payload_text</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">"<"</span><span class="p">):</span>
|
|
<span class="n">aggregate_report</span> <span class="o">=</span> <span class="n">parse_aggregate_report_xml</span><span class="p">(</span>
|
|
<span class="n">payload_text</span><span class="p">,</span>
|
|
<span class="n">config</span><span class="o">=</span><span class="n">cfg</span><span class="p">,</span>
|
|
<span class="n">keep_alive</span><span class="o">=</span><span class="n">keep_alive</span><span class="p">,</span>
|
|
<span class="p">)</span>
|
|
<span class="n">result</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"report_type"</span><span class="p">:</span> <span class="s2">"aggregate"</span><span class="p">,</span> <span class="s2">"report"</span><span class="p">:</span> <span class="n">aggregate_report</span><span class="p">}</span>
|
|
|
|
<span class="k">return</span> <span class="n">result</span>
|
|
|
|
<span class="k">except</span> <span class="p">(</span><span class="ne">TypeError</span><span class="p">,</span> <span class="ne">ValueError</span><span class="p">,</span> <span class="n">binascii</span><span class="o">.</span><span class="n">Error</span><span class="p">):</span>
|
|
<span class="k">pass</span>
|
|
|
|
<span class="k">except</span> <span class="n">InvalidDMARCReport</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
|
<span class="n">error</span> <span class="o">=</span> <span class="p">(</span>
|
|
<span class="sa">f</span><span class="s1">'Message with subject "</span><span class="si">{</span><span class="n">subject</span><span class="si">}</span><span class="s1">" is not a valid DMARC report: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s1">'</span>
|
|
<span class="p">)</span>
|
|
<span class="k">raise</span> <span class="n">ParserError</span><span class="p">(</span><span class="n">error</span><span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">e</span>
|
|
|
|
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
|
<span class="n">error</span> <span class="o">=</span> <span class="sa">f</span><span class="s1">'Unable to parse message with subject "</span><span class="si">{</span><span class="n">subject</span><span class="si">}</span><span class="s1">": </span><span class="si">{</span><span class="n">e</span><span class="si">}{</span><span class="n">_exc_origin</span><span class="p">(</span><span class="n">e</span><span class="p">)</span><span class="si">}</span><span class="s1">'</span>
|
|
<span class="k">raise</span> <span class="n">ParserError</span><span class="p">(</span><span class="n">error</span><span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">e</span>
|
|
|
|
<span class="k">if</span> <span class="n">feedback_report</span> <span class="ow">and</span> <span class="n">sample</span><span class="p">:</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">failure_report</span> <span class="o">=</span> <span class="n">parse_failure_report</span><span class="p">(</span>
|
|
<span class="n">feedback_report</span><span class="p">,</span>
|
|
<span class="n">sample</span><span class="p">,</span>
|
|
<span class="n">msg_date</span><span class="p">,</span>
|
|
<span class="n">config</span><span class="o">=</span><span class="n">cfg</span><span class="p">,</span>
|
|
<span class="p">)</span>
|
|
<span class="k">except</span> <span class="n">InvalidFailureReport</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
|
<span class="n">error</span> <span class="o">=</span> <span class="p">(</span>
|
|
<span class="sa">f</span><span class="s1">'Message with subject "</span><span class="si">{</span><span class="n">subject</span><span class="si">}</span><span class="s1">" '</span>
|
|
<span class="s2">"is not a valid "</span>
|
|
<span class="sa">f</span><span class="s2">"failure DMARC report: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">"</span>
|
|
<span class="p">)</span>
|
|
<span class="k">raise</span> <span class="n">InvalidFailureReport</span><span class="p">(</span><span class="n">error</span><span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">e</span>
|
|
|
|
<span class="n">result</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"report_type"</span><span class="p">:</span> <span class="s2">"failure"</span><span class="p">,</span> <span class="s2">"report"</span><span class="p">:</span> <span class="n">failure_report</span><span class="p">}</span>
|
|
<span class="k">return</span> <span class="n">result</span>
|
|
|
|
<span class="k">if</span> <span class="n">result</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">error</span> <span class="o">=</span> <span class="sa">f</span><span class="s1">'Message with subject "</span><span class="si">{</span><span class="n">subject</span><span class="si">}</span><span class="s1">" is not a valid report'</span>
|
|
<span class="k">raise</span> <span class="n">InvalidDMARCReport</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
|
|
|
|
<span class="k">return</span> <span class="n">result</span></div>
|
|
|
|
|
|
|
|
<span class="c1"># An RFC 5322 header field name (printable ASCII excluding the colon) followed</span>
|
|
<span class="c1"># by a colon at the start of a line, or an mbox "From " separator.</span>
|
|
<span class="n">_email_header_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">"^(From |[\x21-\x39\x3b-\x7e]+:)"</span><span class="p">)</span>
|
|
|
|
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">_looks_like_email</span><span class="p">(</span><span class="n">text</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span>
|
|
<span class="w"> </span><span class="sd">"""Returns True if the first line looks like an email header.</span>
|
|
|
|
<span class="sd"> Callers pass already-``lstrip()``-ed text, so the first line is the first</span>
|
|
<span class="sd"> meaningful line.</span>
|
|
<span class="sd"> """</span>
|
|
<span class="n">first_line</span> <span class="o">=</span> <span class="n">text</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">"</span><span class="se">\n</span><span class="s2">"</span><span class="p">,</span> <span class="mi">1</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
|
|
<span class="k">return</span> <span class="n">_email_header_regex</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="n">first_line</span><span class="p">)</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
|
|
|
|
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">_describe_parse_failure</span><span class="p">(</span>
|
|
<span class="n">content</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="nb">bytes</span><span class="p">,</span>
|
|
<span class="n">aggregate_error</span><span class="p">:</span> <span class="n">InvalidAggregateReport</span><span class="p">,</span>
|
|
<span class="n">smtp_tls_error</span><span class="p">:</span> <span class="n">InvalidSMTPTLSReport</span><span class="p">,</span>
|
|
<span class="n">email_error</span><span class="p">:</span> <span class="n">InvalidDMARCReport</span><span class="p">,</span>
|
|
<span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
|
<span class="w"> </span><span class="sd">"""Builds a human-readable reason for a parse_report_file failure.</span>
|
|
|
|
<span class="sd"> parse_report_file tries the aggregate XML, SMTP TLS JSON, and report-email</span>
|
|
<span class="sd"> parsers in turn; when all three reject the input, only the parser for the</span>
|
|
<span class="sd"> format the content actually resembles produced a meaningful error. The</span>
|
|
<span class="sd"> other two are noise -- a malformed aggregate report is also "not JSON" and</span>
|
|
<span class="sd"> "not an email". Sniff the leading non-whitespace byte to surface the single</span>
|
|
<span class="sd"> relevant reason.</span>
|
|
<span class="sd"> """</span>
|
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">content</span><span class="p">,</span> <span class="p">(</span><span class="nb">bytes</span><span class="p">,</span> <span class="nb">bytearray</span><span class="p">,</span> <span class="nb">memoryview</span><span class="p">)):</span>
|
|
<span class="n">sniff</span> <span class="o">=</span> <span class="nb">bytes</span><span class="p">(</span><span class="n">content</span><span class="p">)[:</span><span class="mi">512</span><span class="p">]</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s2">"utf-8"</span><span class="p">,</span> <span class="n">errors</span><span class="o">=</span><span class="s2">"replace"</span><span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">sniff</span> <span class="o">=</span> <span class="n">content</span><span class="p">[:</span><span class="mi">512</span><span class="p">]</span>
|
|
<span class="n">sniff</span> <span class="o">=</span> <span class="n">sniff</span><span class="o">.</span><span class="n">lstrip</span><span class="p">()</span>
|
|
|
|
<span class="k">if</span> <span class="n">sniff</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">"<"</span><span class="p">):</span>
|
|
<span class="k">return</span> <span class="sa">f</span><span class="s2">"Invalid aggregate report: </span><span class="si">{</span><span class="n">aggregate_error</span><span class="si">}</span><span class="s2">"</span>
|
|
<span class="k">if</span> <span class="n">sniff</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">"{"</span><span class="p">):</span>
|
|
<span class="k">return</span> <span class="sa">f</span><span class="s2">"Invalid SMTP TLS report: </span><span class="si">{</span><span class="n">smtp_tls_error</span><span class="si">}</span><span class="s2">"</span>
|
|
<span class="k">if</span> <span class="n">_looks_like_email</span><span class="p">(</span><span class="n">sniff</span><span class="p">):</span>
|
|
<span class="k">return</span> <span class="sa">f</span><span class="s2">"Invalid report email: </span><span class="si">{</span><span class="n">email_error</span><span class="si">}</span><span class="s2">"</span>
|
|
<span class="k">return</span> <span class="p">(</span>
|
|
<span class="s2">"Not a recognized report format (not a DMARC aggregate XML report, "</span>
|
|
<span class="s2">"an SMTP TLS JSON report, or a DMARC report email)"</span>
|
|
<span class="p">)</span>
|
|
|
|
|
|
<div class="viewcode-block" id="parse_report_file">
|
|
<a class="viewcode-back" href="../api.html#parsedmarc.parse_report_file">[docs]</a>
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">parse_report_file</span><span class="p">(</span>
|
|
<span class="n">input_</span><span class="p">:</span> <span class="nb">bytes</span> <span class="o">|</span> <span class="nb">str</span> <span class="o">|</span> <span class="n">os</span><span class="o">.</span><span class="n">PathLike</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">|</span> <span class="n">os</span><span class="o">.</span><span class="n">PathLike</span><span class="p">[</span><span class="nb">bytes</span><span class="p">]</span> <span class="o">|</span> <span class="n">BinaryIO</span><span class="p">,</span>
|
|
<span class="o">*</span><span class="p">,</span>
|
|
<span class="n">nameservers</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">dns_timeout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="n">DEFAULT_DNS_TIMEOUT</span><span class="p">,</span>
|
|
<span class="n">dns_retries</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="n">DEFAULT_DNS_MAX_RETRIES</span><span class="p">,</span>
|
|
<span class="n">strip_attachment_payloads</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
|
<span class="n">ip_db_path</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">always_use_local_files</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
|
<span class="n">reverse_dns_map_path</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">reverse_dns_map_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">offline</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
|
<span class="n">keep_alive</span><span class="p">:</span> <span class="n">Callable</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">normalize_timespan_threshold_hours</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">24.0</span><span class="p">,</span>
|
|
<span class="n">config</span><span class="p">:</span> <span class="n">ParserConfig</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="p">)</span> <span class="o">-></span> <span class="n">ParsedReport</span><span class="p">:</span>
|
|
<span class="w"> </span><span class="sd">"""Parses a DMARC aggregate or failure file at the given path, a</span>
|
|
<span class="sd"> file-like object. or bytes</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> input_ (str | os.PathLike | bytes | BinaryIO): A path to a file,</span>
|
|
<span class="sd"> a file-like object, or bytes</span>
|
|
<span class="sd"> nameservers (list): A list of one or more nameservers to use</span>
|
|
<span class="sd"> (Cloudflare's public DNS resolvers by default)</span>
|
|
<span class="sd"> dns_timeout (float): Sets the DNS timeout in seconds</span>
|
|
<span class="sd"> dns_retries (int): Number of times to retry DNS queries on timeout</span>
|
|
<span class="sd"> or other transient errors</span>
|
|
<span class="sd"> strip_attachment_payloads (bool): Remove attachment payloads from</span>
|
|
<span class="sd"> failure report results</span>
|
|
<span class="sd"> ip_db_path (str): Path to a MMDB file from IPinfo, MaxMind, or DBIP</span>
|
|
<span class="sd"> always_use_local_files (bool): Do not download files</span>
|
|
<span class="sd"> reverse_dns_map_path (str): Path to a reverse DNS map</span>
|
|
<span class="sd"> reverse_dns_map_url (str): URL to a reverse DNS map</span>
|
|
<span class="sd"> offline (bool): Do not make online queries for geolocation or DNS</span>
|
|
<span class="sd"> keep_alive (callable): Keep alive function. Not part of ``config``;</span>
|
|
<span class="sd"> always applies.</span>
|
|
<span class="sd"> normalize_timespan_threshold_hours (float): Normalize timespans beyond this</span>
|
|
<span class="sd"> config (ParserConfig): a single object carrying all parsing and</span>
|
|
<span class="sd"> enrichment options plus the caches; when provided, the</span>
|
|
<span class="sd"> individual option keyword arguments listed above are ignored in</span>
|
|
<span class="sd"> favor of the config's values.</span>
|
|
|
|
<span class="sd"> Returns:</span>
|
|
<span class="sd"> dict: The parsed DMARC report</span>
|
|
<span class="sd"> """</span>
|
|
<span class="n">cfg</span> <span class="o">=</span> <span class="n">_resolve_config</span><span class="p">(</span>
|
|
<span class="n">config</span><span class="p">,</span>
|
|
<span class="n">offline</span><span class="o">=</span><span class="n">offline</span><span class="p">,</span>
|
|
<span class="n">ip_db_path</span><span class="o">=</span><span class="n">ip_db_path</span><span class="p">,</span>
|
|
<span class="n">always_use_local_files</span><span class="o">=</span><span class="n">always_use_local_files</span><span class="p">,</span>
|
|
<span class="n">reverse_dns_map_path</span><span class="o">=</span><span class="n">reverse_dns_map_path</span><span class="p">,</span>
|
|
<span class="n">reverse_dns_map_url</span><span class="o">=</span><span class="n">reverse_dns_map_url</span><span class="p">,</span>
|
|
<span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
|
|
<span class="n">dns_timeout</span><span class="o">=</span><span class="n">dns_timeout</span><span class="p">,</span>
|
|
<span class="n">dns_retries</span><span class="o">=</span><span class="n">dns_retries</span><span class="p">,</span>
|
|
<span class="n">strip_attachment_payloads</span><span class="o">=</span><span class="n">strip_attachment_payloads</span><span class="p">,</span>
|
|
<span class="n">normalize_timespan_threshold_hours</span><span class="o">=</span><span class="n">normalize_timespan_threshold_hours</span><span class="p">,</span>
|
|
<span class="p">)</span>
|
|
<span class="n">file_object</span><span class="p">:</span> <span class="n">BinaryIO</span>
|
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">input_</span><span class="p">,</span> <span class="p">(</span><span class="nb">str</span><span class="p">,</span> <span class="n">os</span><span class="o">.</span><span class="n">PathLike</span><span class="p">)):</span>
|
|
<span class="n">file_path</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">fspath</span><span class="p">(</span><span class="n">input_</span><span class="p">)</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Parsing </span><span class="si">{</span><span class="n">file_path</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
|
<span class="n">file_object</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="n">file_path</span><span class="p">,</span> <span class="s2">"rb"</span><span class="p">)</span>
|
|
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">input_</span><span class="p">,</span> <span class="p">(</span><span class="nb">bytes</span><span class="p">,</span> <span class="nb">bytearray</span><span class="p">,</span> <span class="nb">memoryview</span><span class="p">)):</span>
|
|
<span class="n">file_object</span> <span class="o">=</span> <span class="n">BytesIO</span><span class="p">(</span><span class="nb">bytes</span><span class="p">(</span><span class="n">input_</span><span class="p">))</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">file_object</span> <span class="o">=</span> <span class="n">input_</span>
|
|
|
|
<span class="n">content</span> <span class="o">=</span> <span class="n">file_object</span><span class="o">.</span><span class="n">read</span><span class="p">()</span>
|
|
<span class="n">file_object</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
|
|
<span class="k">if</span> <span class="n">content</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="n">MAGIC_ZIP</span><span class="p">)</span> <span class="ow">or</span> <span class="n">content</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="n">MAGIC_GZIP</span><span class="p">):</span>
|
|
<span class="n">content</span> <span class="o">=</span> <span class="n">extract_report</span><span class="p">(</span><span class="n">content</span><span class="p">)</span>
|
|
|
|
<span class="n">results</span><span class="p">:</span> <span class="n">ParsedReport</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
|
|
|
|
<span class="c1"># parse_report_file tries the three report formats in turn. When all three</span>
|
|
<span class="c1"># reject the input, keep each format's specific error so the final message</span>
|
|
<span class="c1"># can explain *why* the file is invalid instead of a bare "Not a valid</span>
|
|
<span class="c1"># report" (see _describe_parse_failure).</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">report</span> <span class="o">=</span> <span class="n">parse_aggregate_report_file</span><span class="p">(</span>
|
|
<span class="n">content</span><span class="p">,</span>
|
|
<span class="n">config</span><span class="o">=</span><span class="n">cfg</span><span class="p">,</span>
|
|
<span class="n">keep_alive</span><span class="o">=</span><span class="n">keep_alive</span><span class="p">,</span>
|
|
<span class="p">)</span>
|
|
<span class="n">results</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"report_type"</span><span class="p">:</span> <span class="s2">"aggregate"</span><span class="p">,</span> <span class="s2">"report"</span><span class="p">:</span> <span class="n">report</span><span class="p">}</span>
|
|
<span class="k">except</span> <span class="n">InvalidAggregateReport</span> <span class="k">as</span> <span class="n">aggregate_error</span><span class="p">:</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">report</span> <span class="o">=</span> <span class="n">parse_smtp_tls_report_json</span><span class="p">(</span><span class="n">content</span><span class="p">)</span>
|
|
<span class="n">results</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"report_type"</span><span class="p">:</span> <span class="s2">"smtp_tls"</span><span class="p">,</span> <span class="s2">"report"</span><span class="p">:</span> <span class="n">report</span><span class="p">}</span>
|
|
<span class="k">except</span> <span class="n">InvalidSMTPTLSReport</span> <span class="k">as</span> <span class="n">smtp_tls_error</span><span class="p">:</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">results</span> <span class="o">=</span> <span class="n">parse_report_email</span><span class="p">(</span>
|
|
<span class="n">content</span><span class="p">,</span>
|
|
<span class="n">config</span><span class="o">=</span><span class="n">cfg</span><span class="p">,</span>
|
|
<span class="n">keep_alive</span><span class="o">=</span><span class="n">keep_alive</span><span class="p">,</span>
|
|
<span class="p">)</span>
|
|
<span class="k">except</span> <span class="n">InvalidDMARCReport</span> <span class="k">as</span> <span class="n">email_error</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">ParserError</span><span class="p">(</span>
|
|
<span class="n">_describe_parse_failure</span><span class="p">(</span>
|
|
<span class="n">content</span><span class="p">,</span> <span class="n">aggregate_error</span><span class="p">,</span> <span class="n">smtp_tls_error</span><span class="p">,</span> <span class="n">email_error</span>
|
|
<span class="p">)</span>
|
|
<span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">email_error</span>
|
|
|
|
<span class="k">if</span> <span class="n">results</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">ParserError</span><span class="p">(</span><span class="s2">"Not a valid report"</span><span class="p">)</span>
|
|
<span class="k">return</span> <span class="n">results</span></div>
|
|
|
|
|
|
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">_classify_parsed_email</span><span class="p">(</span>
|
|
<span class="n">parsed_email</span><span class="p">:</span> <span class="n">ParsedReport</span><span class="p">,</span>
|
|
<span class="n">aggregate_reports</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">AggregateReport</span><span class="p">],</span>
|
|
<span class="n">failure_reports</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">FailureReport</span><span class="p">],</span>
|
|
<span class="n">smtp_tls_reports</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">SMTPTLSReport</span><span class="p">],</span>
|
|
<span class="o">*</span><span class="p">,</span>
|
|
<span class="n">seen_aggregate_report_ids</span><span class="p">:</span> <span class="n">ExpiringDict</span><span class="p">,</span>
|
|
<span class="n">pending_aggregate_keys</span><span class="p">:</span> <span class="nb">set</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="p">)</span> <span class="o">-></span> <span class="n">ReportType</span><span class="p">:</span>
|
|
<span class="w"> </span><span class="sd">"""Classify a parsed report email, appending it to the matching list.</span>
|
|
|
|
<span class="sd"> Owns the seen-aggregate-report-ID dedup check against</span>
|
|
<span class="sd"> ``seen_aggregate_report_ids``: an aggregate report already seen (keyed</span>
|
|
<span class="sd"> on ``{org_name}_{report_id}``) is logged and dropped instead of</span>
|
|
<span class="sd"> appended. Shared, unmodified, by the sequential and parallel branches</span>
|
|
<span class="sd"> of ``get_dmarc_reports_from_mbox`` and ``get_dmarc_reports_from_mailbox``</span>
|
|
<span class="sd"> so both dedup identically -- callers pass the config's</span>
|
|
<span class="sd"> ``seen_aggregate_report_ids`` cache so dedup state stays scoped to</span>
|
|
<span class="sd"> whichever ``ParserConfig`` (explicit or module-default) is in effect.</span>
|
|
|
|
<span class="sd"> ``pending_aggregate_keys`` lets a caller *defer* the dedup-cache write:</span>
|
|
<span class="sd"> when a set is supplied, a newly seen key is staged in that set instead</span>
|
|
<span class="sd"> of being written to ``seen_aggregate_report_ids``, and the dedup check</span>
|
|
<span class="sd"> consults both. The caller then folds the staged keys into the cache</span>
|
|
<span class="sd"> once the batch is known to have been saved (see</span>
|
|
<span class="sd"> ``get_dmarc_reports_from_mailbox``), or drops them so a retry reparses</span>
|
|
<span class="sd"> the same reports rather than silently skipping them as duplicates.</span>
|
|
<span class="sd"> ``None`` (the default) keeps the immediate-write behavior.</span>
|
|
|
|
<span class="sd"> Returns the report type so mailbox callers know which UID list to</span>
|
|
<span class="sd"> append the source message's UID to.</span>
|
|
<span class="sd"> """</span>
|
|
<span class="n">report_type</span> <span class="o">=</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">"report_type"</span><span class="p">]</span>
|
|
<span class="c1"># Compare against parsed_email["report_type"] directly (not the</span>
|
|
<span class="c1"># report_type local above) in each branch so pyright's TypedDict</span>
|
|
<span class="c1"># discriminated-union narrowing applies to parsed_email["report"].</span>
|
|
<span class="k">if</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">"report_type"</span><span class="p">]</span> <span class="o">==</span> <span class="s2">"aggregate"</span><span class="p">:</span>
|
|
<span class="n">report_org</span> <span class="o">=</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">"report"</span><span class="p">][</span><span class="s2">"report_metadata"</span><span class="p">][</span><span class="s2">"org_name"</span><span class="p">]</span>
|
|
<span class="n">report_id</span> <span class="o">=</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">"report"</span><span class="p">][</span><span class="s2">"report_metadata"</span><span class="p">][</span><span class="s2">"report_id"</span><span class="p">]</span>
|
|
<span class="n">report_key</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">report_org</span><span class="si">}</span><span class="s2">_</span><span class="si">{</span><span class="n">report_id</span><span class="si">}</span><span class="s2">"</span>
|
|
<span class="n">already_seen</span> <span class="o">=</span> <span class="n">report_key</span> <span class="ow">in</span> <span class="n">seen_aggregate_report_ids</span> <span class="ow">or</span> <span class="p">(</span>
|
|
<span class="n">pending_aggregate_keys</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">report_key</span> <span class="ow">in</span> <span class="n">pending_aggregate_keys</span>
|
|
<span class="p">)</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">already_seen</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="n">pending_aggregate_keys</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">seen_aggregate_report_ids</span><span class="p">[</span><span class="n">report_key</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">pending_aggregate_keys</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">report_key</span><span class="p">)</span>
|
|
<span class="n">aggregate_reports</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">parsed_email</span><span class="p">[</span><span class="s2">"report"</span><span class="p">])</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
|
<span class="sa">f</span><span class="s2">"Skipping duplicate aggregate report from </span><span class="si">{</span><span class="n">report_org</span><span class="si">}</span><span class="s2"> "</span>
|
|
<span class="sa">f</span><span class="s2">"with ID: </span><span class="si">{</span><span class="n">report_id</span><span class="si">}</span><span class="s2">"</span>
|
|
<span class="p">)</span>
|
|
<span class="k">elif</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">"report_type"</span><span class="p">]</span> <span class="o">==</span> <span class="s2">"failure"</span><span class="p">:</span>
|
|
<span class="n">failure_reports</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">parsed_email</span><span class="p">[</span><span class="s2">"report"</span><span class="p">])</span>
|
|
<span class="k">elif</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">"report_type"</span><span class="p">]</span> <span class="o">==</span> <span class="s2">"smtp_tls"</span><span class="p">:</span>
|
|
<span class="n">smtp_tls_reports</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">parsed_email</span><span class="p">[</span><span class="s2">"report"</span><span class="p">])</span>
|
|
<span class="k">return</span> <span class="n">report_type</span>
|
|
|
|
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">_fetch_mailbox_message</span><span class="p">(</span>
|
|
<span class="n">connection</span><span class="p">:</span> <span class="n">MailboxConnection</span><span class="p">,</span> <span class="n">msg_uid</span><span class="p">:</span> <span class="n">Any</span><span class="p">,</span> <span class="n">test</span><span class="p">:</span> <span class="nb">bool</span>
|
|
<span class="p">)</span> <span class="o">-></span> <span class="nb">tuple</span><span class="p">[</span><span class="nb">int</span> <span class="o">|</span> <span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]:</span>
|
|
<span class="w"> </span><span class="sd">"""Fetch one message from ``connection`` by UID, casting the UID to the</span>
|
|
<span class="sd"> type each backend's ``fetch_message`` expects.</span>
|
|
|
|
<span class="sd"> Shared, unmodified, by the sequential and parallel branches of</span>
|
|
<span class="sd"> ``get_dmarc_reports_from_mailbox`` so both fetch identically.</span>
|
|
|
|
<span class="sd"> Returns ``(message_id, msg_content)``; ``message_id`` is the</span>
|
|
<span class="sd"> backend-appropriate id to use for later move/delete calls.</span>
|
|
<span class="sd"> """</span>
|
|
<span class="n">message_id</span><span class="p">:</span> <span class="nb">int</span> <span class="o">|</span> <span class="nb">str</span>
|
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">connection</span><span class="p">,</span> <span class="n">IMAPConnection</span><span class="p">):</span>
|
|
<span class="n">message_id</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">msg_uid</span><span class="p">)</span>
|
|
<span class="n">msg_content</span> <span class="o">=</span> <span class="n">connection</span><span class="o">.</span><span class="n">fetch_message</span><span class="p">(</span><span class="n">message_id</span><span class="p">)</span>
|
|
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">connection</span><span class="p">,</span> <span class="n">MSGraphConnection</span><span class="p">):</span>
|
|
<span class="n">message_id</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">msg_uid</span><span class="p">)</span>
|
|
<span class="n">msg_content</span> <span class="o">=</span> <span class="n">connection</span><span class="o">.</span><span class="n">fetch_message</span><span class="p">(</span><span class="n">message_id</span><span class="p">,</span> <span class="n">mark_read</span><span class="o">=</span><span class="ow">not</span> <span class="n">test</span><span class="p">)</span>
|
|
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">connection</span><span class="p">,</span> <span class="n">MaildirConnection</span><span class="p">):</span>
|
|
<span class="n">message_id</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">msg_uid</span><span class="p">)</span> <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">msg_uid</span><span class="p">,</span> <span class="nb">str</span><span class="p">)</span> <span class="k">else</span> <span class="n">msg_uid</span>
|
|
<span class="n">msg_content</span> <span class="o">=</span> <span class="n">connection</span><span class="o">.</span><span class="n">fetch_message</span><span class="p">(</span><span class="n">message_id</span><span class="p">,</span> <span class="n">mark_read</span><span class="o">=</span><span class="ow">not</span> <span class="n">test</span><span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">message_id</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">msg_uid</span><span class="p">)</span> <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">msg_uid</span><span class="p">,</span> <span class="nb">str</span><span class="p">)</span> <span class="k">else</span> <span class="n">msg_uid</span>
|
|
<span class="n">msg_content</span> <span class="o">=</span> <span class="n">connection</span><span class="o">.</span><span class="n">fetch_message</span><span class="p">(</span><span class="n">message_id</span><span class="p">)</span>
|
|
<span class="k">return</span> <span class="n">message_id</span><span class="p">,</span> <span class="n">msg_content</span>
|
|
|
|
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">_dispose_invalid_message</span><span class="p">(</span>
|
|
<span class="n">connection</span><span class="p">:</span> <span class="n">MailboxConnection</span><span class="p">,</span>
|
|
<span class="n">message_id</span><span class="p">:</span> <span class="nb">int</span> <span class="o">|</span> <span class="nb">str</span><span class="p">,</span>
|
|
<span class="n">delete</span><span class="p">:</span> <span class="nb">bool</span><span class="p">,</span>
|
|
<span class="n">invalid_reports_folder</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
|
<span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="w"> </span><span class="sd">"""Delete or move an unparseable message, per ``delete``.</span>
|
|
|
|
<span class="sd"> Callers pass their effective ``delete_invalid`` value as ``delete``.</span>
|
|
|
|
<span class="sd"> Shared, unmodified, by the sequential and parallel branches of</span>
|
|
<span class="sd"> ``get_dmarc_reports_from_mailbox`` so both dispose of invalid messages</span>
|
|
<span class="sd"> identically.</span>
|
|
<span class="sd"> """</span>
|
|
<span class="k">if</span> <span class="n">delete</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Deleting message UID </span><span class="si">{</span><span class="n">message_id</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">connection</span><span class="p">,</span> <span class="n">IMAPConnection</span><span class="p">):</span>
|
|
<span class="n">connection</span><span class="o">.</span><span class="n">delete_message</span><span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="n">message_id</span><span class="p">))</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">connection</span><span class="o">.</span><span class="n">delete_message</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">message_id</span><span class="p">))</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Moving message UID </span><span class="si">{</span><span class="n">message_id</span><span class="si">}</span><span class="s2"> to </span><span class="si">{</span><span class="n">invalid_reports_folder</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">connection</span><span class="p">,</span> <span class="n">IMAPConnection</span><span class="p">):</span>
|
|
<span class="n">connection</span><span class="o">.</span><span class="n">move_message</span><span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="n">message_id</span><span class="p">),</span> <span class="n">invalid_reports_folder</span><span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">connection</span><span class="o">.</span><span class="n">move_message</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">message_id</span><span class="p">),</span> <span class="n">invalid_reports_folder</span><span class="p">)</span>
|
|
|
|
|
|
<div class="viewcode-block" id="get_dmarc_reports_from_mbox">
|
|
<a class="viewcode-back" href="../api.html#parsedmarc.get_dmarc_reports_from_mbox">[docs]</a>
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_dmarc_reports_from_mbox</span><span class="p">(</span>
|
|
<span class="n">input_</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
|
<span class="o">*</span><span class="p">,</span>
|
|
<span class="n">nameservers</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">dns_timeout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="n">DEFAULT_DNS_TIMEOUT</span><span class="p">,</span>
|
|
<span class="n">dns_retries</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="n">DEFAULT_DNS_MAX_RETRIES</span><span class="p">,</span>
|
|
<span class="n">strip_attachment_payloads</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
|
<span class="n">ip_db_path</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">always_use_local_files</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
|
<span class="n">reverse_dns_map_path</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">reverse_dns_map_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">offline</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
|
<span class="n">normalize_timespan_threshold_hours</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">24.0</span><span class="p">,</span>
|
|
<span class="n">n_procs</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
|
|
<span class="n">config</span><span class="p">:</span> <span class="n">ParserConfig</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="p">)</span> <span class="o">-></span> <span class="n">ParsingResults</span><span class="p">:</span>
|
|
<span class="w"> </span><span class="sd">"""Parses a mailbox in mbox format containing e-mails with attached</span>
|
|
<span class="sd"> DMARC reports</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> input_ (str): A path to a mbox file</span>
|
|
<span class="sd"> nameservers (list): A list of one or more nameservers to use</span>
|
|
<span class="sd"> (Cloudflare's public DNS resolvers by default)</span>
|
|
<span class="sd"> dns_timeout (float): Sets the DNS timeout in seconds</span>
|
|
<span class="sd"> dns_retries (int): Number of times to retry DNS queries on timeout</span>
|
|
<span class="sd"> or other transient errors</span>
|
|
<span class="sd"> strip_attachment_payloads (bool): Remove attachment payloads from</span>
|
|
<span class="sd"> failure report results</span>
|
|
<span class="sd"> always_use_local_files (bool): Do not download files</span>
|
|
<span class="sd"> reverse_dns_map_path (str): Path to a reverse DNS map file</span>
|
|
<span class="sd"> reverse_dns_map_url (str): URL to a reverse DNS map file</span>
|
|
<span class="sd"> ip_db_path (str): Path to a MMDB file from IPinfo, MaxMind, or DBIP</span>
|
|
<span class="sd"> offline (bool): Do not make online queries for geolocation or DNS</span>
|
|
<span class="sd"> normalize_timespan_threshold_hours (float): Normalize timespans beyond this</span>
|
|
<span class="sd"> n_procs (int): Number of processes to use for parsing messages in</span>
|
|
<span class="sd"> parallel. Message reading, deduplication, and result assembly</span>
|
|
<span class="sd"> stay in the calling process; only parsing is parallelized. Not</span>
|
|
<span class="sd"> part of ``config``; always applies.</span>
|
|
<span class="sd"> config (ParserConfig): a single object carrying all parsing and</span>
|
|
<span class="sd"> enrichment options plus the caches; when provided, the</span>
|
|
<span class="sd"> individual option keyword arguments listed above are ignored in</span>
|
|
<span class="sd"> favor of the config's values.</span>
|
|
|
|
<span class="sd"> Returns:</span>
|
|
<span class="sd"> dict: Lists of ``aggregate_reports``, ``failure_reports``, and ``smtp_tls_reports``</span>
|
|
|
|
<span class="sd"> """</span>
|
|
<span class="n">cfg</span> <span class="o">=</span> <span class="n">_resolve_config</span><span class="p">(</span>
|
|
<span class="n">config</span><span class="p">,</span>
|
|
<span class="n">offline</span><span class="o">=</span><span class="n">offline</span><span class="p">,</span>
|
|
<span class="n">ip_db_path</span><span class="o">=</span><span class="n">ip_db_path</span><span class="p">,</span>
|
|
<span class="n">always_use_local_files</span><span class="o">=</span><span class="n">always_use_local_files</span><span class="p">,</span>
|
|
<span class="n">reverse_dns_map_path</span><span class="o">=</span><span class="n">reverse_dns_map_path</span><span class="p">,</span>
|
|
<span class="n">reverse_dns_map_url</span><span class="o">=</span><span class="n">reverse_dns_map_url</span><span class="p">,</span>
|
|
<span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
|
|
<span class="n">dns_timeout</span><span class="o">=</span><span class="n">dns_timeout</span><span class="p">,</span>
|
|
<span class="n">dns_retries</span><span class="o">=</span><span class="n">dns_retries</span><span class="p">,</span>
|
|
<span class="n">strip_attachment_payloads</span><span class="o">=</span><span class="n">strip_attachment_payloads</span><span class="p">,</span>
|
|
<span class="n">normalize_timespan_threshold_hours</span><span class="o">=</span><span class="n">normalize_timespan_threshold_hours</span><span class="p">,</span>
|
|
<span class="p">)</span>
|
|
<span class="n">aggregate_reports</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">AggregateReport</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="n">failure_reports</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">FailureReport</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="n">smtp_tls_reports</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">SMTPTLSReport</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">mbox</span> <span class="o">=</span> <span class="n">mailbox</span><span class="o">.</span><span class="n">mbox</span><span class="p">(</span><span class="n">input_</span><span class="p">)</span>
|
|
<span class="n">message_keys</span> <span class="o">=</span> <span class="n">mbox</span><span class="o">.</span><span class="n">keys</span><span class="p">()</span>
|
|
<span class="n">total_messages</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">message_keys</span><span class="p">)</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Found </span><span class="si">{</span><span class="n">total_messages</span><span class="si">}</span><span class="s2"> messages in </span><span class="si">{</span><span class="n">input_</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
|
|
|
<span class="k">if</span> <span class="n">n_procs</span> <span class="o">></span> <span class="mi">1</span> <span class="ow">and</span> <span class="n">total_messages</span> <span class="o">></span> <span class="mi">1</span><span class="p">:</span>
|
|
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.parallel</span><span class="w"> </span><span class="kn">import</span> <span class="n">_parse_report_email_job</span><span class="p">,</span> <span class="n">parallel_map</span>
|
|
|
|
<span class="n">func</span> <span class="o">=</span> <span class="n">functools</span><span class="o">.</span><span class="n">partial</span><span class="p">(</span><span class="n">_parse_report_email_job</span><span class="p">,</span> <span class="n">config</span><span class="o">=</span><span class="n">cfg</span><span class="p">)</span>
|
|
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">_jobs</span><span class="p">():</span>
|
|
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">total_messages</span><span class="p">):</span>
|
|
<span class="n">message_key</span> <span class="o">=</span> <span class="n">message_keys</span><span class="p">[</span><span class="n">i</span><span class="p">]</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Processing message </span><span class="si">{</span><span class="n">i</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="mi">1</span><span class="si">}</span><span class="s2"> of </span><span class="si">{</span><span class="n">total_messages</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
|
<span class="k">yield</span> <span class="n">mbox</span><span class="o">.</span><span class="n">get_string</span><span class="p">(</span><span class="n">message_key</span><span class="p">)</span>
|
|
|
|
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">tqdm</span><span class="p">(</span>
|
|
<span class="n">parallel_map</span><span class="p">(</span><span class="n">func</span><span class="p">,</span> <span class="n">_jobs</span><span class="p">(),</span> <span class="n">n_procs</span><span class="p">),</span>
|
|
<span class="n">total</span><span class="o">=</span><span class="n">total_messages</span><span class="p">,</span>
|
|
<span class="n">disable</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
|
<span class="p">):</span>
|
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">InvalidDMARCReport</span><span class="p">):</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">result</span><span class="p">))</span>
|
|
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">ParserError</span><span class="p">):</span>
|
|
<span class="k">raise</span> <span class="n">result</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">_classify_parsed_email</span><span class="p">(</span>
|
|
<span class="n">result</span><span class="p">,</span>
|
|
<span class="n">aggregate_reports</span><span class="p">,</span>
|
|
<span class="n">failure_reports</span><span class="p">,</span>
|
|
<span class="n">smtp_tls_reports</span><span class="p">,</span>
|
|
<span class="n">seen_aggregate_report_ids</span><span class="o">=</span><span class="n">cfg</span><span class="o">.</span><span class="n">seen_aggregate_report_ids</span><span class="p">,</span>
|
|
<span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">tqdm</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="n">total_messages</span><span class="p">),</span> <span class="n">disable</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
|
<span class="n">message_key</span> <span class="o">=</span> <span class="n">message_keys</span><span class="p">[</span><span class="n">i</span><span class="p">]</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Processing message </span><span class="si">{</span><span class="n">i</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="mi">1</span><span class="si">}</span><span class="s2"> of </span><span class="si">{</span><span class="n">total_messages</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
|
<span class="n">msg_content</span> <span class="o">=</span> <span class="n">mbox</span><span class="o">.</span><span class="n">get_string</span><span class="p">(</span><span class="n">message_key</span><span class="p">)</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">parsed_email</span> <span class="o">=</span> <span class="n">parse_report_email</span><span class="p">(</span><span class="n">msg_content</span><span class="p">,</span> <span class="n">config</span><span class="o">=</span><span class="n">cfg</span><span class="p">)</span>
|
|
<span class="n">_classify_parsed_email</span><span class="p">(</span>
|
|
<span class="n">parsed_email</span><span class="p">,</span>
|
|
<span class="n">aggregate_reports</span><span class="p">,</span>
|
|
<span class="n">failure_reports</span><span class="p">,</span>
|
|
<span class="n">smtp_tls_reports</span><span class="p">,</span>
|
|
<span class="n">seen_aggregate_report_ids</span><span class="o">=</span><span class="n">cfg</span><span class="o">.</span><span class="n">seen_aggregate_report_ids</span><span class="p">,</span>
|
|
<span class="p">)</span>
|
|
<span class="k">except</span> <span class="n">InvalidDMARCReport</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
|
<span class="k">except</span> <span class="n">mailbox</span><span class="o">.</span><span class="n">NoSuchMailboxError</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">InvalidDMARCReport</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Mailbox </span><span class="si">{</span><span class="n">input_</span><span class="si">}</span><span class="s2"> does not exist"</span><span class="p">)</span>
|
|
<span class="k">return</span> <span class="p">{</span>
|
|
<span class="s2">"aggregate_reports"</span><span class="p">:</span> <span class="n">aggregate_reports</span><span class="p">,</span>
|
|
<span class="s2">"failure_reports"</span><span class="p">:</span> <span class="n">failure_reports</span><span class="p">,</span>
|
|
<span class="s2">"smtp_tls_reports"</span><span class="p">:</span> <span class="n">smtp_tls_reports</span><span class="p">,</span>
|
|
<span class="p">}</span></div>
|
|
|
|
|
|
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">_migrate_forensic_archive_folder</span><span class="p">(</span>
|
|
<span class="n">connection</span><span class="p">:</span> <span class="n">MailboxConnection</span><span class="p">,</span> <span class="n">archive_folder</span><span class="p">:</span> <span class="nb">str</span>
|
|
<span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="w"> </span><span class="sd">"""Consolidate a pre-rename ``<archive>/Forensic`` subfolder into</span>
|
|
<span class="sd"> ``<archive>/Failure``.</span>
|
|
|
|
<span class="sd"> Before failure reports were renamed from "forensic" reports, they were</span>
|
|
<span class="sd"> archived under ``<archive_folder>/Forensic``; they now go to</span>
|
|
<span class="sd"> ``<archive_folder>/Failure``. This best-effort, run-on-startup migration</span>
|
|
<span class="sd"> moves any pre-existing legacy archive into the new location so reports</span>
|
|
<span class="sd"> filed before and after the rename live in the same folder.</span>
|
|
|
|
<span class="sd"> It is a no-op when there is no legacy ``Forensic`` folder (the common</span>
|
|
<span class="sd"> case), and never raises: a mailbox that cannot be reorganized is logged</span>
|
|
<span class="sd"> and skipped, consistent with the rest of parsedmarc's mailbox handling</span>
|
|
<span class="sd"> (warn, don't crash). Uses the folder-management API added in mailsuite</span>
|
|
<span class="sd"> 2.1.0 (``folder_exists`` / ``rename_folder`` / ``merge_folders``).</span>
|
|
<span class="sd"> """</span>
|
|
<span class="n">old_folder</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">archive_folder</span><span class="si">}</span><span class="s2">/Forensic"</span>
|
|
<span class="n">new_folder</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">archive_folder</span><span class="si">}</span><span class="s2">/Failure"</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">connection</span><span class="o">.</span><span class="n">folder_exists</span><span class="p">(</span><span class="n">old_folder</span><span class="p">):</span>
|
|
<span class="k">return</span>
|
|
<span class="k">if</span> <span class="n">connection</span><span class="o">.</span><span class="n">folder_exists</span><span class="p">(</span><span class="n">new_folder</span><span class="p">):</span>
|
|
<span class="c1"># Both exist (e.g. a partial earlier migration, or a manually</span>
|
|
<span class="c1"># created Failure folder): move the legacy folder's messages into</span>
|
|
<span class="c1"># the new one and drop the now-empty legacy folder.</span>
|
|
<span class="n">connection</span><span class="o">.</span><span class="n">merge_folders</span><span class="p">(</span><span class="n">old_folder</span><span class="p">,</span> <span class="n">new_folder</span><span class="p">)</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Merged legacy archive folder </span><span class="si">{</span><span class="n">old_folder</span><span class="si">}</span><span class="s2"> into </span><span class="si">{</span><span class="n">new_folder</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">connection</span><span class="o">.</span><span class="n">rename_folder</span><span class="p">(</span><span class="n">old_folder</span><span class="p">,</span> <span class="n">new_folder</span><span class="p">)</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Renamed legacy archive folder </span><span class="si">{</span><span class="n">old_folder</span><span class="si">}</span><span class="s2"> to </span><span class="si">{</span><span class="n">new_folder</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
|
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
|
<span class="sa">f</span><span class="s2">"Could not migrate legacy archive folder </span><span class="si">{</span><span class="n">old_folder</span><span class="si">}</span><span class="s2"> to </span><span class="si">{</span><span class="n">new_folder</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="n">error</span><span class="si">}</span><span class="s2">"</span>
|
|
<span class="p">)</span>
|
|
|
|
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">_ensure_folder</span><span class="p">(</span><span class="n">connection</span><span class="p">:</span> <span class="n">MailboxConnection</span><span class="p">,</span> <span class="n">folder</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="w"> </span><span class="sd">"""Best-effort create ``folder`` if the backend says it is missing.</span>
|
|
|
|
<span class="sd"> ``get_dmarc_reports_from_mailbox()`` creates its destination folders up</span>
|
|
<span class="sd"> front, but only when ``create_folders`` is set -- watch mode calls it</span>
|
|
<span class="sd"> with ``create_folders=False``, and the ``Unsaved`` holding folder is</span>
|
|
<span class="sd"> only created up front when a ``save_callback`` was supplied. This is the</span>
|
|
<span class="sd"> defensive check immediately before a message is moved there.</span>
|
|
|
|
<span class="sd"> Like ``_migrate_forensic_archive_folder``, it never raises: a backend</span>
|
|
<span class="sd"> that cannot report on or create the folder is logged and skipped, and</span>
|
|
<span class="sd"> the move that follows either succeeds anyway (the folder already</span>
|
|
<span class="sd"> existed) or fails and is logged by its own error handler -- warn, don't</span>
|
|
<span class="sd"> crash.</span>
|
|
<span class="sd"> """</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">connection</span><span class="o">.</span><span class="n">folder_exists</span><span class="p">(</span><span class="n">folder</span><span class="p">):</span>
|
|
<span class="n">connection</span><span class="o">.</span><span class="n">create_folder</span><span class="p">(</span><span class="n">folder</span><span class="p">)</span>
|
|
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Could not create folder </span><span class="si">{</span><span class="n">folder</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="n">error</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
|
|
|
|
|
<div class="viewcode-block" id="get_dmarc_reports_from_mailbox">
|
|
<a class="viewcode-back" href="../api.html#parsedmarc.get_dmarc_reports_from_mailbox">[docs]</a>
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_dmarc_reports_from_mailbox</span><span class="p">(</span>
|
|
<span class="n">connection</span><span class="p">:</span> <span class="n">MailboxConnection</span><span class="p">,</span>
|
|
<span class="o">*</span><span class="p">,</span>
|
|
<span class="n">reports_folder</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"INBOX"</span><span class="p">,</span>
|
|
<span class="n">archive_folder</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"Archive"</span><span class="p">,</span>
|
|
<span class="n">delete</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
|
<span class="n">delete_aggregate</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">delete_failure</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">delete_smtp_tls</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">delete_invalid</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">test</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
|
<span class="n">ip_db_path</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">always_use_local_files</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
|
<span class="n">reverse_dns_map_path</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">reverse_dns_map_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">offline</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
|
<span class="n">nameservers</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">dns_timeout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="n">DEFAULT_DNS_TIMEOUT</span><span class="p">,</span>
|
|
<span class="n">dns_retries</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="n">DEFAULT_DNS_MAX_RETRIES</span><span class="p">,</span>
|
|
<span class="n">strip_attachment_payloads</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
|
<span class="n">results</span><span class="p">:</span> <span class="n">ParsingResults</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">batch_size</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">10</span><span class="p">,</span>
|
|
<span class="n">since</span><span class="p">:</span> <span class="n">datetime</span> <span class="o">|</span> <span class="n">date</span> <span class="o">|</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">create_folders</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
|
|
<span class="n">normalize_timespan_threshold_hours</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">24.0</span><span class="p">,</span>
|
|
<span class="n">n_procs</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
|
|
<span class="n">save_callback</span><span class="p">:</span> <span class="n">Callable</span><span class="p">[[</span><span class="n">ParsingResults</span><span class="p">],</span> <span class="nb">bool</span> <span class="o">|</span> <span class="kc">None</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">max_unsaved_retries</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">2</span><span class="p">,</span>
|
|
<span class="n">config</span><span class="p">:</span> <span class="n">ParserConfig</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="p">)</span> <span class="o">-></span> <span class="n">ParsingResults</span><span class="p">:</span>
|
|
<span class="w"> </span><span class="sd">"""</span>
|
|
<span class="sd"> Fetches and parses DMARC reports from a mailbox</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> connection: A Mailbox connection object</span>
|
|
<span class="sd"> reports_folder (str): The folder where reports can be found</span>
|
|
<span class="sd"> archive_folder (str): The folder to move processed mail to</span>
|
|
<span class="sd"> delete (bool): Delete messages after processing them</span>
|
|
<span class="sd"> delete_aggregate (bool | None): Delete aggregate report messages</span>
|
|
<span class="sd"> after processing them, instead of moving them to the</span>
|
|
<span class="sd"> ``Aggregate`` archive subfolder; ``None`` (the default) inherits</span>
|
|
<span class="sd"> the value of ``delete``</span>
|
|
<span class="sd"> delete_failure (bool | None): Delete failure report messages after</span>
|
|
<span class="sd"> processing them, instead of moving them to the ``Failure``</span>
|
|
<span class="sd"> archive subfolder; ``None`` (the default) inherits the value of</span>
|
|
<span class="sd"> ``delete``</span>
|
|
<span class="sd"> delete_smtp_tls (bool | None): Delete SMTP TLS report messages after</span>
|
|
<span class="sd"> processing them, instead of moving them to the ``SMTP-TLS``</span>
|
|
<span class="sd"> archive subfolder; ``None`` (the default) inherits the value of</span>
|
|
<span class="sd"> ``delete``</span>
|
|
<span class="sd"> delete_invalid (bool | None): Delete unparseable messages, instead</span>
|
|
<span class="sd"> of moving them to the ``Invalid`` archive subfolder where they</span>
|
|
<span class="sd"> can be inspected for debugging; ``None`` (the default) inherits</span>
|
|
<span class="sd"> the value of ``delete``</span>
|
|
<span class="sd"> test (bool): Do not move or delete messages after processing them</span>
|
|
<span class="sd"> ip_db_path (str): Path to a MMDB file from IPinfo, MaxMind, or DBIP</span>
|
|
<span class="sd"> always_use_local_files (bool): Do not download files</span>
|
|
<span class="sd"> reverse_dns_map_path (str): Path to a reverse DNS map file</span>
|
|
<span class="sd"> reverse_dns_map_url (str): URL to a reverse DNS map file</span>
|
|
<span class="sd"> offline (bool): Do not query online for geolocation or DNS</span>
|
|
<span class="sd"> nameservers (list): A list of DNS nameservers to query</span>
|
|
<span class="sd"> dns_timeout (float): Set the DNS query timeout</span>
|
|
<span class="sd"> dns_retries (int): Number of times to retry DNS queries on timeout</span>
|
|
<span class="sd"> or other transient errors</span>
|
|
<span class="sd"> strip_attachment_payloads (bool): Remove attachment payloads from</span>
|
|
<span class="sd"> failure report results</span>
|
|
<span class="sd"> results (dict): Results from the previous run</span>
|
|
<span class="sd"> batch_size (int): Number of messages to read and process before saving</span>
|
|
<span class="sd"> (use 0 for no limit)</span>
|
|
<span class="sd"> since: Search for messages since certain time</span>
|
|
<span class="sd"> (units - {"m":"minutes", "h":"hours", "d":"days", "w":"weeks"})</span>
|
|
<span class="sd"> create_folders (bool): Whether to create the destination folders</span>
|
|
<span class="sd"> (not used in watch)</span>
|
|
<span class="sd"> normalize_timespan_threshold_hours (float): Normalize timespans beyond this</span>
|
|
<span class="sd"> n_procs (int): Number of processes to use for parsing messages in</span>
|
|
<span class="sd"> parallel. Fetching, archiving, and deduplication remain</span>
|
|
<span class="sd"> sequential in the calling process; only parsing is</span>
|
|
<span class="sd"> parallelized. With ``n_procs > 1``, invalid-message disposition</span>
|
|
<span class="sd"> happens after the parsing phase completes, rather than</span>
|
|
<span class="sd"> interleaved message-by-message as it is when ``n_procs`` is 1.</span>
|
|
<span class="sd"> Not part of ``config``; always applies.</span>
|
|
<span class="sd"> save_callback: An optional callable invoked once per fetched batch</span>
|
|
<span class="sd"> with a ``ParsingResults`` dict holding only that batch's newly</span>
|
|
<span class="sd"> parsed reports, after parsing but before any of the batch's</span>
|
|
<span class="sd"> messages are deleted or moved out of ``reports_folder``. It</span>
|
|
<span class="sd"> tells this function whether the batch was actually persisted:</span>
|
|
|
|
<span class="sd"> * Returning ``False`` means "not saved": the batch's messages</span>
|
|
<span class="sd"> are left in ``reports_folder`` for retry on the next run (or</span>
|
|
<span class="sd"> moved to ``{archive_folder}/Unsaved`` once they have failed</span>
|
|
<span class="sd"> ``max_unsaved_retries`` retries -- see below), and the</span>
|
|
<span class="sd"> aggregate-report dedup cache is not updated for that batch, so</span>
|
|
<span class="sd"> a retry reparses the same reports instead of skipping them as</span>
|
|
<span class="sd"> duplicates.</span>
|
|
<span class="sd"> * Raising counts as "not saved" too: the same bookkeeping runs</span>
|
|
<span class="sd"> (the batch's messages are held back or moved to ``Unsaved`` at</span>
|
|
<span class="sd"> the cap, and the failed attempt counts toward</span>
|
|
<span class="sd"> ``max_unsaved_retries``), and the exception is then re-raised</span>
|
|
<span class="sd"> to the caller.</span>
|
|
<span class="sd"> * Any other return value, including ``None``, commits the batch:</span>
|
|
<span class="sd"> the dedup cache is updated and the messages are deleted or</span>
|
|
<span class="sd"> archived exactly as they are with no callback.</span>
|
|
|
|
<span class="sd"> ``None`` (the default) commits every batch, preserving the prior</span>
|
|
<span class="sd"> behavior. The callback is still invoked when ``test`` is</span>
|
|
<span class="sd"> ``True``, so a test run exercises the full pipeline, but neither</span>
|
|
<span class="sd"> the mailbox nor the retry counters are touched regardless of</span>
|
|
<span class="sd"> what it returns.</span>
|
|
<span class="sd"> max_unsaved_retries (int): How many times a message may be *retried*</span>
|
|
<span class="sd"> after ``save_callback`` first reported its batch unsaved, before</span>
|
|
<span class="sd"> it is moved to the ``Unsaved`` archive subfolder instead of</span>
|
|
<span class="sd"> being retried again (default 2, i.e. the initial attempt plus</span>
|
|
<span class="sd"> two retries -- at most three deliveries to any output</span>
|
|
<span class="sd"> destination that does not deduplicate). ``0`` moves a message on</span>
|
|
<span class="sd"> the first failed save; negative values raise ``ValueError``.</span>
|
|
<span class="sd"> Messages moved to ``Unsaved`` are never deleted, whatever the</span>
|
|
<span class="sd"> ``delete`` options say; recover them by fixing the output</span>
|
|
<span class="sd"> destination and moving them back into ``reports_folder``.</span>
|
|
<span class="sd"> Counts are kept in memory, per message and</span>
|
|
<span class="sd"> per process, are reset by a successful save, and are only kept</span>
|
|
<span class="sd"> when a ``save_callback`` is supplied -- so the cap applies</span>
|
|
<span class="sd"> across repeated calls within one process (``watch_inbox()``'s</span>
|
|
<span class="sd"> checks), not across separate one-shot processes, each of which</span>
|
|
<span class="sd"> starts a message's count over. Mailbox orchestration, so like</span>
|
|
<span class="sd"> ``batch_size`` it is not part of ``config``.</span>
|
|
<span class="sd"> config (ParserConfig): a single object carrying all parsing and</span>
|
|
<span class="sd"> enrichment options plus the caches; when provided, it replaces</span>
|
|
<span class="sd"> the individual parsing and enrichment option keyword arguments</span>
|
|
<span class="sd"> listed above (DNS, GeoIP, offline mode, attachment payload</span>
|
|
<span class="sd"> stripping, timespan normalization), whose values are then</span>
|
|
<span class="sd"> ignored. The remaining keyword arguments control mailbox</span>
|
|
<span class="sd"> handling and orchestration rather than parsing (the folder</span>
|
|
<span class="sd"> names, the ``delete`` options, ``test``, ``since``,</span>
|
|
<span class="sd"> ``batch_size``, ``save_callback``, ``max_unsaved_retries``);</span>
|
|
<span class="sd"> they are not part of ``config`` and always apply.</span>
|
|
|
|
<span class="sd"> Returns:</span>
|
|
<span class="sd"> dict: Lists of ``aggregate_reports``, ``failure_reports``, and ``smtp_tls_reports``</span>
|
|
<span class="sd"> """</span>
|
|
<span class="c1"># Each per-report-type flag inherits the overall ``delete`` value when it</span>
|
|
<span class="c1"># is left unset (``None``). Resolve once, up front: every decision below</span>
|
|
<span class="c1"># reads only these four resolved flags. The raw ``delete`` parameter is</span>
|
|
<span class="c1"># still forwarded verbatim to the recursive self-call at the end of this</span>
|
|
<span class="c1"># function, where it is inert because all four resolved flags accompany</span>
|
|
<span class="c1"># it.</span>
|
|
<span class="n">delete_aggregate</span> <span class="o">=</span> <span class="n">delete</span> <span class="k">if</span> <span class="n">delete_aggregate</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">delete_aggregate</span>
|
|
<span class="n">delete_failure</span> <span class="o">=</span> <span class="n">delete</span> <span class="k">if</span> <span class="n">delete_failure</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">delete_failure</span>
|
|
<span class="n">delete_smtp_tls</span> <span class="o">=</span> <span class="n">delete</span> <span class="k">if</span> <span class="n">delete_smtp_tls</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">delete_smtp_tls</span>
|
|
<span class="n">delete_invalid</span> <span class="o">=</span> <span class="n">delete</span> <span class="k">if</span> <span class="n">delete_invalid</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">delete_invalid</span>
|
|
|
|
<span class="k">if</span> <span class="n">test</span> <span class="ow">and</span> <span class="p">(</span>
|
|
<span class="n">delete_aggregate</span> <span class="ow">or</span> <span class="n">delete_failure</span> <span class="ow">or</span> <span class="n">delete_smtp_tls</span> <span class="ow">or</span> <span class="n">delete_invalid</span>
|
|
<span class="p">):</span>
|
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"delete options and test are mutually exclusive"</span><span class="p">)</span>
|
|
|
|
<span class="k">if</span> <span class="n">max_unsaved_retries</span> <span class="o"><</span> <span class="mi">0</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"max_unsaved_retries must be >= 0"</span><span class="p">)</span>
|
|
|
|
<span class="k">if</span> <span class="n">connection</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"Must supply a connection"</span><span class="p">)</span>
|
|
|
|
<span class="n">cfg</span> <span class="o">=</span> <span class="n">_resolve_config</span><span class="p">(</span>
|
|
<span class="n">config</span><span class="p">,</span>
|
|
<span class="n">offline</span><span class="o">=</span><span class="n">offline</span><span class="p">,</span>
|
|
<span class="n">ip_db_path</span><span class="o">=</span><span class="n">ip_db_path</span><span class="p">,</span>
|
|
<span class="n">always_use_local_files</span><span class="o">=</span><span class="n">always_use_local_files</span><span class="p">,</span>
|
|
<span class="n">reverse_dns_map_path</span><span class="o">=</span><span class="n">reverse_dns_map_path</span><span class="p">,</span>
|
|
<span class="n">reverse_dns_map_url</span><span class="o">=</span><span class="n">reverse_dns_map_url</span><span class="p">,</span>
|
|
<span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
|
|
<span class="n">dns_timeout</span><span class="o">=</span><span class="n">dns_timeout</span><span class="p">,</span>
|
|
<span class="n">dns_retries</span><span class="o">=</span><span class="n">dns_retries</span><span class="p">,</span>
|
|
<span class="n">strip_attachment_payloads</span><span class="o">=</span><span class="n">strip_attachment_payloads</span><span class="p">,</span>
|
|
<span class="n">normalize_timespan_threshold_hours</span><span class="o">=</span><span class="n">normalize_timespan_threshold_hours</span><span class="p">,</span>
|
|
<span class="p">)</span>
|
|
|
|
<span class="c1"># current_time useful to fetch_messages later in the program</span>
|
|
<span class="n">current_time</span><span class="p">:</span> <span class="n">datetime</span> <span class="o">|</span> <span class="n">date</span> <span class="o">|</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
|
|
|
|
<span class="n">aggregate_reports</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">AggregateReport</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="n">failure_reports</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">FailureReport</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="n">smtp_tls_reports</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">SMTPTLSReport</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="c1"># This call's own reports, kept separate from the accumulated lists</span>
|
|
<span class="c1"># above (which carry earlier batches' reports in via ``results``) so the</span>
|
|
<span class="c1"># save callback is handed only what this batch parsed.</span>
|
|
<span class="n">batch_aggregate_reports</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">AggregateReport</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="n">batch_failure_reports</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">FailureReport</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="n">batch_smtp_tls_reports</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">SMTPTLSReport</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="c1"># Aggregate dedup keys are staged here and only written to the shared</span>
|
|
<span class="c1"># cache once the batch is known to be saved, so an unsaved batch (or a</span>
|
|
<span class="c1"># mid-batch crash) leaves the cache clean and the reports are reparsed</span>
|
|
<span class="c1"># on the retry instead of being dropped as duplicates.</span>
|
|
<span class="n">pending_aggregate_keys</span><span class="p">:</span> <span class="nb">set</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
|
|
<span class="n">aggregate_report_msg_uids</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="n">failure_report_msg_uids</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="n">smtp_tls_msg_uids</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="n">aggregate_reports_folder</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">archive_folder</span><span class="si">}</span><span class="s2">/Aggregate"</span>
|
|
<span class="n">failure_reports_folder</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">archive_folder</span><span class="si">}</span><span class="s2">/Failure"</span>
|
|
<span class="n">smtp_tls_reports_folder</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">archive_folder</span><span class="si">}</span><span class="s2">/SMTP-TLS"</span>
|
|
<span class="n">invalid_reports_folder</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">archive_folder</span><span class="si">}</span><span class="s2">/Invalid"</span>
|
|
<span class="n">unsaved_reports_folder</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">archive_folder</span><span class="si">}</span><span class="s2">/Unsaved"</span>
|
|
|
|
<span class="k">if</span> <span class="n">results</span><span class="p">:</span>
|
|
<span class="n">aggregate_reports</span> <span class="o">=</span> <span class="n">results</span><span class="p">[</span><span class="s2">"aggregate_reports"</span><span class="p">]</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
|
<span class="n">failure_reports</span> <span class="o">=</span> <span class="n">results</span><span class="p">[</span><span class="s2">"failure_reports"</span><span class="p">]</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
|
<span class="n">smtp_tls_reports</span> <span class="o">=</span> <span class="n">results</span><span class="p">[</span><span class="s2">"smtp_tls_reports"</span><span class="p">]</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
|
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">test</span> <span class="ow">and</span> <span class="n">create_folders</span><span class="p">:</span>
|
|
<span class="n">_migrate_forensic_archive_folder</span><span class="p">(</span><span class="n">connection</span><span class="p">,</span> <span class="n">archive_folder</span><span class="p">)</span>
|
|
<span class="n">connection</span><span class="o">.</span><span class="n">create_folder</span><span class="p">(</span><span class="n">archive_folder</span><span class="p">)</span>
|
|
<span class="n">connection</span><span class="o">.</span><span class="n">create_folder</span><span class="p">(</span><span class="n">aggregate_reports_folder</span><span class="p">)</span>
|
|
<span class="n">connection</span><span class="o">.</span><span class="n">create_folder</span><span class="p">(</span><span class="n">failure_reports_folder</span><span class="p">)</span>
|
|
<span class="n">connection</span><span class="o">.</span><span class="n">create_folder</span><span class="p">(</span><span class="n">smtp_tls_reports_folder</span><span class="p">)</span>
|
|
<span class="n">connection</span><span class="o">.</span><span class="n">create_folder</span><span class="p">(</span><span class="n">invalid_reports_folder</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">save_callback</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="c1"># Only reachable when a callback can report a batch unsaved;</span>
|
|
<span class="c1"># without one no message is ever held back, so the folder would</span>
|
|
<span class="c1"># sit empty in every mailbox.</span>
|
|
<span class="n">connection</span><span class="o">.</span><span class="n">create_folder</span><span class="p">(</span><span class="n">unsaved_reports_folder</span><span class="p">)</span>
|
|
|
|
<span class="k">if</span> <span class="n">since</span> <span class="ow">and</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">since</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
|
|
<span class="n">_since</span> <span class="o">=</span> <span class="mi">1440</span> <span class="c1"># default one day</span>
|
|
<span class="k">if</span> <span class="n">re</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="sa">r</span><span class="s2">"\d+[mhdw]$"</span><span class="p">,</span> <span class="n">since</span><span class="p">):</span>
|
|
<span class="n">s</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="sa">r</span><span class="s2">"(\d+)"</span><span class="p">,</span> <span class="n">since</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">s</span><span class="p">[</span><span class="mi">2</span><span class="p">]</span> <span class="o">==</span> <span class="s2">"m"</span><span class="p">:</span>
|
|
<span class="n">_since</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">s</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span>
|
|
<span class="k">elif</span> <span class="n">s</span><span class="p">[</span><span class="mi">2</span><span class="p">]</span> <span class="o">==</span> <span class="s2">"h"</span><span class="p">:</span>
|
|
<span class="n">_since</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">s</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span> <span class="o">*</span> <span class="mi">60</span>
|
|
<span class="k">elif</span> <span class="n">s</span><span class="p">[</span><span class="mi">2</span><span class="p">]</span> <span class="o">==</span> <span class="s2">"d"</span><span class="p">:</span>
|
|
<span class="n">_since</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">s</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span> <span class="o">*</span> <span class="mi">60</span> <span class="o">*</span> <span class="mi">24</span>
|
|
<span class="k">elif</span> <span class="n">s</span><span class="p">[</span><span class="mi">2</span><span class="p">]</span> <span class="o">==</span> <span class="s2">"w"</span><span class="p">:</span>
|
|
<span class="n">_since</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">s</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span> <span class="o">*</span> <span class="mi">60</span> <span class="o">*</span> <span class="mi">24</span> <span class="o">*</span> <span class="mi">7</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
|
<span class="sa">f</span><span class="s2">"Incorrect format for 'since' option. Provided value: </span><span class="si">{</span><span class="n">since</span><span class="si">}</span><span class="s2">, "</span>
|
|
<span class="s2">"expected values: (5m|3h|2d|1w). Ignoring option, fetching "</span>
|
|
<span class="s2">"messages for last 24hrs. SMTP does not support a time or "</span>
|
|
<span class="s2">"timezone in since. See "</span>
|
|
<span class="s2">"https://www.rfc-editor.org/rfc/rfc3501#page-52"</span>
|
|
<span class="p">)</span>
|
|
|
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">connection</span><span class="p">,</span> <span class="n">IMAPConnection</span><span class="p">):</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
|
<span class="s2">"Only days and weeks values in 'since' option are considered "</span>
|
|
<span class="s2">"for IMAP connections. Examples: 2d or 1w"</span>
|
|
<span class="p">)</span>
|
|
<span class="n">since</span> <span class="o">=</span> <span class="p">(</span><span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">(</span><span class="n">timezone</span><span class="o">.</span><span class="n">utc</span><span class="p">)</span> <span class="o">-</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">minutes</span><span class="o">=</span><span class="n">_since</span><span class="p">))</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span>
|
|
<span class="s2">"</span><span class="si">%d</span><span class="s2">-%b-%Y"</span>
|
|
<span class="p">)</span>
|
|
<span class="n">current_time</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">(</span><span class="n">timezone</span><span class="o">.</span><span class="n">utc</span><span class="p">)</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"</span><span class="si">%d</span><span class="s2">-%b-%Y"</span><span class="p">)</span>
|
|
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">connection</span><span class="p">,</span> <span class="n">MSGraphConnection</span><span class="p">):</span>
|
|
<span class="n">since</span> <span class="o">=</span> <span class="p">(</span><span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">(</span><span class="n">timezone</span><span class="o">.</span><span class="n">utc</span><span class="p">)</span> <span class="o">-</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">minutes</span><span class="o">=</span><span class="n">_since</span><span class="p">))</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span>
|
|
<span class="n">current_time</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">(</span><span class="n">timezone</span><span class="o">.</span><span class="n">utc</span><span class="p">)</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span>
|
|
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">connection</span><span class="p">,</span> <span class="n">GmailConnection</span><span class="p">):</span>
|
|
<span class="n">since</span> <span class="o">=</span> <span class="p">(</span><span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">(</span><span class="n">timezone</span><span class="o">.</span><span class="n">utc</span><span class="p">)</span> <span class="o">-</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">minutes</span><span class="o">=</span><span class="n">_since</span><span class="p">))</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span>
|
|
<span class="s2">"</span><span class="si">%s</span><span class="s2">"</span>
|
|
<span class="p">)</span>
|
|
<span class="n">current_time</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">(</span><span class="n">timezone</span><span class="o">.</span><span class="n">utc</span><span class="p">)</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"</span><span class="si">%s</span><span class="s2">"</span><span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="k">pass</span>
|
|
|
|
<span class="n">messages</span> <span class="o">=</span> <span class="n">connection</span><span class="o">.</span><span class="n">fetch_messages</span><span class="p">(</span>
|
|
<span class="n">reports_folder</span><span class="p">,</span> <span class="n">batch_size</span><span class="o">=</span><span class="n">batch_size</span><span class="p">,</span> <span class="n">since</span><span class="o">=</span><span class="n">since</span>
|
|
<span class="p">)</span>
|
|
<span class="n">total_messages</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">messages</span><span class="p">)</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Found </span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="n">messages</span><span class="p">)</span><span class="si">}</span><span class="s2"> messages in </span><span class="si">{</span><span class="n">reports_folder</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
|
|
|
<span class="k">if</span> <span class="n">batch_size</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">since</span><span class="p">:</span>
|
|
<span class="n">message_limit</span> <span class="o">=</span> <span class="nb">min</span><span class="p">(</span><span class="n">total_messages</span><span class="p">,</span> <span class="n">batch_size</span><span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">message_limit</span> <span class="o">=</span> <span class="n">total_messages</span>
|
|
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Processing </span><span class="si">{</span><span class="n">message_limit</span><span class="si">}</span><span class="s2"> messages"</span><span class="p">)</span>
|
|
|
|
<span class="k">if</span> <span class="n">n_procs</span> <span class="o">></span> <span class="mi">1</span> <span class="ow">and</span> <span class="n">message_limit</span> <span class="o">></span> <span class="mi">1</span><span class="p">:</span>
|
|
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.parallel</span><span class="w"> </span><span class="kn">import</span> <span class="n">_parse_report_email_job</span><span class="p">,</span> <span class="n">parallel_map</span>
|
|
|
|
<span class="c1"># The config's caches (ip_address_cache, seen_aggregate_report_ids,</span>
|
|
<span class="c1"># reverse_dns_map) never cross the process boundary --</span>
|
|
<span class="c1"># ParserConfig.__getstate__ drops them, and each worker accumulates</span>
|
|
<span class="c1"># its own via the module defaults it rebinds to on unpickling (see</span>
|
|
<span class="c1"># ParserConfig.__setstate__). keep_alive is a bound method of the</span>
|
|
<span class="c1"># live connection object and is not a ParserConfig field, so it is</span>
|
|
<span class="c1"># never submitted to the pool either; the heartbeat passed to</span>
|
|
<span class="c1"># parallel_map below keeps the connection alive instead.</span>
|
|
<span class="n">func</span> <span class="o">=</span> <span class="n">functools</span><span class="o">.</span><span class="n">partial</span><span class="p">(</span><span class="n">_parse_report_email_job</span><span class="p">,</span> <span class="n">config</span><span class="o">=</span><span class="n">cfg</span><span class="p">)</span>
|
|
|
|
<span class="c1"># parallel_map yields results in submission order, so the oldest</span>
|
|
<span class="c1"># queued id always belongs to the next yielded result; popping as</span>
|
|
<span class="c1"># results arrive keeps this queue no larger than the in-flight</span>
|
|
<span class="c1"># submission window.</span>
|
|
<span class="n">fetched_ids</span><span class="p">:</span> <span class="n">deque</span><span class="p">[</span><span class="nb">int</span> <span class="o">|</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">deque</span><span class="p">()</span>
|
|
<span class="n">invalid_msg_ids</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">int</span> <span class="o">|</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
|
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">_jobs</span><span class="p">():</span>
|
|
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">message_limit</span><span class="p">):</span>
|
|
<span class="n">msg_uid</span> <span class="o">=</span> <span class="n">messages</span><span class="p">[</span><span class="n">i</span><span class="p">]</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
|
<span class="sa">f</span><span class="s2">"Processing message </span><span class="si">{</span><span class="n">i</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="mi">1</span><span class="si">}</span><span class="s2"> of </span><span class="si">{</span><span class="n">message_limit</span><span class="si">}</span><span class="s2">: UID </span><span class="si">{</span><span class="n">msg_uid</span><span class="si">}</span><span class="s2">"</span>
|
|
<span class="p">)</span>
|
|
<span class="n">message_id</span><span class="p">,</span> <span class="n">msg_content</span> <span class="o">=</span> <span class="n">_fetch_mailbox_message</span><span class="p">(</span>
|
|
<span class="n">connection</span><span class="p">,</span> <span class="n">msg_uid</span><span class="p">,</span> <span class="n">test</span>
|
|
<span class="p">)</span>
|
|
<span class="n">fetched_ids</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">message_id</span><span class="p">)</span>
|
|
<span class="k">yield</span> <span class="n">msg_content</span>
|
|
|
|
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">parallel_map</span><span class="p">(</span>
|
|
<span class="n">func</span><span class="p">,</span> <span class="n">_jobs</span><span class="p">(),</span> <span class="n">n_procs</span><span class="p">,</span> <span class="n">heartbeat</span><span class="o">=</span><span class="n">connection</span><span class="o">.</span><span class="n">keepalive</span>
|
|
<span class="p">):</span>
|
|
<span class="n">message_id</span> <span class="o">=</span> <span class="n">fetched_ids</span><span class="o">.</span><span class="n">popleft</span><span class="p">()</span>
|
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">ParserError</span><span class="p">):</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">result</span><span class="p">))</span>
|
|
<span class="n">invalid_msg_ids</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">message_id</span><span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">report_type</span> <span class="o">=</span> <span class="n">_classify_parsed_email</span><span class="p">(</span>
|
|
<span class="n">result</span><span class="p">,</span>
|
|
<span class="n">batch_aggregate_reports</span><span class="p">,</span>
|
|
<span class="n">batch_failure_reports</span><span class="p">,</span>
|
|
<span class="n">batch_smtp_tls_reports</span><span class="p">,</span>
|
|
<span class="n">seen_aggregate_report_ids</span><span class="o">=</span><span class="n">cfg</span><span class="o">.</span><span class="n">seen_aggregate_report_ids</span><span class="p">,</span>
|
|
<span class="n">pending_aggregate_keys</span><span class="o">=</span><span class="n">pending_aggregate_keys</span><span class="p">,</span>
|
|
<span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">report_type</span> <span class="o">==</span> <span class="s2">"aggregate"</span><span class="p">:</span>
|
|
<span class="n">aggregate_report_msg_uids</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">message_id</span><span class="p">)</span>
|
|
<span class="k">elif</span> <span class="n">report_type</span> <span class="o">==</span> <span class="s2">"failure"</span><span class="p">:</span>
|
|
<span class="n">failure_report_msg_uids</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">message_id</span><span class="p">)</span>
|
|
<span class="k">elif</span> <span class="n">report_type</span> <span class="o">==</span> <span class="s2">"smtp_tls"</span><span class="p">:</span>
|
|
<span class="n">smtp_tls_msg_uids</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">message_id</span><span class="p">)</span>
|
|
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">test</span><span class="p">:</span>
|
|
<span class="k">for</span> <span class="n">invalid_message_id</span> <span class="ow">in</span> <span class="n">invalid_msg_ids</span><span class="p">:</span>
|
|
<span class="n">_dispose_invalid_message</span><span class="p">(</span>
|
|
<span class="n">connection</span><span class="p">,</span>
|
|
<span class="n">invalid_message_id</span><span class="p">,</span>
|
|
<span class="n">delete_invalid</span><span class="p">,</span>
|
|
<span class="n">invalid_reports_folder</span><span class="p">,</span>
|
|
<span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">message_limit</span><span class="p">):</span>
|
|
<span class="n">msg_uid</span> <span class="o">=</span> <span class="n">messages</span><span class="p">[</span><span class="n">i</span><span class="p">]</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
|
<span class="sa">f</span><span class="s2">"Processing message </span><span class="si">{</span><span class="n">i</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="mi">1</span><span class="si">}</span><span class="s2"> of </span><span class="si">{</span><span class="n">message_limit</span><span class="si">}</span><span class="s2">: UID </span><span class="si">{</span><span class="n">msg_uid</span><span class="si">}</span><span class="s2">"</span>
|
|
<span class="p">)</span>
|
|
<span class="n">message_id</span><span class="p">,</span> <span class="n">msg_content</span> <span class="o">=</span> <span class="n">_fetch_mailbox_message</span><span class="p">(</span><span class="n">connection</span><span class="p">,</span> <span class="n">msg_uid</span><span class="p">,</span> <span class="n">test</span><span class="p">)</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">parsed_email</span> <span class="o">=</span> <span class="n">parse_report_email</span><span class="p">(</span>
|
|
<span class="n">msg_content</span><span class="p">,</span>
|
|
<span class="n">config</span><span class="o">=</span><span class="n">cfg</span><span class="p">,</span>
|
|
<span class="n">keep_alive</span><span class="o">=</span><span class="n">connection</span><span class="o">.</span><span class="n">keepalive</span><span class="p">,</span>
|
|
<span class="p">)</span>
|
|
<span class="n">report_type</span> <span class="o">=</span> <span class="n">_classify_parsed_email</span><span class="p">(</span>
|
|
<span class="n">parsed_email</span><span class="p">,</span>
|
|
<span class="n">batch_aggregate_reports</span><span class="p">,</span>
|
|
<span class="n">batch_failure_reports</span><span class="p">,</span>
|
|
<span class="n">batch_smtp_tls_reports</span><span class="p">,</span>
|
|
<span class="n">seen_aggregate_report_ids</span><span class="o">=</span><span class="n">cfg</span><span class="o">.</span><span class="n">seen_aggregate_report_ids</span><span class="p">,</span>
|
|
<span class="n">pending_aggregate_keys</span><span class="o">=</span><span class="n">pending_aggregate_keys</span><span class="p">,</span>
|
|
<span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">report_type</span> <span class="o">==</span> <span class="s2">"aggregate"</span><span class="p">:</span>
|
|
<span class="n">aggregate_report_msg_uids</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">message_id</span><span class="p">)</span>
|
|
<span class="k">elif</span> <span class="n">report_type</span> <span class="o">==</span> <span class="s2">"failure"</span><span class="p">:</span>
|
|
<span class="n">failure_report_msg_uids</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">message_id</span><span class="p">)</span>
|
|
<span class="k">elif</span> <span class="n">report_type</span> <span class="o">==</span> <span class="s2">"smtp_tls"</span><span class="p">:</span>
|
|
<span class="n">smtp_tls_msg_uids</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">message_id</span><span class="p">)</span>
|
|
<span class="k">except</span> <span class="n">ParserError</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">test</span><span class="p">:</span>
|
|
<span class="n">_dispose_invalid_message</span><span class="p">(</span>
|
|
<span class="n">connection</span><span class="p">,</span> <span class="n">message_id</span><span class="p">,</span> <span class="n">delete_invalid</span><span class="p">,</span> <span class="n">invalid_reports_folder</span>
|
|
<span class="p">)</span>
|
|
|
|
<span class="c1"># Ask the caller whether this batch actually made it to its output</span>
|
|
<span class="c1"># destinations before touching a single message. A callback that says</span>
|
|
<span class="c1"># otherwise (or raises) means the reports exist nowhere else yet, so no</span>
|
|
<span class="c1"># message may be archived or deleted -- only retained for retry, or moved</span>
|
|
<span class="c1"># intact to the Unsaved folder once retries run out (#242).</span>
|
|
<span class="n">batch_results</span><span class="p">:</span> <span class="n">ParsingResults</span> <span class="o">=</span> <span class="p">{</span>
|
|
<span class="s2">"aggregate_reports"</span><span class="p">:</span> <span class="n">batch_aggregate_reports</span><span class="p">,</span>
|
|
<span class="s2">"failure_reports"</span><span class="p">:</span> <span class="n">batch_failure_reports</span><span class="p">,</span>
|
|
<span class="s2">"smtp_tls_reports"</span><span class="p">:</span> <span class="n">batch_smtp_tls_reports</span><span class="p">,</span>
|
|
<span class="p">}</span>
|
|
<span class="n">persisted</span> <span class="o">=</span> <span class="kc">True</span>
|
|
<span class="n">callback_error</span><span class="p">:</span> <span class="ne">Exception</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="k">if</span> <span class="n">save_callback</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">persisted</span> <span class="o">=</span> <span class="n">save_callback</span><span class="p">(</span><span class="n">batch_results</span><span class="p">)</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">False</span>
|
|
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
|
<span class="c1"># A raising callback is a failed save too: the failure</span>
|
|
<span class="c1"># bookkeeping below still runs (count the attempt, hold or shelve</span>
|
|
<span class="c1"># the messages) before the exception is re-raised, so a callback</span>
|
|
<span class="c1"># that always raises -- e.g. the CLI's under</span>
|
|
<span class="c1"># ``fail_on_output_error`` -- is still bounded by</span>
|
|
<span class="c1"># ``max_unsaved_retries`` even when the caller's watch loop</span>
|
|
<span class="c1"># swallows the exception and keeps checking, as mailsuite's IMAP</span>
|
|
<span class="c1"># and Maildir backends do.</span>
|
|
<span class="n">persisted</span> <span class="o">=</span> <span class="kc">False</span>
|
|
<span class="n">callback_error</span> <span class="o">=</span> <span class="n">error</span>
|
|
|
|
<span class="n">batch_msg_uids</span> <span class="o">=</span> <span class="p">(</span>
|
|
<span class="n">aggregate_report_msg_uids</span> <span class="o">+</span> <span class="n">failure_report_msg_uids</span> <span class="o">+</span> <span class="n">smtp_tls_msg_uids</span>
|
|
<span class="p">)</span>
|
|
|
|
<span class="k">if</span> <span class="n">persisted</span><span class="p">:</span>
|
|
<span class="c1"># Committing: the reports are safely stored elsewhere, so record</span>
|
|
<span class="c1"># their dedup keys and forget any earlier failures for these</span>
|
|
<span class="c1"># messages.</span>
|
|
<span class="k">for</span> <span class="n">report_key</span> <span class="ow">in</span> <span class="n">pending_aggregate_keys</span><span class="p">:</span>
|
|
<span class="n">cfg</span><span class="o">.</span><span class="n">seen_aggregate_report_ids</span><span class="p">[</span><span class="n">report_key</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">test</span><span class="p">:</span>
|
|
<span class="k">for</span> <span class="n">msg_uid</span> <span class="ow">in</span> <span class="n">batch_msg_uids</span><span class="p">:</span>
|
|
<span class="n">_FAILED_SAVE_ATTEMPTS</span><span class="o">.</span><span class="n">pop</span><span class="p">((</span><span class="n">reports_folder</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">msg_uid</span><span class="p">)),</span> <span class="kc">None</span><span class="p">)</span>
|
|
<span class="k">elif</span> <span class="ow">not</span> <span class="n">test</span><span class="p">:</span>
|
|
<span class="c1"># Held back for retry. Messages that have now failed the initial</span>
|
|
<span class="c1"># attempt plus ``max_unsaved_retries`` retries stop being retried and</span>
|
|
<span class="c1"># move to the Unsaved folder, bounding duplicate delivery to output</span>
|
|
<span class="c1"># destinations that do not deduplicate; the rest stay put.</span>
|
|
<span class="n">retained_uids</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">int</span> <span class="o">|</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="n">over_cap_uids</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">int</span> <span class="o">|</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="n">highest_attempt</span> <span class="o">=</span> <span class="mi">0</span>
|
|
<span class="k">for</span> <span class="n">msg_uid</span> <span class="ow">in</span> <span class="n">batch_msg_uids</span><span class="p">:</span>
|
|
<span class="n">attempts_key</span> <span class="o">=</span> <span class="p">(</span><span class="n">reports_folder</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">msg_uid</span><span class="p">))</span>
|
|
<span class="n">attempts</span> <span class="o">=</span> <span class="n">_FAILED_SAVE_ATTEMPTS</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">attempts_key</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span> <span class="o">+</span> <span class="mi">1</span>
|
|
<span class="n">_FAILED_SAVE_ATTEMPTS</span><span class="p">[</span><span class="n">attempts_key</span><span class="p">]</span> <span class="o">=</span> <span class="n">attempts</span>
|
|
<span class="k">if</span> <span class="n">attempts</span> <span class="o">></span> <span class="n">max_unsaved_retries</span><span class="p">:</span>
|
|
<span class="n">over_cap_uids</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">msg_uid</span><span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">retained_uids</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">msg_uid</span><span class="p">)</span>
|
|
<span class="n">highest_attempt</span> <span class="o">=</span> <span class="nb">max</span><span class="p">(</span><span class="n">highest_attempt</span><span class="p">,</span> <span class="n">attempts</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">retained_uids</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span>
|
|
<span class="sa">f</span><span class="s2">"Reports were not saved: leaving </span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="n">retained_uids</span><span class="p">)</span><span class="si">}</span><span class="s2"> "</span>
|
|
<span class="sa">f</span><span class="s2">"message(s) in </span><span class="si">{</span><span class="n">reports_folder</span><span class="si">}</span><span class="s2"> to retry on the next run "</span>
|
|
<span class="sa">f</span><span class="s2">"or check (failed attempt </span><span class="si">{</span><span class="n">highest_attempt</span><span class="si">}</span><span class="s2"> of "</span>
|
|
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">max_unsaved_retries</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="mi">1</span><span class="si">}</span><span class="s2">)"</span>
|
|
<span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">over_cap_uids</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span>
|
|
<span class="sa">f</span><span class="s2">"Reports were not saved after </span><span class="si">{</span><span class="n">max_unsaved_retries</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="mi">1</span><span class="si">}</span><span class="s2"> "</span>
|
|
<span class="sa">f</span><span class="s2">"attempt(s): moving </span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="n">over_cap_uids</span><span class="p">)</span><span class="si">}</span><span class="s2"> message(s) from "</span>
|
|
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">reports_folder</span><span class="si">}</span><span class="s2"> to </span><span class="si">{</span><span class="n">unsaved_reports_folder</span><span class="si">}</span><span class="s2"> instead of "</span>
|
|
<span class="s2">"retrying them further. They are never deleted -- fix the "</span>
|
|
<span class="sa">f</span><span class="s2">"output destination, then move them back to </span><span class="si">{</span><span class="n">reports_folder</span><span class="si">}</span><span class="s2">"</span>
|
|
<span class="p">)</span>
|
|
<span class="n">_ensure_folder</span><span class="p">(</span><span class="n">connection</span><span class="p">,</span> <span class="n">unsaved_reports_folder</span><span class="p">)</span>
|
|
<span class="k">for</span> <span class="n">msg_uid</span> <span class="ow">in</span> <span class="n">over_cap_uids</span><span class="p">:</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">connection</span><span class="o">.</span><span class="n">move_message</span><span class="p">(</span><span class="n">msg_uid</span><span class="p">,</span> <span class="n">unsaved_reports_folder</span><span class="p">)</span>
|
|
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
|
<span class="n">e</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"Error moving message UID </span><span class="si">{</span><span class="n">msg_uid</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">"</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Mailbox error: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="c1"># Drop the counter only once the message is actually out</span>
|
|
<span class="c1"># of the retry loop. Clearing it before a failed move</span>
|
|
<span class="c1"># would hand the still-in-place message a fresh set of</span>
|
|
<span class="c1"># under-cap retries (and deliveries); keeping it means</span>
|
|
<span class="c1"># the next failed save classifies the message over-cap</span>
|
|
<span class="c1"># again and re-attempts the move instead.</span>
|
|
<span class="n">_FAILED_SAVE_ATTEMPTS</span><span class="o">.</span><span class="n">pop</span><span class="p">((</span><span class="n">reports_folder</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">msg_uid</span><span class="p">)),</span> <span class="kc">None</span><span class="p">)</span>
|
|
|
|
<span class="k">if</span> <span class="n">callback_error</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">callback_error</span>
|
|
|
|
<span class="n">aggregate_reports</span> <span class="o">+=</span> <span class="n">batch_aggregate_reports</span>
|
|
<span class="n">failure_reports</span> <span class="o">+=</span> <span class="n">batch_failure_reports</span>
|
|
<span class="n">smtp_tls_reports</span> <span class="o">+=</span> <span class="n">batch_smtp_tls_reports</span>
|
|
|
|
<span class="k">if</span> <span class="n">persisted</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">test</span><span class="p">:</span>
|
|
<span class="c1"># Each report type is disposed of according to its own effective</span>
|
|
<span class="c1"># delete flag: deleted outright, or moved to its archive subfolder.</span>
|
|
<span class="k">for</span> <span class="n">msg_uids</span><span class="p">,</span> <span class="n">delete_type</span><span class="p">,</span> <span class="n">destination_folder</span><span class="p">,</span> <span class="n">label</span> <span class="ow">in</span> <span class="p">(</span>
|
|
<span class="p">(</span>
|
|
<span class="n">aggregate_report_msg_uids</span><span class="p">,</span>
|
|
<span class="n">delete_aggregate</span><span class="p">,</span>
|
|
<span class="n">aggregate_reports_folder</span><span class="p">,</span>
|
|
<span class="s2">"aggregate report"</span><span class="p">,</span>
|
|
<span class="p">),</span>
|
|
<span class="p">(</span>
|
|
<span class="n">failure_report_msg_uids</span><span class="p">,</span>
|
|
<span class="n">delete_failure</span><span class="p">,</span>
|
|
<span class="n">failure_reports_folder</span><span class="p">,</span>
|
|
<span class="s2">"failure report"</span><span class="p">,</span>
|
|
<span class="p">),</span>
|
|
<span class="p">(</span>
|
|
<span class="n">smtp_tls_msg_uids</span><span class="p">,</span>
|
|
<span class="n">delete_smtp_tls</span><span class="p">,</span>
|
|
<span class="n">smtp_tls_reports_folder</span><span class="p">,</span>
|
|
<span class="s2">"SMTP TLS report"</span><span class="p">,</span>
|
|
<span class="p">),</span>
|
|
<span class="p">):</span>
|
|
<span class="n">number_of_msgs</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">msg_uids</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">number_of_msgs</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
|
|
<span class="k">continue</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">delete_type</span><span class="p">:</span>
|
|
<span class="n">message</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"Moving </span><span class="si">{</span><span class="n">label</span><span class="si">}</span><span class="s2"> messages from"</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">message</span><span class="si">}</span><span class="s2"> </span><span class="si">{</span><span class="n">reports_folder</span><span class="si">}</span><span class="s2"> to </span><span class="si">{</span><span class="n">destination_folder</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
|
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">number_of_msgs</span><span class="p">):</span>
|
|
<span class="n">msg_uid</span> <span class="o">=</span> <span class="n">msg_uids</span><span class="p">[</span><span class="n">i</span><span class="p">]</span>
|
|
<span class="k">if</span> <span class="n">delete_type</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
|
<span class="sa">f</span><span class="s2">"Deleting message </span><span class="si">{</span><span class="n">i</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="mi">1</span><span class="si">}</span><span class="s2"> of </span><span class="si">{</span><span class="n">number_of_msgs</span><span class="si">}</span><span class="s2">: UID </span><span class="si">{</span><span class="n">msg_uid</span><span class="si">}</span><span class="s2">"</span>
|
|
<span class="p">)</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">connection</span><span class="o">.</span><span class="n">delete_message</span><span class="p">(</span><span class="n">msg_uid</span><span class="p">)</span>
|
|
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
|
<span class="n">message</span> <span class="o">=</span> <span class="s2">"Error deleting message UID"</span>
|
|
<span class="n">e</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">message</span><span class="si">}</span><span class="s2"> </span><span class="si">{</span><span class="n">msg_uid</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">"</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Mailbox error: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">message</span> <span class="o">=</span> <span class="s2">"Moving message"</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
|
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">message</span><span class="si">}</span><span class="s2"> </span><span class="si">{</span><span class="n">i</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="mi">1</span><span class="si">}</span><span class="s2"> of </span><span class="si">{</span><span class="n">number_of_msgs</span><span class="si">}</span><span class="s2">: UID </span><span class="si">{</span><span class="n">msg_uid</span><span class="si">}</span><span class="s2">"</span>
|
|
<span class="p">)</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">connection</span><span class="o">.</span><span class="n">move_message</span><span class="p">(</span><span class="n">msg_uid</span><span class="p">,</span> <span class="n">destination_folder</span><span class="p">)</span>
|
|
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
|
<span class="n">e</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"Error moving message UID </span><span class="si">{</span><span class="n">msg_uid</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">"</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Mailbox error: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
|
<span class="n">results</span> <span class="o">=</span> <span class="p">{</span>
|
|
<span class="s2">"aggregate_reports"</span><span class="p">:</span> <span class="n">aggregate_reports</span><span class="p">,</span>
|
|
<span class="s2">"failure_reports"</span><span class="p">:</span> <span class="n">failure_reports</span><span class="p">,</span>
|
|
<span class="s2">"smtp_tls_reports"</span><span class="p">:</span> <span class="n">smtp_tls_reports</span><span class="p">,</span>
|
|
<span class="p">}</span>
|
|
|
|
<span class="c1"># An unsaved batch left its messages in ``reports_folder``, so the</span>
|
|
<span class="c1"># re-check below would find them again and immediately reprocess the</span>
|
|
<span class="c1"># very messages that just failed -- burning through the retry cap in one</span>
|
|
<span class="c1"># call. Skip it and let the next run retry them.</span>
|
|
<span class="k">if</span> <span class="n">persisted</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">test</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">batch_size</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="n">current_time</span><span class="p">:</span>
|
|
<span class="n">total_messages</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span>
|
|
<span class="n">connection</span><span class="o">.</span><span class="n">fetch_messages</span><span class="p">(</span><span class="n">reports_folder</span><span class="p">,</span> <span class="n">since</span><span class="o">=</span><span class="n">current_time</span><span class="p">)</span>
|
|
<span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">total_messages</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">connection</span><span class="o">.</span><span class="n">fetch_messages</span><span class="p">(</span><span class="n">reports_folder</span><span class="p">))</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">total_messages</span> <span class="o">=</span> <span class="mi">0</span>
|
|
|
|
<span class="k">if</span> <span class="n">total_messages</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
|
<span class="c1"># Process emails that came in during the last run</span>
|
|
<span class="n">results</span> <span class="o">=</span> <span class="n">get_dmarc_reports_from_mailbox</span><span class="p">(</span>
|
|
<span class="n">connection</span><span class="o">=</span><span class="n">connection</span><span class="p">,</span>
|
|
<span class="n">reports_folder</span><span class="o">=</span><span class="n">reports_folder</span><span class="p">,</span>
|
|
<span class="n">archive_folder</span><span class="o">=</span><span class="n">archive_folder</span><span class="p">,</span>
|
|
<span class="n">delete</span><span class="o">=</span><span class="n">delete</span><span class="p">,</span>
|
|
<span class="n">delete_aggregate</span><span class="o">=</span><span class="n">delete_aggregate</span><span class="p">,</span>
|
|
<span class="n">delete_failure</span><span class="o">=</span><span class="n">delete_failure</span><span class="p">,</span>
|
|
<span class="n">delete_smtp_tls</span><span class="o">=</span><span class="n">delete_smtp_tls</span><span class="p">,</span>
|
|
<span class="n">delete_invalid</span><span class="o">=</span><span class="n">delete_invalid</span><span class="p">,</span>
|
|
<span class="n">test</span><span class="o">=</span><span class="n">test</span><span class="p">,</span>
|
|
<span class="n">results</span><span class="o">=</span><span class="n">results</span><span class="p">,</span>
|
|
<span class="n">since</span><span class="o">=</span><span class="n">current_time</span><span class="p">,</span>
|
|
<span class="n">n_procs</span><span class="o">=</span><span class="n">n_procs</span><span class="p">,</span>
|
|
<span class="n">save_callback</span><span class="o">=</span><span class="n">save_callback</span><span class="p">,</span>
|
|
<span class="n">max_unsaved_retries</span><span class="o">=</span><span class="n">max_unsaved_retries</span><span class="p">,</span>
|
|
<span class="n">config</span><span class="o">=</span><span class="n">cfg</span><span class="p">,</span>
|
|
<span class="p">)</span>
|
|
|
|
<span class="k">return</span> <span class="n">results</span></div>
|
|
|
|
|
|
|
|
<div class="viewcode-block" id="watch_inbox">
|
|
<a class="viewcode-back" href="../api.html#parsedmarc.watch_inbox">[docs]</a>
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">watch_inbox</span><span class="p">(</span>
|
|
<span class="n">mailbox_connection</span><span class="p">:</span> <span class="n">MailboxConnection</span><span class="p">,</span>
|
|
<span class="n">callback</span><span class="p">:</span> <span class="n">Callable</span><span class="p">,</span>
|
|
<span class="o">*</span><span class="p">,</span>
|
|
<span class="n">reports_folder</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"INBOX"</span><span class="p">,</span>
|
|
<span class="n">archive_folder</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"Archive"</span><span class="p">,</span>
|
|
<span class="n">delete</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
|
<span class="n">delete_aggregate</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">delete_failure</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">delete_smtp_tls</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">delete_invalid</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">test</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
|
<span class="n">check_timeout</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">30</span><span class="p">,</span>
|
|
<span class="n">ip_db_path</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">always_use_local_files</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
|
<span class="n">reverse_dns_map_path</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">reverse_dns_map_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">offline</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
|
<span class="n">nameservers</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">dns_timeout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="n">DEFAULT_DNS_TIMEOUT</span><span class="p">,</span>
|
|
<span class="n">dns_retries</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="n">DEFAULT_DNS_MAX_RETRIES</span><span class="p">,</span>
|
|
<span class="n">strip_attachment_payloads</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
|
<span class="n">batch_size</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">10</span><span class="p">,</span>
|
|
<span class="n">since</span><span class="p">:</span> <span class="n">datetime</span> <span class="o">|</span> <span class="n">date</span> <span class="o">|</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">normalize_timespan_threshold_hours</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">24.0</span><span class="p">,</span>
|
|
<span class="n">config_reloading</span><span class="p">:</span> <span class="n">Callable</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">n_procs</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
|
|
<span class="n">max_unsaved_retries</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">2</span><span class="p">,</span>
|
|
<span class="n">config</span><span class="p">:</span> <span class="n">ParserConfig</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="p">):</span>
|
|
<span class="w"> </span><span class="sd">"""</span>
|
|
<span class="sd"> Watches the mailbox for new messages and</span>
|
|
<span class="sd"> sends the results to a callback function</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> mailbox_connection: The mailbox connection object</span>
|
|
<span class="sd"> callback: The callback function to receive the parsing results.</span>
|
|
<span class="sd"> Passed straight through to ``get_dmarc_reports_from_mailbox()``</span>
|
|
<span class="sd"> as its ``save_callback``, so it now runs once per fetched batch,</span>
|
|
<span class="sd"> with only that batch's reports, *before* those messages are</span>
|
|
<span class="sd"> deleted or moved out of ``reports_folder`` -- rather than once</span>
|
|
<span class="sd"> afterward with the whole check's accumulated results. Returning</span>
|
|
<span class="sd"> ``False`` reports the batch as unsaved, leaving its messages in</span>
|
|
<span class="sd"> place to be retried on the next check instead of archived or</span>
|
|
<span class="sd"> deleted (see ``save_callback`` and ``max_unsaved_retries`` on</span>
|
|
<span class="sd"> ``get_dmarc_reports_from_mailbox()``). Raising counts as an</span>
|
|
<span class="sd"> unsaved batch too -- same retention and retry cap -- before the</span>
|
|
<span class="sd"> exception reaches the mailbox backend's watch loop; what happens</span>
|
|
<span class="sd"> then is backend-specific: the Microsoft Graph and Gmail backends</span>
|
|
<span class="sd"> let it propagate and end the watch, while mailsuite's IMAP and</span>
|
|
<span class="sd"> Maildir watch loops log it and keep checking.</span>
|
|
<span class="sd"> reports_folder (str): The IMAP folder where reports can be found</span>
|
|
<span class="sd"> archive_folder (str): The folder to move processed mail to</span>
|
|
<span class="sd"> delete (bool): Delete messages after processing them</span>
|
|
<span class="sd"> delete_aggregate (bool | None): Delete aggregate report messages</span>
|
|
<span class="sd"> after processing them, instead of moving them to the</span>
|
|
<span class="sd"> ``Aggregate`` archive subfolder; ``None`` (the default) inherits</span>
|
|
<span class="sd"> the value of ``delete``</span>
|
|
<span class="sd"> delete_failure (bool | None): Delete failure report messages after</span>
|
|
<span class="sd"> processing them, instead of moving them to the ``Failure``</span>
|
|
<span class="sd"> archive subfolder; ``None`` (the default) inherits the value of</span>
|
|
<span class="sd"> ``delete``</span>
|
|
<span class="sd"> delete_smtp_tls (bool | None): Delete SMTP TLS report messages after</span>
|
|
<span class="sd"> processing them, instead of moving them to the ``SMTP-TLS``</span>
|
|
<span class="sd"> archive subfolder; ``None`` (the default) inherits the value of</span>
|
|
<span class="sd"> ``delete``</span>
|
|
<span class="sd"> delete_invalid (bool | None): Delete unparseable messages, instead</span>
|
|
<span class="sd"> of moving them to the ``Invalid`` archive subfolder where they</span>
|
|
<span class="sd"> can be inspected for debugging; ``None`` (the default) inherits</span>
|
|
<span class="sd"> the value of ``delete``</span>
|
|
<span class="sd"> test (bool): Do not move or delete messages after processing them</span>
|
|
<span class="sd"> check_timeout (int): Number of seconds to wait for a IMAP IDLE response</span>
|
|
<span class="sd"> or the number of seconds until the next mail check</span>
|
|
<span class="sd"> ip_db_path (str): Path to a MMDB file from IPinfo, MaxMind, or DBIP</span>
|
|
<span class="sd"> always_use_local_files (bool): Do not download files</span>
|
|
<span class="sd"> reverse_dns_map_path (str): Path to a reverse DNS map file</span>
|
|
<span class="sd"> reverse_dns_map_url (str): URL to a reverse DNS map file</span>
|
|
<span class="sd"> offline (bool): Do not query online for geolocation or DNS</span>
|
|
<span class="sd"> nameservers (list): A list of one or more nameservers to use</span>
|
|
<span class="sd"> (Cloudflare's public DNS resolvers by default)</span>
|
|
<span class="sd"> dns_timeout (float): Set the DNS query timeout</span>
|
|
<span class="sd"> dns_retries (int): Number of times to retry DNS queries on timeout</span>
|
|
<span class="sd"> or other transient errors</span>
|
|
<span class="sd"> strip_attachment_payloads (bool): Replace attachment payloads in</span>
|
|
<span class="sd"> failure report samples with None</span>
|
|
<span class="sd"> batch_size (int): Number of messages to read and process before saving</span>
|
|
<span class="sd"> since: Search for messages since certain time</span>
|
|
<span class="sd"> normalize_timespan_threshold_hours (float): Normalize timespans beyond this</span>
|
|
<span class="sd"> config_reloading: Optional callable that returns True when a config</span>
|
|
<span class="sd"> reload (or shutdown) has been requested (e.g. via SIGHUP/SIGTERM).</span>
|
|
<span class="sd"> Polled by the mailbox backend between checks, including the IMAP</span>
|
|
<span class="sd"> IDLE loop, so the watcher exits cleanly at a safe boundary.</span>
|
|
<span class="sd"> n_procs (int): Number of processes to use for parsing messages in</span>
|
|
<span class="sd"> parallel. Passed through to ``get_dmarc_reports_from_mailbox``</span>
|
|
<span class="sd"> on each check. Not part of ``config``; always applies.</span>
|
|
<span class="sd"> max_unsaved_retries (int): How many times a message may be retried</span>
|
|
<span class="sd"> after ``callback`` first reported its batch unsaved, before it is</span>
|
|
<span class="sd"> moved to the ``Unsaved`` archive subfolder (default 2). Passed</span>
|
|
<span class="sd"> through to ``get_dmarc_reports_from_mailbox``, where it is</span>
|
|
<span class="sd"> documented in full. Not part of ``config``; always applies.</span>
|
|
<span class="sd"> config (ParserConfig): a single object carrying all parsing and</span>
|
|
<span class="sd"> enrichment options plus the caches; when provided, it replaces</span>
|
|
<span class="sd"> the individual parsing and enrichment option keyword arguments</span>
|
|
<span class="sd"> listed above (DNS, GeoIP, offline mode, attachment payload</span>
|
|
<span class="sd"> stripping, timespan normalization), whose values are then</span>
|
|
<span class="sd"> ignored. The remaining keyword arguments control mailbox</span>
|
|
<span class="sd"> handling and orchestration rather than parsing (the folder</span>
|
|
<span class="sd"> names, the ``delete`` options, ``test``, ``since``,</span>
|
|
<span class="sd"> ``batch_size``, ``max_unsaved_retries``); they are not part of</span>
|
|
<span class="sd"> ``config`` and always apply.</span>
|
|
<span class="sd"> """</span>
|
|
<span class="c1"># Validate before the watch loop starts: raised inside a check, this</span>
|
|
<span class="c1"># would be swallowed and endlessly retried by the IMAP and Maildir</span>
|
|
<span class="c1"># backends' per-check exception handling instead of surfacing.</span>
|
|
<span class="k">if</span> <span class="n">max_unsaved_retries</span> <span class="o"><</span> <span class="mi">0</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"max_unsaved_retries must be >= 0"</span><span class="p">)</span>
|
|
|
|
<span class="n">cfg</span> <span class="o">=</span> <span class="n">_resolve_config</span><span class="p">(</span>
|
|
<span class="n">config</span><span class="p">,</span>
|
|
<span class="n">offline</span><span class="o">=</span><span class="n">offline</span><span class="p">,</span>
|
|
<span class="n">ip_db_path</span><span class="o">=</span><span class="n">ip_db_path</span><span class="p">,</span>
|
|
<span class="n">always_use_local_files</span><span class="o">=</span><span class="n">always_use_local_files</span><span class="p">,</span>
|
|
<span class="n">reverse_dns_map_path</span><span class="o">=</span><span class="n">reverse_dns_map_path</span><span class="p">,</span>
|
|
<span class="n">reverse_dns_map_url</span><span class="o">=</span><span class="n">reverse_dns_map_url</span><span class="p">,</span>
|
|
<span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
|
|
<span class="n">dns_timeout</span><span class="o">=</span><span class="n">dns_timeout</span><span class="p">,</span>
|
|
<span class="n">dns_retries</span><span class="o">=</span><span class="n">dns_retries</span><span class="p">,</span>
|
|
<span class="n">strip_attachment_payloads</span><span class="o">=</span><span class="n">strip_attachment_payloads</span><span class="p">,</span>
|
|
<span class="n">normalize_timespan_threshold_hours</span><span class="o">=</span><span class="n">normalize_timespan_threshold_hours</span><span class="p">,</span>
|
|
<span class="p">)</span>
|
|
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">check_callback</span><span class="p">(</span><span class="n">connection</span><span class="p">):</span>
|
|
<span class="n">get_dmarc_reports_from_mailbox</span><span class="p">(</span>
|
|
<span class="n">connection</span><span class="o">=</span><span class="n">connection</span><span class="p">,</span>
|
|
<span class="n">reports_folder</span><span class="o">=</span><span class="n">reports_folder</span><span class="p">,</span>
|
|
<span class="n">archive_folder</span><span class="o">=</span><span class="n">archive_folder</span><span class="p">,</span>
|
|
<span class="n">delete</span><span class="o">=</span><span class="n">delete</span><span class="p">,</span>
|
|
<span class="n">delete_aggregate</span><span class="o">=</span><span class="n">delete_aggregate</span><span class="p">,</span>
|
|
<span class="n">delete_failure</span><span class="o">=</span><span class="n">delete_failure</span><span class="p">,</span>
|
|
<span class="n">delete_smtp_tls</span><span class="o">=</span><span class="n">delete_smtp_tls</span><span class="p">,</span>
|
|
<span class="n">delete_invalid</span><span class="o">=</span><span class="n">delete_invalid</span><span class="p">,</span>
|
|
<span class="n">test</span><span class="o">=</span><span class="n">test</span><span class="p">,</span>
|
|
<span class="n">batch_size</span><span class="o">=</span><span class="n">batch_size</span><span class="p">,</span>
|
|
<span class="n">since</span><span class="o">=</span><span class="n">since</span><span class="p">,</span>
|
|
<span class="n">create_folders</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
|
<span class="n">n_procs</span><span class="o">=</span><span class="n">n_procs</span><span class="p">,</span>
|
|
<span class="n">save_callback</span><span class="o">=</span><span class="n">callback</span><span class="p">,</span>
|
|
<span class="n">max_unsaved_retries</span><span class="o">=</span><span class="n">max_unsaved_retries</span><span class="p">,</span>
|
|
<span class="n">config</span><span class="o">=</span><span class="n">cfg</span><span class="p">,</span>
|
|
<span class="p">)</span>
|
|
|
|
<span class="n">watch_kwargs</span><span class="p">:</span> <span class="nb">dict</span> <span class="o">=</span> <span class="p">{</span>
|
|
<span class="s2">"check_callback"</span><span class="p">:</span> <span class="n">check_callback</span><span class="p">,</span>
|
|
<span class="s2">"check_timeout"</span><span class="p">:</span> <span class="n">check_timeout</span><span class="p">,</span>
|
|
<span class="p">}</span>
|
|
<span class="k">if</span> <span class="n">config_reloading</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">watch_kwargs</span><span class="p">[</span><span class="s2">"config_reloading"</span><span class="p">]</span> <span class="o">=</span> <span class="n">config_reloading</span>
|
|
|
|
<span class="n">mailbox_connection</span><span class="o">.</span><span class="n">watch</span><span class="p">(</span><span class="o">**</span><span class="n">watch_kwargs</span><span class="p">)</span></div>
|
|
|
|
|
|
|
|
<div class="viewcode-block" id="append_json">
|
|
<a class="viewcode-back" href="../api.html#parsedmarc.append_json">[docs]</a>
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">append_json</span><span class="p">(</span>
|
|
<span class="n">filename</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
|
<span class="n">reports</span><span class="p">:</span> <span class="n">Sequence</span><span class="p">[</span><span class="n">AggregateReport</span><span class="p">]</span>
|
|
<span class="o">|</span> <span class="n">Sequence</span><span class="p">[</span><span class="n">FailureReport</span><span class="p">]</span>
|
|
<span class="o">|</span> <span class="n">Sequence</span><span class="p">[</span><span class="n">SMTPTLSReport</span><span class="p">],</span>
|
|
<span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="w"> </span><span class="sd">"""Append ``reports`` to a JSON array on disk, creating the file</span>
|
|
<span class="sd"> if needed.</span>
|
|
|
|
<span class="sd"> Reads the existing array (if the file exists and parses cleanly),</span>
|
|
<span class="sd"> merges the new reports onto the end, and rewrites the file as a</span>
|
|
<span class="sd"> single valid JSON array. An earlier version of this used an</span>
|
|
<span class="sd"> ``open(..., "a+")`` + ``seek()`` + overwrite pattern, but Python's</span>
|
|
<span class="sd"> documentation is explicit that on POSIX, ``a`` / ``a+`` writes</span>
|
|
<span class="sd"> *always* go to EOF regardless of seek position — so the second</span>
|
|
<span class="sd"> call onto an existing file produced ``[...],\\n[...]``-style</span>
|
|
<span class="sd"> corrupted output. Read-merge-write is the only way to get a valid</span>
|
|
<span class="sd"> JSON array out of repeated appends.</span>
|
|
<span class="sd"> """</span>
|
|
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">reports</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
|
|
<span class="c1"># Don't create an empty-array file for an empty input; if a</span>
|
|
<span class="c1"># file already exists, leave it alone.</span>
|
|
<span class="k">return</span>
|
|
|
|
<span class="n">existing</span><span class="p">:</span> <span class="nb">list</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isfile</span><span class="p">(</span><span class="n">filename</span><span class="p">)</span> <span class="ow">and</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">getsize</span><span class="p">(</span><span class="n">filename</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="s2">"r"</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s2">"utf-8"</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
|
|
<span class="n">loaded</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">read</span><span class="p">())</span>
|
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">loaded</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
|
|
<span class="n">existing</span> <span class="o">=</span> <span class="n">loaded</span>
|
|
<span class="k">except</span> <span class="p">(</span><span class="n">json</span><span class="o">.</span><span class="n">JSONDecodeError</span><span class="p">,</span> <span class="ne">OSError</span><span class="p">):</span>
|
|
<span class="c1"># Corrupted or unreadable: overwrite cleanly rather than</span>
|
|
<span class="c1"># silently fail to record.</span>
|
|
<span class="n">existing</span> <span class="o">=</span> <span class="p">[]</span>
|
|
|
|
<span class="n">merged</span> <span class="o">=</span> <span class="n">existing</span> <span class="o">+</span> <span class="nb">list</span><span class="p">(</span><span class="n">reports</span><span class="p">)</span>
|
|
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="s2">"w"</span><span class="p">,</span> <span class="n">newline</span><span class="o">=</span><span class="s2">"</span><span class="se">\n</span><span class="s2">"</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s2">"utf-8"</span><span class="p">)</span> <span class="k">as</span> <span class="n">output</span><span class="p">:</span>
|
|
<span class="n">json</span><span class="o">.</span><span class="n">dump</span><span class="p">(</span><span class="n">merged</span><span class="p">,</span> <span class="n">output</span><span class="p">,</span> <span class="n">ensure_ascii</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">indent</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span></div>
|
|
|
|
|
|
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">append_csv</span><span class="p">(</span><span class="n">filename</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">csv</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="s2">"a+"</span><span class="p">,</span> <span class="n">newline</span><span class="o">=</span><span class="s2">"</span><span class="se">\n</span><span class="s2">"</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s2">"utf-8"</span><span class="p">)</span> <span class="k">as</span> <span class="n">output</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="n">output</span><span class="o">.</span><span class="n">seek</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">os</span><span class="o">.</span><span class="n">SEEK_END</span><span class="p">)</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>
|
|
<span class="c1"># strip the headers from the CSV</span>
|
|
<span class="n">_headers</span><span class="p">,</span> <span class="n">csv</span> <span class="o">=</span> <span class="n">csv</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">"</span><span class="se">\n</span><span class="s2">"</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">csv</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
|
|
<span class="c1"># not appending anything, don't do any dance to</span>
|
|
<span class="c1"># append it correctly</span>
|
|
<span class="k">return</span>
|
|
<span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">csv</span><span class="p">)</span>
|
|
|
|
|
|
<div class="viewcode-block" id="save_output">
|
|
<a class="viewcode-back" href="../api.html#parsedmarc.save_output">[docs]</a>
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">save_output</span><span class="p">(</span>
|
|
<span class="n">results</span><span class="p">:</span> <span class="n">ParsingResults</span><span class="p">,</span>
|
|
<span class="o">*</span><span class="p">,</span>
|
|
<span class="n">output_directory</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"output"</span><span class="p">,</span>
|
|
<span class="n">aggregate_json_filename</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"aggregate.json"</span><span class="p">,</span>
|
|
<span class="n">failure_json_filename</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"failure.json"</span><span class="p">,</span>
|
|
<span class="n">smtp_tls_json_filename</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"smtp_tls.json"</span><span class="p">,</span>
|
|
<span class="n">aggregate_csv_filename</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"aggregate.csv"</span><span class="p">,</span>
|
|
<span class="n">failure_csv_filename</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"failure.csv"</span><span class="p">,</span>
|
|
<span class="n">smtp_tls_csv_filename</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"smtp_tls.csv"</span><span class="p">,</span>
|
|
<span class="p">):</span>
|
|
<span class="w"> </span><span class="sd">"""</span>
|
|
<span class="sd"> Save report data in the given directory</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> results: Parsing results</span>
|
|
<span class="sd"> output_directory (str): The path to the directory to save in</span>
|
|
<span class="sd"> aggregate_json_filename (str): Filename for the aggregate JSON file</span>
|
|
<span class="sd"> failure_json_filename (str): Filename for the failure JSON file</span>
|
|
<span class="sd"> smtp_tls_json_filename (str): Filename for the SMTP TLS JSON file</span>
|
|
<span class="sd"> aggregate_csv_filename (str): Filename for the aggregate CSV file</span>
|
|
<span class="sd"> failure_csv_filename (str): Filename for the failure CSV file</span>
|
|
<span class="sd"> smtp_tls_csv_filename (str): Filename for the SMTP TLS CSV file</span>
|
|
<span class="sd"> """</span>
|
|
|
|
<span class="n">aggregate_reports</span> <span class="o">=</span> <span class="n">results</span><span class="p">[</span><span class="s2">"aggregate_reports"</span><span class="p">]</span>
|
|
<span class="n">failure_reports</span> <span class="o">=</span> <span class="n">results</span><span class="p">[</span><span class="s2">"failure_reports"</span><span class="p">]</span>
|
|
<span class="n">smtp_tls_reports</span> <span class="o">=</span> <span class="n">results</span><span class="p">[</span><span class="s2">"smtp_tls_reports"</span><span class="p">]</span>
|
|
<span class="n">output_directory</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">expanduser</span><span class="p">(</span><span class="n">output_directory</span><span class="p">)</span>
|
|
|
|
<span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">output_directory</span><span class="p">):</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isdir</span><span class="p">(</span><span class="n">output_directory</span><span class="p">):</span>
|
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">output_directory</span><span class="si">}</span><span class="s2"> is not a directory"</span><span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">os</span><span class="o">.</span><span class="n">makedirs</span><span class="p">(</span><span class="n">output_directory</span><span class="p">)</span>
|
|
|
|
<span class="n">append_json</span><span class="p">(</span>
|
|
<span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">output_directory</span><span class="p">,</span> <span class="n">aggregate_json_filename</span><span class="p">),</span> <span class="n">aggregate_reports</span>
|
|
<span class="p">)</span>
|
|
|
|
<span class="n">append_csv</span><span class="p">(</span>
|
|
<span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">output_directory</span><span class="p">,</span> <span class="n">aggregate_csv_filename</span><span class="p">),</span>
|
|
<span class="n">parsed_aggregate_reports_to_csv</span><span class="p">(</span><span class="n">aggregate_reports</span><span class="p">),</span>
|
|
<span class="p">)</span>
|
|
|
|
<span class="n">append_json</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">output_directory</span><span class="p">,</span> <span class="n">failure_json_filename</span><span class="p">),</span> <span class="n">failure_reports</span><span class="p">)</span>
|
|
|
|
<span class="n">append_csv</span><span class="p">(</span>
|
|
<span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">output_directory</span><span class="p">,</span> <span class="n">failure_csv_filename</span><span class="p">),</span>
|
|
<span class="n">parsed_failure_reports_to_csv</span><span class="p">(</span><span class="n">failure_reports</span><span class="p">),</span>
|
|
<span class="p">)</span>
|
|
|
|
<span class="n">append_json</span><span class="p">(</span>
|
|
<span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">output_directory</span><span class="p">,</span> <span class="n">smtp_tls_json_filename</span><span class="p">),</span> <span class="n">smtp_tls_reports</span>
|
|
<span class="p">)</span>
|
|
|
|
<span class="n">append_csv</span><span class="p">(</span>
|
|
<span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">output_directory</span><span class="p">,</span> <span class="n">smtp_tls_csv_filename</span><span class="p">),</span>
|
|
<span class="n">parsed_smtp_tls_reports_to_csv</span><span class="p">(</span><span class="n">smtp_tls_reports</span><span class="p">),</span>
|
|
<span class="p">)</span>
|
|
|
|
<span class="n">samples_directory</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">output_directory</span><span class="p">,</span> <span class="s2">"samples"</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">samples_directory</span><span class="p">):</span>
|
|
<span class="n">os</span><span class="o">.</span><span class="n">makedirs</span><span class="p">(</span><span class="n">samples_directory</span><span class="p">)</span>
|
|
|
|
<span class="n">sample_filenames</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="k">for</span> <span class="n">failure_report</span> <span class="ow">in</span> <span class="n">failure_reports</span><span class="p">:</span>
|
|
<span class="n">sample</span> <span class="o">=</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"sample"</span><span class="p">]</span>
|
|
<span class="n">message_count</span> <span class="o">=</span> <span class="mi">0</span>
|
|
<span class="n">parsed_sample</span> <span class="o">=</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">]</span>
|
|
<span class="n">subject</span> <span class="o">=</span> <span class="p">(</span>
|
|
<span class="n">parsed_sample</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"filename_safe_subject"</span><span class="p">)</span>
|
|
<span class="ow">or</span> <span class="n">parsed_sample</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"subject"</span><span class="p">)</span>
|
|
<span class="ow">or</span> <span class="s2">"sample"</span>
|
|
<span class="p">)</span>
|
|
<span class="n">filename</span> <span class="o">=</span> <span class="n">subject</span>
|
|
|
|
<span class="k">while</span> <span class="n">filename</span> <span class="ow">in</span> <span class="n">sample_filenames</span><span class="p">:</span>
|
|
<span class="n">message_count</span> <span class="o">+=</span> <span class="mi">1</span>
|
|
<span class="n">filename</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">subject</span><span class="si">}</span><span class="s2"> (</span><span class="si">{</span><span class="n">message_count</span><span class="si">}</span><span class="s2">)"</span>
|
|
|
|
<span class="n">sample_filenames</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">filename</span><span class="p">)</span>
|
|
|
|
<span class="n">filename</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">filename</span><span class="si">}</span><span class="s2">.eml"</span>
|
|
<span class="n">path</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">samples_directory</span><span class="p">,</span> <span class="n">filename</span><span class="p">)</span>
|
|
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="s2">"w"</span><span class="p">,</span> <span class="n">newline</span><span class="o">=</span><span class="s2">"</span><span class="se">\n</span><span class="s2">"</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s2">"utf-8"</span><span class="p">)</span> <span class="k">as</span> <span class="n">sample_file</span><span class="p">:</span>
|
|
<span class="n">sample_file</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">sample</span><span class="p">)</span></div>
|
|
|
|
|
|
|
|
<div class="viewcode-block" id="get_report_zip">
|
|
<a class="viewcode-back" href="../api.html#parsedmarc.get_report_zip">[docs]</a>
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_report_zip</span><span class="p">(</span><span class="n">results</span><span class="p">:</span> <span class="n">ParsingResults</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bytes</span><span class="p">:</span>
|
|
<span class="w"> </span><span class="sd">"""</span>
|
|
<span class="sd"> Creates a zip file of parsed report output</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> results: The parsed results</span>
|
|
|
|
<span class="sd"> Returns:</span>
|
|
<span class="sd"> bytes: zip file bytes</span>
|
|
<span class="sd"> """</span>
|
|
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">add_subdir</span><span class="p">(</span><span class="n">root_path</span><span class="p">,</span> <span class="n">subdir</span><span class="p">):</span>
|
|
<span class="n">subdir_path</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">root_path</span><span class="p">,</span> <span class="n">subdir</span><span class="p">)</span>
|
|
<span class="k">for</span> <span class="n">subdir_root</span><span class="p">,</span> <span class="n">subdir_dirs</span><span class="p">,</span> <span class="n">subdir_files</span> <span class="ow">in</span> <span class="n">os</span><span class="o">.</span><span class="n">walk</span><span class="p">(</span><span class="n">subdir_path</span><span class="p">):</span>
|
|
<span class="k">for</span> <span class="n">subdir_file</span> <span class="ow">in</span> <span class="n">subdir_files</span><span class="p">:</span>
|
|
<span class="n">subdir_file_path</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">root_path</span><span class="p">,</span> <span class="n">subdir</span><span class="p">,</span> <span class="n">subdir_file</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isfile</span><span class="p">(</span><span class="n">subdir_file_path</span><span class="p">):</span>
|
|
<span class="n">rel_path</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">relpath</span><span class="p">(</span><span class="n">subdir_root</span><span class="p">,</span> <span class="n">subdir_file_path</span><span class="p">)</span>
|
|
<span class="n">subdir_arc_name</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">rel_path</span><span class="p">,</span> <span class="n">subdir_file</span><span class="p">)</span>
|
|
<span class="n">zip_file</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">subdir_file_path</span><span class="p">,</span> <span class="n">subdir_arc_name</span><span class="p">)</span>
|
|
<span class="k">for</span> <span class="n">subdir</span> <span class="ow">in</span> <span class="n">subdir_dirs</span><span class="p">:</span>
|
|
<span class="n">add_subdir</span><span class="p">(</span><span class="n">subdir_path</span><span class="p">,</span> <span class="n">subdir</span><span class="p">)</span>
|
|
|
|
<span class="n">storage</span> <span class="o">=</span> <span class="n">BytesIO</span><span class="p">()</span>
|
|
<span class="n">tmp_dir</span> <span class="o">=</span> <span class="n">tempfile</span><span class="o">.</span><span class="n">mkdtemp</span><span class="p">()</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">save_output</span><span class="p">(</span><span class="n">results</span><span class="p">,</span> <span class="n">output_directory</span><span class="o">=</span><span class="n">tmp_dir</span><span class="p">)</span>
|
|
<span class="k">with</span> <span class="n">zipfile</span><span class="o">.</span><span class="n">ZipFile</span><span class="p">(</span><span class="n">storage</span><span class="p">,</span> <span class="s2">"w"</span><span class="p">,</span> <span class="n">zipfile</span><span class="o">.</span><span class="n">ZIP_DEFLATED</span><span class="p">)</span> <span class="k">as</span> <span class="n">zip_file</span><span class="p">:</span>
|
|
<span class="k">for</span> <span class="n">root</span><span class="p">,</span> <span class="n">dirs</span><span class="p">,</span> <span class="n">files</span> <span class="ow">in</span> <span class="n">os</span><span class="o">.</span><span class="n">walk</span><span class="p">(</span><span class="n">tmp_dir</span><span class="p">):</span>
|
|
<span class="k">for</span> <span class="n">file</span> <span class="ow">in</span> <span class="n">files</span><span class="p">:</span>
|
|
<span class="n">file_path</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">root</span><span class="p">,</span> <span class="n">file</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isfile</span><span class="p">(</span><span class="n">file_path</span><span class="p">):</span>
|
|
<span class="n">arcname</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">relpath</span><span class="p">(</span><span class="n">root</span><span class="p">,</span> <span class="n">tmp_dir</span><span class="p">),</span> <span class="n">file</span><span class="p">)</span>
|
|
<span class="n">zip_file</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">file_path</span><span class="p">,</span> <span class="n">arcname</span><span class="p">)</span>
|
|
<span class="k">for</span> <span class="n">directory</span> <span class="ow">in</span> <span class="n">dirs</span><span class="p">:</span>
|
|
<span class="n">dir_path</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">root</span><span class="p">,</span> <span class="n">directory</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isdir</span><span class="p">(</span><span class="n">dir_path</span><span class="p">):</span>
|
|
<span class="n">zip_file</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">dir_path</span><span class="p">,</span> <span class="n">directory</span><span class="p">)</span>
|
|
<span class="n">add_subdir</span><span class="p">(</span><span class="n">root</span><span class="p">,</span> <span class="n">directory</span><span class="p">)</span>
|
|
<span class="k">finally</span><span class="p">:</span>
|
|
<span class="n">shutil</span><span class="o">.</span><span class="n">rmtree</span><span class="p">(</span><span class="n">tmp_dir</span><span class="p">)</span>
|
|
|
|
<span class="k">return</span> <span class="n">storage</span><span class="o">.</span><span class="n">getvalue</span><span class="p">()</span></div>
|
|
|
|
|
|
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">_build_report_email_content</span><span class="p">(</span>
|
|
<span class="n">results</span><span class="p">:</span> <span class="n">ParsingResults</span><span class="p">,</span>
|
|
<span class="o">*</span><span class="p">,</span>
|
|
<span class="n">subject</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">attachment_filename</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">message</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="p">)</span> <span class="o">-></span> <span class="nb">tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">,</span> <span class="nb">list</span><span class="p">[</span><span class="nb">tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">bytes</span><span class="p">]]]:</span>
|
|
<span class="w"> </span><span class="sd">"""Builds the subject, plain-text body, and zip attachment shared by</span>
|
|
<span class="sd"> every report-summary email transport.</span>
|
|
|
|
<span class="sd"> Returns:</span>
|
|
<span class="sd"> A ``(subject, plain_message, attachments)`` tuple.</span>
|
|
<span class="sd"> """</span>
|
|
<span class="n">date_string</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2">"</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">attachment_filename</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">attachment_filename</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s2">".zip"</span><span class="p">):</span>
|
|
<span class="n">attachment_filename</span> <span class="o">+=</span> <span class="s2">".zip"</span>
|
|
<span class="n">filename</span> <span class="o">=</span> <span class="n">attachment_filename</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">filename</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"DMARC-</span><span class="si">{</span><span class="n">date_string</span><span class="si">}</span><span class="s2">.zip"</span>
|
|
|
|
<span class="k">if</span> <span class="n">subject</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">subject</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"DMARC results for </span><span class="si">{</span><span class="n">date_string</span><span class="si">}</span><span class="s2">"</span>
|
|
<span class="k">if</span> <span class="n">message</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">message</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"DMARC results for </span><span class="si">{</span><span class="n">date_string</span><span class="si">}</span><span class="s2">"</span>
|
|
<span class="n">zip_bytes</span> <span class="o">=</span> <span class="n">get_report_zip</span><span class="p">(</span><span class="n">results</span><span class="p">)</span>
|
|
<span class="n">attachments</span> <span class="o">=</span> <span class="p">[(</span><span class="n">filename</span><span class="p">,</span> <span class="n">zip_bytes</span><span class="p">)]</span>
|
|
|
|
<span class="k">return</span> <span class="n">subject</span><span class="p">,</span> <span class="n">message</span><span class="p">,</span> <span class="n">attachments</span>
|
|
|
|
|
|
<div class="viewcode-block" id="email_results">
|
|
<a class="viewcode-back" href="../api.html#parsedmarc.email_results">[docs]</a>
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">email_results</span><span class="p">(</span>
|
|
<span class="n">results</span><span class="p">:</span> <span class="n">ParsingResults</span><span class="p">,</span>
|
|
<span class="n">host</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
|
<span class="n">mail_from</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
|
<span class="n">mail_to</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="o">*</span><span class="p">,</span>
|
|
<span class="n">mail_cc</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">mail_bcc</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">port</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">0</span><span class="p">,</span>
|
|
<span class="n">require_encryption</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
|
<span class="n">verify</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
|
|
<span class="n">username</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">password</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">subject</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">attachment_filename</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">message</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="p">):</span>
|
|
<span class="w"> </span><span class="sd">"""</span>
|
|
<span class="sd"> Emails parsing results as a zip file</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> results (dict): Parsing results</span>
|
|
<span class="sd"> host (str): Mail server hostname or IP address</span>
|
|
<span class="sd"> mail_from: The value of the message from header</span>
|
|
<span class="sd"> mail_to (list): A list of addresses to mail to</span>
|
|
<span class="sd"> mail_cc (list): A list of addresses to CC</span>
|
|
<span class="sd"> mail_bcc (list): A list addresses to BCC</span>
|
|
<span class="sd"> port (int): Port to use</span>
|
|
<span class="sd"> require_encryption (bool): Require a secure connection from the start</span>
|
|
<span class="sd"> verify (bool): verify the SSL/TLS certificate</span>
|
|
<span class="sd"> username (str): An optional username</span>
|
|
<span class="sd"> password (str): An optional password</span>
|
|
<span class="sd"> subject (str): Overrides the default message subject</span>
|
|
<span class="sd"> attachment_filename (str): Override the default attachment filename</span>
|
|
<span class="sd"> message (str): Override the default plain text body</span>
|
|
<span class="sd"> """</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Emailing report"</span><span class="p">)</span>
|
|
<span class="k">assert</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">mail_to</span><span class="p">,</span> <span class="nb">list</span><span class="p">)</span>
|
|
|
|
<span class="n">subject</span><span class="p">,</span> <span class="n">message</span><span class="p">,</span> <span class="n">attachments</span> <span class="o">=</span> <span class="n">_build_report_email_content</span><span class="p">(</span>
|
|
<span class="n">results</span><span class="p">,</span>
|
|
<span class="n">subject</span><span class="o">=</span><span class="n">subject</span><span class="p">,</span>
|
|
<span class="n">attachment_filename</span><span class="o">=</span><span class="n">attachment_filename</span><span class="p">,</span>
|
|
<span class="n">message</span><span class="o">=</span><span class="n">message</span><span class="p">,</span>
|
|
<span class="p">)</span>
|
|
|
|
<span class="n">send_email</span><span class="p">(</span>
|
|
<span class="n">host</span><span class="p">,</span>
|
|
<span class="n">mail_from</span><span class="p">,</span>
|
|
<span class="n">mail_to</span><span class="p">,</span>
|
|
<span class="n">message_cc</span><span class="o">=</span><span class="n">mail_cc</span><span class="p">,</span>
|
|
<span class="n">message_bcc</span><span class="o">=</span><span class="n">mail_bcc</span><span class="p">,</span>
|
|
<span class="n">port</span><span class="o">=</span><span class="n">port</span><span class="p">,</span>
|
|
<span class="n">require_encryption</span><span class="o">=</span><span class="n">require_encryption</span><span class="p">,</span>
|
|
<span class="n">verify</span><span class="o">=</span><span class="n">verify</span><span class="p">,</span>
|
|
<span class="n">username</span><span class="o">=</span><span class="n">username</span><span class="p">,</span>
|
|
<span class="n">password</span><span class="o">=</span><span class="n">password</span><span class="p">,</span>
|
|
<span class="n">subject</span><span class="o">=</span><span class="n">subject</span><span class="p">,</span>
|
|
<span class="n">attachments</span><span class="o">=</span><span class="n">attachments</span><span class="p">,</span>
|
|
<span class="n">plain_message</span><span class="o">=</span><span class="n">message</span><span class="p">,</span>
|
|
<span class="p">)</span></div>
|
|
|
|
|
|
|
|
<div class="viewcode-block" id="email_results_via_msgraph">
|
|
<a class="viewcode-back" href="../api.html#parsedmarc.email_results_via_msgraph">[docs]</a>
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">email_results_via_msgraph</span><span class="p">(</span>
|
|
<span class="n">results</span><span class="p">:</span> <span class="n">ParsingResults</span><span class="p">,</span>
|
|
<span class="n">connection</span><span class="p">:</span> <span class="n">MSGraphConnection</span><span class="p">,</span>
|
|
<span class="n">mail_to</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span>
|
|
<span class="o">*</span><span class="p">,</span>
|
|
<span class="n">mail_cc</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">mail_bcc</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">subject</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">attachment_filename</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">message</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="w"> </span><span class="sd">"""</span>
|
|
<span class="sd"> Emails parsing results as a zip file via an already-authenticated</span>
|
|
<span class="sd"> Microsoft Graph mailbox connection (``/users/{mailbox}/sendMail``),</span>
|
|
<span class="sd"> saving a copy to Sent Items.</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> results (dict): Parsing results</span>
|
|
<span class="sd"> connection (MSGraphConnection): An already-authenticated Microsoft</span>
|
|
<span class="sd"> Graph mailbox connection</span>
|
|
<span class="sd"> mail_to (list): A list of addresses to mail to</span>
|
|
<span class="sd"> mail_cc (list): A list of addresses to CC</span>
|
|
<span class="sd"> mail_bcc (list): A list addresses to BCC</span>
|
|
<span class="sd"> subject (str): Overrides the default message subject</span>
|
|
<span class="sd"> attachment_filename (str): Override the default attachment filename</span>
|
|
<span class="sd"> message (str): Override the default plain text body</span>
|
|
<span class="sd"> """</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Emailing report via Microsoft Graph"</span><span class="p">)</span>
|
|
|
|
<span class="n">subject</span><span class="p">,</span> <span class="n">message</span><span class="p">,</span> <span class="n">attachments</span> <span class="o">=</span> <span class="n">_build_report_email_content</span><span class="p">(</span>
|
|
<span class="n">results</span><span class="p">,</span>
|
|
<span class="n">subject</span><span class="o">=</span><span class="n">subject</span><span class="p">,</span>
|
|
<span class="n">attachment_filename</span><span class="o">=</span><span class="n">attachment_filename</span><span class="p">,</span>
|
|
<span class="n">message</span><span class="o">=</span><span class="n">message</span><span class="p">,</span>
|
|
<span class="p">)</span>
|
|
|
|
<span class="c1"># Graph derives the From header from the authenticated mailbox and</span>
|
|
<span class="c1"># ignores message_from; it's still passed for API parity with</span>
|
|
<span class="c1"># send_message()'s signature.</span>
|
|
<span class="n">connection</span><span class="o">.</span><span class="n">send_message</span><span class="p">(</span>
|
|
<span class="n">message_from</span><span class="o">=</span><span class="n">connection</span><span class="o">.</span><span class="n">mailbox_name</span> <span class="ow">or</span> <span class="s2">""</span><span class="p">,</span>
|
|
<span class="n">message_to</span><span class="o">=</span><span class="n">mail_to</span><span class="p">,</span>
|
|
<span class="n">message_cc</span><span class="o">=</span><span class="n">mail_cc</span><span class="p">,</span>
|
|
<span class="n">message_bcc</span><span class="o">=</span><span class="n">mail_bcc</span><span class="p">,</span>
|
|
<span class="n">subject</span><span class="o">=</span><span class="n">subject</span><span class="p">,</span>
|
|
<span class="n">attachments</span><span class="o">=</span><span class="n">attachments</span><span class="p">,</span>
|
|
<span class="n">plain_message</span><span class="o">=</span><span class="n">message</span><span class="p">,</span>
|
|
<span class="p">)</span></div>
|
|
|
|
|
|
|
|
<span class="c1"># Backward-compatible aliases</span>
|
|
<span class="n">parse_forensic_report</span> <span class="o">=</span> <span class="n">parse_failure_report</span>
|
|
<span class="n">parsed_forensic_reports_to_csv_rows</span> <span class="o">=</span> <span class="n">parsed_failure_reports_to_csv_rows</span>
|
|
<span class="n">parsed_forensic_reports_to_csv</span> <span class="o">=</span> <span class="n">parsed_failure_reports_to_csv</span>
|
|
</pre></div>
|
|
|
|
</div>
|
|
</div>
|
|
<footer>
|
|
|
|
<hr/>
|
|
|
|
<div role="contentinfo">
|
|
<p>© Copyright 2018 - 2025, Sean Whalen and contributors.</p>
|
|
</div>
|
|
|
|
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
|
|
<a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
|
|
provided by <a href="https://readthedocs.org">Read the Docs</a>.
|
|
|
|
|
|
</footer>
|
|
</div>
|
|
</div>
|
|
</section>
|
|
</div>
|
|
<script>
|
|
jQuery(function () {
|
|
SphinxRtdTheme.Navigation.enable(true);
|
|
});
|
|
</script>
|
|
|
|
</body>
|
|
</html> |