mirror of
https://github.com/domainaware/parsedmarc.git
synced 2026-07-08 01:35:08 +00:00
Update docs
This commit is contained in:
Sean Whalen
+2
-2
@@ -5,14 +5,14 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Overview: module code — parsedmarc 9.11.2 documentation</title>
|
||||
<title>Overview: module code — parsedmarc 10.0.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="../_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<script src="../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../_static/documentation_options.js?v=de4344a5"></script>
|
||||
<script src="../_static/documentation_options.js?v=335988e4"></script>
|
||||
<script src="../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="../_static/js/theme.js"></script>
|
||||
|
||||
+367
-129
@@ -5,14 +5,14 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc — parsedmarc 9.11.2 documentation</title>
|
||||
<title>parsedmarc — parsedmarc 10.0.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="../_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<script src="../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../_static/documentation_options.js?v=de4344a5"></script>
|
||||
<script src="../_static/documentation_options.js?v=335988e4"></script>
|
||||
<script src="../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="../_static/js/theme.js"></script>
|
||||
@@ -134,7 +134,8 @@
|
||||
<span class="p">)</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.types</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
|
||||
<span class="n">AggregateReport</span><span class="p">,</span>
|
||||
<span class="n">ForensicReport</span><span class="p">,</span>
|
||||
<span class="n">FailureReport</span><span class="p">,</span>
|
||||
<span class="n">ForensicReport</span> <span class="k">as</span> <span class="n">ForensicReport</span><span class="p">,</span>
|
||||
<span class="n">ParsedReport</span><span class="p">,</span>
|
||||
<span class="n">ParsingResults</span><span class="p">,</span>
|
||||
<span class="n">SMTPTLSReport</span><span class="p">,</span>
|
||||
@@ -155,10 +156,33 @@
|
||||
<span class="n">xml_header_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">"^<\?xml .*?>"</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">MULTILINE</span><span class="p">)</span>
|
||||
<span class="n">xml_schema_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">"</??xs:schema.*>"</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">MULTILINE</span><span class="p">)</span>
|
||||
<span class="n">text_report_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">"\s*([a-zA-Z\s]+):\s(.+)"</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">MULTILINE</span><span class="p">)</span>
|
||||
<span class="c1"># Captures the value of any xmlns (default or prefixed) declaration so the</span>
|
||||
<span class="c1"># RFC 9990 namespace can be detected before xmltodict drops it.</span>
|
||||
<span class="n">xml_namespace_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span>
|
||||
<span class="w"> </span><span class="sa">r</span><span class="sd">"""xmlns(?::[a-zA-Z_][\w.-]*)?\s*=\s*["']([^"']+)["']"""</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="c1"># The XML namespace assigned to DMARC aggregate reports by RFC 9990.</span>
|
||||
<span class="n">RFC_9990_NAMESPACE</span> <span class="o">=</span> <span class="s2">"urn:ietf:params:xml:ns:dmarc-2.0"</span>
|
||||
|
||||
<span class="c1"># PolicyOverrideType enumeration from RFC 9990. Compared to RFC 7489,</span>
|
||||
<span class="c1"># `policy_test_mode` was added (emitted when t=y suppresses enforcement)</span>
|
||||
<span class="c1"># and `forwarded` / `sampled_out` were removed.</span>
|
||||
<span class="n">RFC_9990_POLICY_OVERRIDE_TYPES</span> <span class="o">=</span> <span class="nb">frozenset</span><span class="p">(</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s2">"local_policy"</span><span class="p">,</span>
|
||||
<span class="s2">"mailing_list"</span><span class="p">,</span>
|
||||
<span class="s2">"other"</span><span class="p">,</span>
|
||||
<span class="s2">"policy_test_mode"</span><span class="p">,</span>
|
||||
<span class="s2">"trusted_forwarder"</span><span class="p">,</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">RFC_7489_REMOVED_POLICY_OVERRIDE_TYPES</span> <span class="o">=</span> <span class="nb">frozenset</span><span class="p">({</span><span class="s2">"forwarded"</span><span class="p">,</span> <span class="s2">"sampled_out"</span><span class="p">})</span>
|
||||
|
||||
<span class="n">MAGIC_ZIP</span> <span class="o">=</span> <span class="sa">b</span><span class="s2">"</span><span class="se">\x50\x4b\x03\x04</span><span class="s2">"</span>
|
||||
<span class="n">MAGIC_GZIP</span> <span class="o">=</span> <span class="sa">b</span><span class="s2">"</span><span class="se">\x1f\x8b</span><span class="s2">"</span>
|
||||
<span class="n">MAGIC_XML</span> <span class="o">=</span> <span class="sa">b</span><span class="s2">"</span><span class="se">\x3c\x3f\x78\x6d\x6c\x20</span><span class="s2">"</span>
|
||||
<span class="n">MAGIC_XML_TAG</span> <span class="o">=</span> <span class="sa">b</span><span class="s2">"</span><span class="se">\x3c</span><span class="s2">"</span> <span class="c1"># '<' - XML starting with an element tag (no declaration)</span>
|
||||
<span class="n">MAGIC_JSON</span> <span class="o">=</span> <span class="sa">b</span><span class="s2">"</span><span class="se">\7</span><span class="s2">b"</span>
|
||||
|
||||
<span class="n">EMAIL_SAMPLE_CONTENT_TYPES</span> <span class="o">=</span> <span class="p">(</span>
|
||||
@@ -205,13 +229,37 @@
|
||||
|
||||
|
||||
|
||||
<div class="viewcode-block" id="InvalidForensicReport">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.InvalidForensicReport">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">InvalidForensicReport</span><span class="p">(</span><span class="n">InvalidDMARCReport</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Raised when an invalid DMARC forensic report is encountered"""</span></div>
|
||||
<div class="viewcode-block" id="InvalidFailureReport">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.InvalidFailureReport">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">InvalidFailureReport</span><span class="p">(</span><span class="n">InvalidDMARCReport</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Raised when an invalid DMARC failure report is encountered"""</span></div>
|
||||
|
||||
|
||||
|
||||
<span class="c1"># Backward-compatible alias</span>
|
||||
<span class="n">InvalidForensicReport</span> <span class="o">=</span> <span class="n">InvalidFailureReport</span>
|
||||
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">_text</span><span class="p">(</span><span class="n">value</span><span class="p">:</span> <span class="n">Any</span><span class="p">)</span> <span class="o">-></span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
|
||||
<span class="w"> </span><span class="sd">"""Unwrap a possibly-langAttrString value parsed by xmltodict.</span>
|
||||
|
||||
<span class="sd"> RFC 9990 changed several aggregate-report elements (extra_contact_info,</span>
|
||||
<span class="sd"> error, comment, human_result) to type ``langAttrString`` — an</span>
|
||||
<span class="sd"> xs:simpleContent string with an optional ``lang`` attribute. When the</span>
|
||||
<span class="sd"> attribute is present, xmltodict parses the element as</span>
|
||||
<span class="sd"> ``{"#text": "...", "@lang": "en"}`` instead of a plain string. Returns</span>
|
||||
<span class="sd"> the text payload for both shapes, ``None`` for unset values, and leaves</span>
|
||||
<span class="sd"> other scalar shapes untouched so callers can preserve whatever the</span>
|
||||
<span class="sd"> reporter sent.</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">if</span> <span class="n">value</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="kc">None</span>
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
|
||||
<span class="n">text</span> <span class="o">=</span> <span class="n">value</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"#text"</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="kc">None</span> <span class="k">if</span> <span class="n">text</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="nb">str</span><span class="p">(</span><span class="n">text</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="n">value</span>
|
||||
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">_bucket_interval_by_day</span><span class="p">(</span>
|
||||
<span class="n">begin</span><span class="p">:</span> <span class="n">datetime</span><span class="p">,</span>
|
||||
<span class="n">end</span><span class="p">:</span> <span class="n">datetime</span><span class="p">,</span>
|
||||
@@ -404,6 +452,7 @@
|
||||
<span class="n">nameservers</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">dns_timeout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="n">DEFAULT_DNS_TIMEOUT</span><span class="p">,</span>
|
||||
<span class="n">dns_retries</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="n">DEFAULT_DNS_MAX_RETRIES</span><span class="p">,</span>
|
||||
<span class="n">is_rfc_9990</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
||||
<span class="p">)</span> <span class="o">-></span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]:</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Converts a record from a DMARC aggregate report into a more consistent</span>
|
||||
@@ -453,8 +502,6 @@
|
||||
<span class="p">}</span>
|
||||
<span class="k">if</span> <span class="s2">"disposition"</span> <span class="ow">in</span> <span class="n">policy_evaluated</span><span class="p">:</span>
|
||||
<span class="n">new_policy_evaluated</span><span class="p">[</span><span class="s2">"disposition"</span><span class="p">]</span> <span class="o">=</span> <span class="n">policy_evaluated</span><span class="p">[</span><span class="s2">"disposition"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="n">new_policy_evaluated</span><span class="p">[</span><span class="s2">"disposition"</span><span class="p">]</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s2">"pass"</span><span class="p">:</span>
|
||||
<span class="n">new_policy_evaluated</span><span class="p">[</span><span class="s2">"disposition"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"none"</span>
|
||||
<span class="k">if</span> <span class="s2">"dkim"</span> <span class="ow">in</span> <span class="n">policy_evaluated</span><span class="p">:</span>
|
||||
<span class="n">new_policy_evaluated</span><span class="p">[</span><span class="s2">"dkim"</span><span class="p">]</span> <span class="o">=</span> <span class="n">policy_evaluated</span><span class="p">[</span><span class="s2">"dkim"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="s2">"spf"</span> <span class="ow">in</span> <span class="n">policy_evaluated</span><span class="p">:</span>
|
||||
@@ -479,8 +526,27 @@
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">reasons</span> <span class="o">=</span> <span class="p">[</span><span class="n">policy_evaluated</span><span class="p">[</span><span class="s2">"reason"</span><span class="p">]]</span>
|
||||
<span class="k">for</span> <span class="n">reason</span> <span class="ow">in</span> <span class="n">reasons</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="s2">"comment"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">reason</span><span class="p">:</span>
|
||||
<span class="n">reason</span><span class="p">[</span><span class="s2">"comment"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="c1"># `comment` is langAttrString in RFC 9990 — unwrap {"#text": ..., "@lang": ...}</span>
|
||||
<span class="n">reason</span><span class="p">[</span><span class="s2">"comment"</span><span class="p">]</span> <span class="o">=</span> <span class="n">_text</span><span class="p">(</span><span class="n">reason</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"comment"</span><span class="p">))</span>
|
||||
<span class="n">reason_type</span> <span class="o">=</span> <span class="n">reason</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"type"</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">is_rfc_9990</span> <span class="ow">and</span> <span class="n">reason_type</span> <span class="ow">in</span> <span class="n">RFC_7489_REMOVED_POLICY_OVERRIDE_TYPES</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
||||
<span class="s2">"Policy override reason type </span><span class="si">%r</span><span class="s2"> was removed in RFC 9990; "</span>
|
||||
<span class="s2">"expected one of </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span>
|
||||
<span class="n">reason_type</span><span class="p">,</span>
|
||||
<span class="nb">sorted</span><span class="p">(</span><span class="n">RFC_9990_POLICY_OVERRIDE_TYPES</span><span class="p">),</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">elif</span> <span class="p">(</span>
|
||||
<span class="n">is_rfc_9990</span>
|
||||
<span class="ow">and</span> <span class="n">reason_type</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
|
||||
<span class="ow">and</span> <span class="n">reason_type</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">RFC_9990_POLICY_OVERRIDE_TYPES</span>
|
||||
<span class="p">):</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
||||
<span class="s2">"Unknown policy override reason type </span><span class="si">%r</span><span class="s2"> per RFC 9990; "</span>
|
||||
<span class="s2">"expected one of </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span>
|
||||
<span class="n">reason_type</span><span class="p">,</span>
|
||||
<span class="nb">sorted</span><span class="p">(</span><span class="n">RFC_9990_POLICY_OVERRIDE_TYPES</span><span class="p">),</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">new_policy_evaluated</span><span class="p">[</span><span class="s2">"policy_override_reasons"</span><span class="p">]</span> <span class="o">=</span> <span class="n">reasons</span>
|
||||
<span class="n">new_record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">]</span> <span class="o">=</span> <span class="n">new_policy_evaluated</span>
|
||||
<span class="k">if</span> <span class="s2">"identities"</span> <span class="ow">in</span> <span class="n">record</span><span class="p">:</span>
|
||||
@@ -510,11 +576,18 @@
|
||||
<span class="k">if</span> <span class="s2">"selector"</span> <span class="ow">in</span> <span class="n">result</span> <span class="ow">and</span> <span class="n">result</span><span class="p">[</span><span class="s2">"selector"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">new_result</span><span class="p">[</span><span class="s2">"selector"</span><span class="p">]</span> <span class="o">=</span> <span class="n">result</span><span class="p">[</span><span class="s2">"selector"</span><span class="p">]</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">is_rfc_9990</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
||||
<span class="s2">"DKIM auth result for </span><span class="si">%r</span><span class="s2"> is missing the 'selector' "</span>
|
||||
<span class="s2">"element, which is REQUIRED by RFC 9990"</span><span class="p">,</span>
|
||||
<span class="n">result</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">],</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">new_result</span><span class="p">[</span><span class="s2">"selector"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"none"</span>
|
||||
<span class="k">if</span> <span class="s2">"result"</span> <span class="ow">in</span> <span class="n">result</span> <span class="ow">and</span> <span class="n">result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">new_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">]</span> <span class="o">=</span> <span class="n">result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">]</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">new_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"none"</span>
|
||||
<span class="n">new_result</span><span class="p">[</span><span class="s2">"human_result"</span><span class="p">]</span> <span class="o">=</span> <span class="n">_text</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"human_result"</span><span class="p">))</span>
|
||||
<span class="n">new_record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">][</span><span class="s2">"dkim"</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">new_result</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">auth_results</span><span class="p">[</span><span class="s2">"spf"</span><span class="p">],</span> <span class="nb">list</span><span class="p">):</span>
|
||||
@@ -530,6 +603,7 @@
|
||||
<span class="n">new_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">]</span> <span class="o">=</span> <span class="n">result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">]</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">new_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"none"</span>
|
||||
<span class="n">new_result</span><span class="p">[</span><span class="s2">"human_result"</span><span class="p">]</span> <span class="o">=</span> <span class="n">_text</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"human_result"</span><span class="p">))</span>
|
||||
<span class="n">new_record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">new_result</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="s2">"envelope_from"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">new_record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">]:</span>
|
||||
@@ -813,6 +887,21 @@
|
||||
<span class="c1"># Parse XML and recover from errors</span>
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">xml</span><span class="p">,</span> <span class="nb">bytes</span><span class="p">):</span>
|
||||
<span class="n">xml</span> <span class="o">=</span> <span class="n">xml</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="n">errors</span><span class="o">=</span><span class="s2">"ignore"</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># Detect the XML namespace before any rewriting strips it. The dmarc-2.0</span>
|
||||
<span class="c1"># namespace is one of the indicators for an RFC 9990 report but it is</span>
|
||||
<span class="c1"># NOT a reliable sole discriminator: the <version> element value is</span>
|
||||
<span class="c1"># ambiguous (RFC 9990's appendix sample uses <version>1.0</version></span>
|
||||
<span class="c1"># inside the dmarc-2.0 namespace), and real-world reporters frequently</span>
|
||||
<span class="c1"># emit RFC 9990-shaped reports without declaring the namespace at all.</span>
|
||||
<span class="c1"># The final `is_rfc_9990` decision is made post-parse so that</span>
|
||||
<span class="c1"># RFC 9990-only fields (np, testing, discovery_method, generator,</span>
|
||||
<span class="c1"># human_result) can also vote it in.</span>
|
||||
<span class="n">xml_namespace</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="n">namespace_match</span> <span class="o">=</span> <span class="n">xml_namespace_regex</span><span class="o">.</span><span class="n">search</span><span class="p">(</span><span class="n">xml</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">namespace_match</span><span class="p">:</span>
|
||||
<span class="n">xml_namespace</span> <span class="o">=</span> <span class="n">namespace_match</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
|
||||
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">xmltodict</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">xml</span><span class="p">)[</span><span class="s2">"feedback"</span><span class="p">]</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
@@ -836,16 +925,24 @@
|
||||
|
||||
<span class="n">report</span> <span class="o">=</span> <span class="n">xmltodict</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">xml</span><span class="p">)[</span><span class="s2">"feedback"</span><span class="p">]</span>
|
||||
<span class="n">report_metadata</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">]</span>
|
||||
<span class="c1"># <email> is xs:string in both RFC 7489 and RFC 9990, but defensive</span>
|
||||
<span class="c1"># parsing in the wild: some reporters emit it with an xml:lang or</span>
|
||||
<span class="c1"># similar attribute, which xmltodict turns into a dict.</span>
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">report_metadata</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"email"</span><span class="p">),</span> <span class="nb">dict</span><span class="p">):</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
||||
<span class="s2">"Discarding malformed <email> in report_metadata: </span><span class="si">%r</span><span class="s2">"</span><span class="p">,</span>
|
||||
<span class="n">report_metadata</span><span class="p">[</span><span class="s2">"email"</span><span class="p">],</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">report_metadata</span><span class="p">[</span><span class="s2">"email"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="n">unwrapped</span> <span class="o">=</span> <span class="n">_text</span><span class="p">(</span><span class="n">report_metadata</span><span class="p">[</span><span class="s2">"email"</span><span class="p">])</span>
|
||||
<span class="k">if</span> <span class="n">unwrapped</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
||||
<span class="s2">"Discarding malformed <email> in report_metadata: </span><span class="si">%r</span><span class="s2">"</span><span class="p">,</span>
|
||||
<span class="n">report_metadata</span><span class="p">[</span><span class="s2">"email"</span><span class="p">],</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">report_metadata</span><span class="p">[</span><span class="s2">"email"</span><span class="p">]</span> <span class="o">=</span> <span class="n">unwrapped</span>
|
||||
<span class="n">schema</span> <span class="o">=</span> <span class="s2">"draft"</span>
|
||||
<span class="k">if</span> <span class="s2">"version"</span> <span class="ow">in</span> <span class="n">report</span><span class="p">:</span>
|
||||
<span class="n">schema</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"version"</span><span class="p">]</span>
|
||||
<span class="n">new_report</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"xml_schema"</span><span class="p">:</span> <span class="n">schema</span><span class="p">}</span>
|
||||
<span class="n">new_report</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s2">"xml_schema"</span><span class="p">:</span> <span class="n">schema</span><span class="p">,</span>
|
||||
<span class="s2">"xml_namespace"</span><span class="p">:</span> <span class="n">xml_namespace</span><span class="p">,</span>
|
||||
<span class="p">}</span>
|
||||
<span class="n">new_report_metadata</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
|
||||
<span class="k">if</span> <span class="n">report_metadata</span><span class="p">[</span><span class="s2">"org_name"</span><span class="p">]</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">report_metadata</span><span class="p">[</span><span class="s2">"email"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
@@ -866,9 +963,9 @@
|
||||
<span class="p">)</span>
|
||||
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"org_name"</span><span class="p">]</span> <span class="o">=</span> <span class="n">org_name</span>
|
||||
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"org_email"</span><span class="p">]</span> <span class="o">=</span> <span class="n">report_metadata</span><span class="p">[</span><span class="s2">"email"</span><span class="p">]</span>
|
||||
<span class="n">extra</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="k">if</span> <span class="s2">"extra_contact_info"</span> <span class="ow">in</span> <span class="n">report_metadata</span><span class="p">:</span>
|
||||
<span class="n">extra</span> <span class="o">=</span> <span class="n">report_metadata</span><span class="p">[</span><span class="s2">"extra_contact_info"</span><span class="p">]</span>
|
||||
<span class="c1"># extra_contact_info is langAttrString in RFC 9990 (xs:string in</span>
|
||||
<span class="c1"># RFC 7489) — unwrap {"#text": ..., "@lang": ...} if present.</span>
|
||||
<span class="n">extra</span> <span class="o">=</span> <span class="n">_text</span><span class="p">(</span><span class="n">report_metadata</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"extra_contact_info"</span><span class="p">))</span>
|
||||
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"org_extra_contact_info"</span><span class="p">]</span> <span class="o">=</span> <span class="n">extra</span>
|
||||
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"report_id"</span><span class="p">]</span> <span class="o">=</span> <span class="n">report_metadata</span><span class="p">[</span><span class="s2">"report_id"</span><span class="p">]</span>
|
||||
<span class="n">report_id</span> <span class="o">=</span> <span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"report_id"</span><span class="p">]</span>
|
||||
@@ -896,17 +993,38 @@
|
||||
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="c1"># <error> is langAttrString in RFC 9990 (xs:string in RFC 7489) and</span>
|
||||
<span class="c1"># was cardinality-narrowed from "unbounded" to "1" in RFC 9990, but</span>
|
||||
<span class="c1"># the parser still accepts a list for backward compatibility with</span>
|
||||
<span class="c1"># RFC 7489 reports that carry multiple errors.</span>
|
||||
<span class="k">if</span> <span class="s2">"error"</span> <span class="ow">in</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">]:</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">][</span><span class="s2">"error"</span><span class="p">],</span> <span class="nb">list</span><span class="p">):</span>
|
||||
<span class="n">errors</span> <span class="o">=</span> <span class="p">[</span><span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">][</span><span class="s2">"error"</span><span class="p">]]</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">errors</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">][</span><span class="s2">"error"</span><span class="p">]</span>
|
||||
<span class="n">raw_errors</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">][</span><span class="s2">"error"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">raw_errors</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
|
||||
<span class="n">raw_errors</span> <span class="o">=</span> <span class="p">[</span><span class="n">raw_errors</span><span class="p">]</span>
|
||||
<span class="n">errors</span> <span class="o">=</span> <span class="p">[</span><span class="n">text</span> <span class="k">for</span> <span class="n">text</span> <span class="ow">in</span> <span class="p">(</span><span class="n">_text</span><span class="p">(</span><span class="n">e</span><span class="p">)</span> <span class="k">for</span> <span class="n">e</span> <span class="ow">in</span> <span class="n">raw_errors</span><span class="p">)</span> <span class="k">if</span> <span class="n">text</span><span class="p">]</span>
|
||||
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"errors"</span><span class="p">]</span> <span class="o">=</span> <span class="n">errors</span>
|
||||
<span class="c1"># <generator> is a plain xs:string in RFC 9990 but apply _text() so</span>
|
||||
<span class="c1"># a malformed reporter that decorates it with attributes still</span>
|
||||
<span class="c1"># yields a string instead of breaking downstream consumers.</span>
|
||||
<span class="n">generator</span> <span class="o">=</span> <span class="n">_text</span><span class="p">(</span><span class="n">report_metadata</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"generator"</span><span class="p">))</span>
|
||||
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"generator"</span><span class="p">]</span> <span class="o">=</span> <span class="n">generator</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">]</span> <span class="o">=</span> <span class="n">new_report_metadata</span>
|
||||
<span class="n">records</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="n">policy_published</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">policy_published</span><span class="p">)</span> <span class="ow">is</span> <span class="nb">list</span><span class="p">:</span>
|
||||
<span class="n">policy_published</span> <span class="o">=</span> <span class="n">policy_published</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
|
||||
|
||||
<span class="c1"># Final RFC 9990 detection: the dmarc-2.0 XML namespace OR any</span>
|
||||
<span class="c1"># RFC 9990-only field. Real-world reporters that follow the schema</span>
|
||||
<span class="c1"># without declaring the namespace still get RFC 9990-aware</span>
|
||||
<span class="c1"># warnings (missing DKIM selector, removed override-reason types,</span>
|
||||
<span class="c1"># etc.) and a truthful audit trail in `xml_namespace`.</span>
|
||||
<span class="n">rfc_9990_only_policy_fields</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"np"</span><span class="p">,</span> <span class="s2">"testing"</span><span class="p">,</span> <span class="s2">"discovery_method"</span><span class="p">}</span>
|
||||
<span class="n">is_rfc_9990</span> <span class="o">=</span> <span class="p">(</span>
|
||||
<span class="n">xml_namespace</span> <span class="o">==</span> <span class="n">RFC_9990_NAMESPACE</span>
|
||||
<span class="ow">or</span> <span class="s2">"generator"</span> <span class="ow">in</span> <span class="n">report_metadata</span>
|
||||
<span class="ow">or</span> <span class="nb">any</span><span class="p">(</span><span class="n">f</span> <span class="ow">in</span> <span class="n">policy_published</span> <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="n">rfc_9990_only_policy_fields</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">new_policy_published</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
|
||||
<span class="n">new_policy_published</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">]</span> <span class="o">=</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">]</span>
|
||||
<span class="n">adkim</span> <span class="o">=</span> <span class="s2">"r"</span>
|
||||
@@ -925,16 +1043,39 @@
|
||||
<span class="k">if</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"sp"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">sp</span> <span class="o">=</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"sp"</span><span class="p">]</span>
|
||||
<span class="n">new_policy_published</span><span class="p">[</span><span class="s2">"sp"</span><span class="p">]</span> <span class="o">=</span> <span class="n">sp</span>
|
||||
<span class="n">pct</span> <span class="o">=</span> <span class="s2">"100"</span>
|
||||
<span class="n">pct</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="k">if</span> <span class="s2">"pct"</span> <span class="ow">in</span> <span class="n">policy_published</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"pct"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">pct</span> <span class="o">=</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"pct"</span><span class="p">]</span>
|
||||
<span class="n">new_policy_published</span><span class="p">[</span><span class="s2">"pct"</span><span class="p">]</span> <span class="o">=</span> <span class="n">pct</span>
|
||||
<span class="n">fo</span> <span class="o">=</span> <span class="s2">"0"</span>
|
||||
<span class="n">fo</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="k">if</span> <span class="s2">"fo"</span> <span class="ow">in</span> <span class="n">policy_published</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"fo"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">fo</span> <span class="o">=</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"fo"</span><span class="p">]</span>
|
||||
<span class="n">new_policy_published</span><span class="p">[</span><span class="s2">"fo"</span><span class="p">]</span> <span class="o">=</span> <span class="n">fo</span>
|
||||
<span class="n">np_</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="k">if</span> <span class="s2">"np"</span> <span class="ow">in</span> <span class="n">policy_published</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"np"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">np_</span> <span class="o">=</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"np"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="n">np_</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">(</span><span class="s2">"none"</span><span class="p">,</span> <span class="s2">"quarantine"</span><span class="p">,</span> <span class="s2">"reject"</span><span class="p">):</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"Invalid np value: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">np_</span><span class="p">))</span>
|
||||
<span class="n">new_policy_published</span><span class="p">[</span><span class="s2">"np"</span><span class="p">]</span> <span class="o">=</span> <span class="n">np_</span>
|
||||
<span class="n">testing</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="k">if</span> <span class="s2">"testing"</span> <span class="ow">in</span> <span class="n">policy_published</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"testing"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">testing</span> <span class="o">=</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"testing"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="n">testing</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">(</span><span class="s2">"n"</span><span class="p">,</span> <span class="s2">"y"</span><span class="p">):</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"Invalid testing value: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">testing</span><span class="p">))</span>
|
||||
<span class="n">new_policy_published</span><span class="p">[</span><span class="s2">"testing"</span><span class="p">]</span> <span class="o">=</span> <span class="n">testing</span>
|
||||
<span class="n">discovery_method</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="k">if</span> <span class="s2">"discovery_method"</span> <span class="ow">in</span> <span class="n">policy_published</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"discovery_method"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">discovery_method</span> <span class="o">=</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"discovery_method"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="n">discovery_method</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">(</span><span class="s2">"psl"</span><span class="p">,</span> <span class="s2">"treewalk"</span><span class="p">):</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
||||
<span class="s2">"Invalid discovery_method value: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">discovery_method</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">new_policy_published</span><span class="p">[</span><span class="s2">"discovery_method"</span><span class="p">]</span> <span class="o">=</span> <span class="n">discovery_method</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">]</span> <span class="o">=</span> <span class="n">new_policy_published</span>
|
||||
|
||||
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">"record"</span><span class="p">])</span> <span class="ow">is</span> <span class="nb">list</span><span class="p">:</span>
|
||||
@@ -954,6 +1095,7 @@
|
||||
<span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
|
||||
<span class="n">dns_timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">,</span>
|
||||
<span class="n">dns_retries</span><span class="o">=</span><span class="n">retries</span><span class="p">,</span>
|
||||
<span class="n">is_rfc_9990</span><span class="o">=</span><span class="n">is_rfc_9990</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">_append_parsed_record</span><span class="p">(</span>
|
||||
<span class="n">parsed_record</span><span class="o">=</span><span class="n">report_record</span><span class="p">,</span>
|
||||
@@ -976,6 +1118,7 @@
|
||||
<span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
|
||||
<span class="n">dns_timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">,</span>
|
||||
<span class="n">dns_retries</span><span class="o">=</span><span class="n">retries</span><span class="p">,</span>
|
||||
<span class="n">is_rfc_9990</span><span class="o">=</span><span class="n">is_rfc_9990</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">_append_parsed_record</span><span class="p">(</span>
|
||||
<span class="n">parsed_record</span><span class="o">=</span><span class="n">report_record</span><span class="p">,</span>
|
||||
@@ -1072,6 +1215,7 @@
|
||||
<span class="p">)</span>
|
||||
<span class="k">elif</span> <span class="p">(</span>
|
||||
<span class="n">header</span><span class="p">[:</span> <span class="nb">len</span><span class="p">(</span><span class="n">MAGIC_XML</span><span class="p">)]</span> <span class="o">==</span> <span class="n">MAGIC_XML</span>
|
||||
<span class="ow">or</span> <span class="n">header</span><span class="p">[:</span> <span class="nb">len</span><span class="p">(</span><span class="n">MAGIC_XML_TAG</span><span class="p">)]</span> <span class="o">==</span> <span class="n">MAGIC_XML_TAG</span>
|
||||
<span class="ow">or</span> <span class="n">header</span><span class="p">[:</span> <span class="nb">len</span><span class="p">(</span><span class="n">MAGIC_JSON</span><span class="p">)]</span> <span class="o">==</span> <span class="n">MAGIC_JSON</span>
|
||||
<span class="p">):</span>
|
||||
<span class="n">report</span> <span class="o">=</span> <span class="n">file_object</span><span class="o">.</span><span class="n">read</span><span class="p">()</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="n">errors</span><span class="o">=</span><span class="s2">"ignore"</span><span class="p">)</span>
|
||||
@@ -1210,6 +1354,9 @@
|
||||
<span class="n">sp</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"sp"</span><span class="p">]</span>
|
||||
<span class="n">pct</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"pct"</span><span class="p">]</span>
|
||||
<span class="n">fo</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"fo"</span><span class="p">]</span>
|
||||
<span class="n">np_</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"np"</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||||
<span class="n">testing</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"testing"</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||||
<span class="n">discovery_method</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"discovery_method"</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||||
|
||||
<span class="n">report_dict</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
|
||||
<span class="n">xml_schema</span><span class="o">=</span><span class="n">xml_schema</span><span class="p">,</span>
|
||||
@@ -1226,8 +1373,11 @@
|
||||
<span class="n">aspf</span><span class="o">=</span><span class="n">aspf</span><span class="p">,</span>
|
||||
<span class="n">p</span><span class="o">=</span><span class="n">p</span><span class="p">,</span>
|
||||
<span class="n">sp</span><span class="o">=</span><span class="n">sp</span><span class="p">,</span>
|
||||
<span class="n">np</span><span class="o">=</span><span class="n">np_</span><span class="p">,</span>
|
||||
<span class="n">pct</span><span class="o">=</span><span class="n">pct</span><span class="p">,</span>
|
||||
<span class="n">fo</span><span class="o">=</span><span class="n">fo</span><span class="p">,</span>
|
||||
<span class="n">testing</span><span class="o">=</span><span class="n">testing</span><span class="p">,</span>
|
||||
<span class="n">discovery_method</span><span class="o">=</span><span class="n">discovery_method</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">record</span> <span class="ow">in</span> <span class="n">report</span><span class="p">[</span><span class="s2">"records"</span><span class="p">]:</span>
|
||||
@@ -1329,8 +1479,11 @@
|
||||
<span class="s2">"aspf"</span><span class="p">,</span>
|
||||
<span class="s2">"p"</span><span class="p">,</span>
|
||||
<span class="s2">"sp"</span><span class="p">,</span>
|
||||
<span class="s2">"np"</span><span class="p">,</span>
|
||||
<span class="s2">"pct"</span><span class="p">,</span>
|
||||
<span class="s2">"fo"</span><span class="p">,</span>
|
||||
<span class="s2">"testing"</span><span class="p">,</span>
|
||||
<span class="s2">"discovery_method"</span><span class="p">,</span>
|
||||
<span class="s2">"source_ip_address"</span><span class="p">,</span>
|
||||
<span class="s2">"source_country"</span><span class="p">,</span>
|
||||
<span class="s2">"source_reverse_dns"</span><span class="p">,</span>
|
||||
@@ -1372,9 +1525,9 @@
|
||||
|
||||
|
||||
|
||||
<div class="viewcode-block" id="parse_forensic_report">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.parse_forensic_report">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">parse_forensic_report</span><span class="p">(</span>
|
||||
<div class="viewcode-block" id="parse_failure_report">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.parse_failure_report">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">parse_failure_report</span><span class="p">(</span>
|
||||
<span class="n">feedback_report</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
||||
<span class="n">sample</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
||||
<span class="n">msg_date</span><span class="p">:</span> <span class="n">datetime</span><span class="p">,</span>
|
||||
@@ -1388,9 +1541,9 @@
|
||||
<span class="n">dns_timeout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="n">DEFAULT_DNS_TIMEOUT</span><span class="p">,</span>
|
||||
<span class="n">dns_retries</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="n">DEFAULT_DNS_MAX_RETRIES</span><span class="p">,</span>
|
||||
<span class="n">strip_attachment_payloads</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
||||
<span class="p">)</span> <span class="o">-></span> <span class="n">ForensicReport</span><span class="p">:</span>
|
||||
<span class="p">)</span> <span class="o">-></span> <span class="n">FailureReport</span><span class="p">:</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Converts a DMARC forensic report and sample to a dict</span>
|
||||
<span class="sd"> Converts a DMARC failure report and sample to a dict</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> feedback_report (str): A message's feedback report as a string</span>
|
||||
@@ -1407,7 +1560,7 @@
|
||||
<span class="sd"> dns_retries (int): Number of times to retry DNS queries on timeout</span>
|
||||
<span class="sd"> or other transient errors</span>
|
||||
<span class="sd"> strip_attachment_payloads (bool): Remove attachment payloads from</span>
|
||||
<span class="sd"> forensic report results</span>
|
||||
<span class="sd"> failure report results</span>
|
||||
|
||||
<span class="sd"> Returns:</span>
|
||||
<span class="sd"> dict: A parsed report and sample</span>
|
||||
@@ -1423,7 +1576,7 @@
|
||||
|
||||
<span class="k">if</span> <span class="s2">"arrival_date"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_report</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">msg_date</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidForensicReport</span><span class="p">(</span><span class="s2">"Forensic sample is not a valid email"</span><span class="p">)</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidFailureReport</span><span class="p">(</span><span class="s2">"Failure sample is not a valid email"</span><span class="p">)</span>
|
||||
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"arrival_date"</span><span class="p">]</span> <span class="o">=</span> <span class="n">msg_date</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span>
|
||||
|
||||
<span class="k">if</span> <span class="s2">"version"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_report</span><span class="p">:</span>
|
||||
@@ -1465,21 +1618,37 @@
|
||||
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">]</span> <span class="o">=</span> <span class="n">parsed_report_source</span>
|
||||
<span class="k">del</span> <span class="n">parsed_report</span><span class="p">[</span><span class="s2">"source_ip"</span><span class="p">]</span>
|
||||
|
||||
<span class="c1"># Identity-Alignment is REQUIRED per RFC 9991 §4. Default silently for</span>
|
||||
<span class="c1"># backward compatibility with pre-9991 reporters, but log so the</span>
|
||||
<span class="c1"># offending reporter is visible. Values are CFWS-separated per the</span>
|
||||
<span class="c1"># ABNF, so each mechanism is stripped after splitting.</span>
|
||||
<span class="k">if</span> <span class="s2">"identity_alignment"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_report</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
||||
<span class="s2">"Failure report missing required 'Identity-Alignment' "</span>
|
||||
<span class="s2">"field (RFC 9991 §4); defaulting to no aligned mechanisms"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"authentication_mechanisms"</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="k">elif</span> <span class="n">parsed_report</span><span class="p">[</span><span class="s2">"identity_alignment"</span><span class="p">]</span> <span class="o">==</span> <span class="s2">"none"</span><span class="p">:</span>
|
||||
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"authentication_mechanisms"</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="k">del</span> <span class="n">parsed_report</span><span class="p">[</span><span class="s2">"identity_alignment"</span><span class="p">]</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">auth_mechanisms</span> <span class="o">=</span> <span class="n">parsed_report</span><span class="p">[</span><span class="s2">"identity_alignment"</span><span class="p">]</span>
|
||||
<span class="n">auth_mechanisms</span> <span class="o">=</span> <span class="n">auth_mechanisms</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">","</span><span class="p">)</span>
|
||||
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"authentication_mechanisms"</span><span class="p">]</span> <span class="o">=</span> <span class="n">auth_mechanisms</span>
|
||||
<span class="n">raw_alignment</span> <span class="o">=</span> <span class="n">parsed_report</span><span class="p">[</span><span class="s2">"identity_alignment"</span><span class="p">]</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
|
||||
<span class="k">if</span> <span class="n">raw_alignment</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s2">"none"</span><span class="p">:</span>
|
||||
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"authentication_mechanisms"</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"authentication_mechanisms"</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span>
|
||||
<span class="n">m</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span> <span class="k">for</span> <span class="n">m</span> <span class="ow">in</span> <span class="n">raw_alignment</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">","</span><span class="p">)</span> <span class="k">if</span> <span class="n">m</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
|
||||
<span class="p">]</span>
|
||||
<span class="k">del</span> <span class="n">parsed_report</span><span class="p">[</span><span class="s2">"identity_alignment"</span><span class="p">]</span>
|
||||
|
||||
<span class="c1"># Auth-Failure is REQUIRED per RFC 9991 §4. Comma-separated per ABNF</span>
|
||||
<span class="c1"># so strip each token.</span>
|
||||
<span class="k">if</span> <span class="s2">"auth_failure"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_report</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
||||
<span class="s2">"Failure report missing required 'Auth-Failure' field "</span>
|
||||
<span class="s2">"(RFC 9991 §4); defaulting to 'dmarc'"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"auth_failure"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"dmarc"</span>
|
||||
<span class="n">auth_failure</span> <span class="o">=</span> <span class="n">parsed_report</span><span class="p">[</span><span class="s2">"auth_failure"</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">","</span><span class="p">)</span>
|
||||
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"auth_failure"</span><span class="p">]</span> <span class="o">=</span> <span class="n">auth_failure</span>
|
||||
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"auth_failure"</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span>
|
||||
<span class="n">f</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span> <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="n">parsed_report</span><span class="p">[</span><span class="s2">"auth_failure"</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">","</span><span class="p">)</span> <span class="k">if</span> <span class="n">f</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
|
||||
<span class="p">]</span>
|
||||
|
||||
<span class="n">optional_fields</span> <span class="o">=</span> <span class="p">[</span>
|
||||
<span class="s2">"original_envelope_id"</span><span class="p">,</span>
|
||||
@@ -1510,30 +1679,30 @@
|
||||
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"sample"</span><span class="p">]</span> <span class="o">=</span> <span class="n">sample</span>
|
||||
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">]</span> <span class="o">=</span> <span class="n">parsed_sample</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">cast</span><span class="p">(</span><span class="n">ForensicReport</span><span class="p">,</span> <span class="n">parsed_report</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="n">cast</span><span class="p">(</span><span class="n">FailureReport</span><span class="p">,</span> <span class="n">parsed_report</span><span class="p">)</span>
|
||||
|
||||
<span class="k">except</span> <span class="ne">KeyError</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidForensicReport</span><span class="p">(</span><span class="s2">"Missing value: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidFailureReport</span><span class="p">(</span><span class="s2">"Missing value: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
||||
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidForensicReport</span><span class="p">(</span><span class="s2">"Unexpected error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
<span class="k">raise</span> <span class="n">InvalidFailureReport</span><span class="p">(</span><span class="s2">"Unexpected error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
|
||||
|
||||
|
||||
<div class="viewcode-block" id="parsed_forensic_reports_to_csv_rows">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.parsed_forensic_reports_to_csv_rows">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">parsed_forensic_reports_to_csv_rows</span><span class="p">(</span>
|
||||
<span class="n">reports</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">ForensicReport</span><span class="p">,</span> <span class="nb">list</span><span class="p">[</span><span class="n">ForensicReport</span><span class="p">]],</span>
|
||||
<div class="viewcode-block" id="parsed_failure_reports_to_csv_rows">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.parsed_failure_reports_to_csv_rows">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">parsed_failure_reports_to_csv_rows</span><span class="p">(</span>
|
||||
<span class="n">reports</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">FailureReport</span><span class="p">,</span> <span class="nb">list</span><span class="p">[</span><span class="n">FailureReport</span><span class="p">]],</span>
|
||||
<span class="p">)</span> <span class="o">-></span> <span class="nb">list</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]]:</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Converts one or more parsed forensic reports to a list of dicts in flat CSV</span>
|
||||
<span class="sd"> Converts one or more parsed failure reports to a list of dicts in flat CSV</span>
|
||||
<span class="sd"> format</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> reports: A parsed forensic report or list of parsed forensic reports</span>
|
||||
<span class="sd"> reports: A parsed failure report or list of parsed failure reports</span>
|
||||
|
||||
<span class="sd"> Returns:</span>
|
||||
<span class="sd"> list: Parsed forensic report data as a list of dicts in flat CSV format</span>
|
||||
<span class="sd"> list: Parsed failure report data as a list of dicts in flat CSV format</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">reports</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
|
||||
<span class="n">reports</span> <span class="o">=</span> <span class="p">[</span><span class="n">reports</span><span class="p">]</span>
|
||||
@@ -1564,20 +1733,20 @@
|
||||
|
||||
|
||||
|
||||
<div class="viewcode-block" id="parsed_forensic_reports_to_csv">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.parsed_forensic_reports_to_csv">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">parsed_forensic_reports_to_csv</span><span class="p">(</span>
|
||||
<span class="n">reports</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">ForensicReport</span><span class="p">,</span> <span class="nb">list</span><span class="p">[</span><span class="n">ForensicReport</span><span class="p">]],</span>
|
||||
<div class="viewcode-block" id="parsed_failure_reports_to_csv">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.parsed_failure_reports_to_csv">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">parsed_failure_reports_to_csv</span><span class="p">(</span>
|
||||
<span class="n">reports</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">FailureReport</span><span class="p">,</span> <span class="nb">list</span><span class="p">[</span><span class="n">FailureReport</span><span class="p">]],</span>
|
||||
<span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Converts one or more parsed forensic reports to flat CSV format, including</span>
|
||||
<span class="sd"> Converts one or more parsed failure reports to flat CSV format, including</span>
|
||||
<span class="sd"> headers</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> reports: A parsed forensic report or list of parsed forensic reports</span>
|
||||
<span class="sd"> reports: A parsed failure report or list of parsed failure reports</span>
|
||||
|
||||
<span class="sd"> Returns:</span>
|
||||
<span class="sd"> str: Parsed forensic report data in flat CSV format, including headers</span>
|
||||
<span class="sd"> str: Parsed failure report data in flat CSV format, including headers</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">fields</span> <span class="o">=</span> <span class="p">[</span>
|
||||
<span class="s2">"feedback_type"</span><span class="p">,</span>
|
||||
@@ -1612,7 +1781,7 @@
|
||||
<span class="n">csv_writer</span> <span class="o">=</span> <span class="n">DictWriter</span><span class="p">(</span><span class="n">csv_file</span><span class="p">,</span> <span class="n">fieldnames</span><span class="o">=</span><span class="n">fields</span><span class="p">)</span>
|
||||
<span class="n">csv_writer</span><span class="o">.</span><span class="n">writeheader</span><span class="p">()</span>
|
||||
|
||||
<span class="n">rows</span> <span class="o">=</span> <span class="n">parsed_forensic_reports_to_csv_rows</span><span class="p">(</span><span class="n">reports</span><span class="p">)</span>
|
||||
<span class="n">rows</span> <span class="o">=</span> <span class="n">parsed_failure_reports_to_csv_rows</span><span class="p">(</span><span class="n">reports</span><span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="n">rows</span><span class="p">:</span>
|
||||
<span class="n">new_row</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
|
||||
@@ -1656,13 +1825,13 @@
|
||||
<span class="sd"> dns_retries (int): Number of times to retry DNS queries on timeout</span>
|
||||
<span class="sd"> or other transient errors</span>
|
||||
<span class="sd"> strip_attachment_payloads (bool): Remove attachment payloads from</span>
|
||||
<span class="sd"> forensic report results</span>
|
||||
<span class="sd"> failure report results</span>
|
||||
<span class="sd"> keep_alive (callable): keep alive function</span>
|
||||
<span class="sd"> normalize_timespan_threshold_hours (float): Normalize timespans beyond this</span>
|
||||
|
||||
<span class="sd"> Returns:</span>
|
||||
<span class="sd"> dict:</span>
|
||||
<span class="sd"> * ``report_type``: ``aggregate`` or ``forensic``</span>
|
||||
<span class="sd"> * ``report_type``: ``aggregate`` or ``failure``</span>
|
||||
<span class="sd"> * ``report``: The parsed report</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">result</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">ParsedReport</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
@@ -1806,7 +1975,7 @@
|
||||
|
||||
<span class="k">if</span> <span class="n">feedback_report</span> <span class="ow">and</span> <span class="n">sample</span><span class="p">:</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">forensic_report</span> <span class="o">=</span> <span class="n">parse_forensic_report</span><span class="p">(</span>
|
||||
<span class="n">failure_report</span> <span class="o">=</span> <span class="n">parse_failure_report</span><span class="p">(</span>
|
||||
<span class="n">feedback_report</span><span class="p">,</span>
|
||||
<span class="n">sample</span><span class="p">,</span>
|
||||
<span class="n">msg_date</span><span class="p">,</span>
|
||||
@@ -1820,17 +1989,17 @@
|
||||
<span class="n">dns_retries</span><span class="o">=</span><span class="n">dns_retries</span><span class="p">,</span>
|
||||
<span class="n">strip_attachment_payloads</span><span class="o">=</span><span class="n">strip_attachment_payloads</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">except</span> <span class="n">InvalidForensicReport</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">except</span> <span class="n">InvalidFailureReport</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">error</span> <span class="o">=</span> <span class="p">(</span>
|
||||
<span class="s1">'Message with subject "</span><span class="si">{0}</span><span class="s1">" '</span>
|
||||
<span class="s2">"is not a valid "</span>
|
||||
<span class="s2">"forensic DMARC report: </span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">subject</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
|
||||
<span class="s2">"failure DMARC report: </span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">subject</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidForensicReport</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidFailureReport</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidForensicReport</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidFailureReport</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
|
||||
<span class="n">result</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"report_type"</span><span class="p">:</span> <span class="s2">"forensic"</span><span class="p">,</span> <span class="s2">"report"</span><span class="p">:</span> <span class="n">forensic_report</span><span class="p">}</span>
|
||||
<span class="n">result</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"report_type"</span><span class="p">:</span> <span class="s2">"failure"</span><span class="p">,</span> <span class="s2">"report"</span><span class="p">:</span> <span class="n">failure_report</span><span class="p">}</span>
|
||||
<span class="k">return</span> <span class="n">result</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">result</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
@@ -1858,7 +2027,7 @@
|
||||
<span class="n">keep_alive</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Callable</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">normalize_timespan_threshold_hours</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mi">24</span><span class="p">,</span>
|
||||
<span class="p">)</span> <span class="o">-></span> <span class="n">ParsedReport</span><span class="p">:</span>
|
||||
<span class="w"> </span><span class="sd">"""Parses a DMARC aggregate or forensic file at the given path, a</span>
|
||||
<span class="w"> </span><span class="sd">"""Parses a DMARC aggregate or failure file at the given path, a</span>
|
||||
<span class="sd"> file-like object. or bytes</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
@@ -1870,7 +2039,7 @@
|
||||
<span class="sd"> dns_retries (int): Number of times to retry DNS queries on timeout</span>
|
||||
<span class="sd"> or other transient errors</span>
|
||||
<span class="sd"> strip_attachment_payloads (bool): Remove attachment payloads from</span>
|
||||
<span class="sd"> forensic report results</span>
|
||||
<span class="sd"> failure report results</span>
|
||||
<span class="sd"> ip_db_path (str): Path to a MMDB file from IPinfo, MaxMind, or DBIP</span>
|
||||
<span class="sd"> always_use_local_files (bool): Do not download files</span>
|
||||
<span class="sd"> reverse_dns_map_path (str): Path to a reverse DNS map</span>
|
||||
@@ -1969,7 +2138,7 @@
|
||||
<span class="sd"> dns_retries (int): Number of times to retry DNS queries on timeout</span>
|
||||
<span class="sd"> or other transient errors</span>
|
||||
<span class="sd"> strip_attachment_payloads (bool): Remove attachment payloads from</span>
|
||||
<span class="sd"> forensic report results</span>
|
||||
<span class="sd"> failure report results</span>
|
||||
<span class="sd"> always_use_local_files (bool): Do not download files</span>
|
||||
<span class="sd"> reverse_dns_map_path (str): Path to a reverse DNS map file</span>
|
||||
<span class="sd"> reverse_dns_map_url (str): URL to a reverse DNS map file</span>
|
||||
@@ -1978,11 +2147,11 @@
|
||||
<span class="sd"> normalize_timespan_threshold_hours (float): Normalize timespans beyond this</span>
|
||||
|
||||
<span class="sd"> Returns:</span>
|
||||
<span class="sd"> dict: Lists of ``aggregate_reports``, ``forensic_reports``, and ``smtp_tls_reports``</span>
|
||||
<span class="sd"> dict: Lists of ``aggregate_reports``, ``failure_reports``, and ``smtp_tls_reports``</span>
|
||||
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">aggregate_reports</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">AggregateReport</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="n">forensic_reports</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">ForensicReport</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="n">failure_reports</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">FailureReport</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="n">smtp_tls_reports</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">SMTPTLSReport</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">mbox</span> <span class="o">=</span> <span class="n">mailbox</span><span class="o">.</span><span class="n">mbox</span><span class="p">(</span><span class="n">input_</span><span class="p">)</span>
|
||||
@@ -2020,8 +2189,8 @@
|
||||
<span class="s2">"Skipping duplicate aggregate report "</span>
|
||||
<span class="sa">f</span><span class="s2">"from </span><span class="si">{</span><span class="n">report_org</span><span class="si">}</span><span class="s2"> with ID: </span><span class="si">{</span><span class="n">report_id</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">elif</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">"report_type"</span><span class="p">]</span> <span class="o">==</span> <span class="s2">"forensic"</span><span class="p">:</span>
|
||||
<span class="n">forensic_reports</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">parsed_email</span><span class="p">[</span><span class="s2">"report"</span><span class="p">])</span>
|
||||
<span class="k">elif</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">"report_type"</span><span class="p">]</span> <span class="o">==</span> <span class="s2">"failure"</span><span class="p">:</span>
|
||||
<span class="n">failure_reports</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">parsed_email</span><span class="p">[</span><span class="s2">"report"</span><span class="p">])</span>
|
||||
<span class="k">elif</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">"report_type"</span><span class="p">]</span> <span class="o">==</span> <span class="s2">"smtp_tls"</span><span class="p">:</span>
|
||||
<span class="n">smtp_tls_reports</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">parsed_email</span><span class="p">[</span><span class="s2">"report"</span><span class="p">])</span>
|
||||
<span class="k">except</span> <span class="n">InvalidDMARCReport</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
||||
@@ -2030,12 +2199,60 @@
|
||||
<span class="k">raise</span> <span class="n">InvalidDMARCReport</span><span class="p">(</span><span class="s2">"Mailbox </span><span class="si">{0}</span><span class="s2"> does not exist"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">input_</span><span class="p">))</span>
|
||||
<span class="k">return</span> <span class="p">{</span>
|
||||
<span class="s2">"aggregate_reports"</span><span class="p">:</span> <span class="n">aggregate_reports</span><span class="p">,</span>
|
||||
<span class="s2">"forensic_reports"</span><span class="p">:</span> <span class="n">forensic_reports</span><span class="p">,</span>
|
||||
<span class="s2">"failure_reports"</span><span class="p">:</span> <span class="n">failure_reports</span><span class="p">,</span>
|
||||
<span class="s2">"smtp_tls_reports"</span><span class="p">:</span> <span class="n">smtp_tls_reports</span><span class="p">,</span>
|
||||
<span class="p">}</span></div>
|
||||
|
||||
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">_migrate_forensic_archive_folder</span><span class="p">(</span>
|
||||
<span class="n">connection</span><span class="p">:</span> <span class="n">MailboxConnection</span><span class="p">,</span> <span class="n">archive_folder</span><span class="p">:</span> <span class="nb">str</span>
|
||||
<span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="w"> </span><span class="sd">"""Consolidate a pre-rename ``<archive>/Forensic`` subfolder into</span>
|
||||
<span class="sd"> ``<archive>/Failure``.</span>
|
||||
|
||||
<span class="sd"> Before failure reports were renamed from "forensic" reports, they were</span>
|
||||
<span class="sd"> archived under ``<archive_folder>/Forensic``; they now go to</span>
|
||||
<span class="sd"> ``<archive_folder>/Failure``. This best-effort, run-on-startup migration</span>
|
||||
<span class="sd"> moves any pre-existing legacy archive into the new location so reports</span>
|
||||
<span class="sd"> filed before and after the rename live in the same folder.</span>
|
||||
|
||||
<span class="sd"> It is a no-op when there is no legacy ``Forensic`` folder (the common</span>
|
||||
<span class="sd"> case), and never raises: a mailbox that cannot be reorganized is logged</span>
|
||||
<span class="sd"> and skipped, consistent with the rest of parsedmarc's mailbox handling</span>
|
||||
<span class="sd"> (warn, don't crash). Uses the folder-management API added in mailsuite</span>
|
||||
<span class="sd"> 2.1.0 (``folder_exists`` / ``rename_folder`` / ``merge_folders``).</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">old_folder</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">/Forensic"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">archive_folder</span><span class="p">)</span>
|
||||
<span class="n">new_folder</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">/Failure"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">archive_folder</span><span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">connection</span><span class="o">.</span><span class="n">folder_exists</span><span class="p">(</span><span class="n">old_folder</span><span class="p">):</span>
|
||||
<span class="k">return</span>
|
||||
<span class="k">if</span> <span class="n">connection</span><span class="o">.</span><span class="n">folder_exists</span><span class="p">(</span><span class="n">new_folder</span><span class="p">):</span>
|
||||
<span class="c1"># Both exist (e.g. a partial earlier migration, or a manually</span>
|
||||
<span class="c1"># created Failure folder): move the legacy folder's messages into</span>
|
||||
<span class="c1"># the new one and drop the now-empty legacy folder.</span>
|
||||
<span class="n">connection</span><span class="o">.</span><span class="n">merge_folders</span><span class="p">(</span><span class="n">old_folder</span><span class="p">,</span> <span class="n">new_folder</span><span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
|
||||
<span class="s2">"Merged legacy archive folder </span><span class="si">{0}</span><span class="s2"> into </span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">old_folder</span><span class="p">,</span> <span class="n">new_folder</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">connection</span><span class="o">.</span><span class="n">rename_folder</span><span class="p">(</span><span class="n">old_folder</span><span class="p">,</span> <span class="n">new_folder</span><span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
|
||||
<span class="s2">"Renamed legacy archive folder </span><span class="si">{0}</span><span class="s2"> to </span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">old_folder</span><span class="p">,</span> <span class="n">new_folder</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
||||
<span class="s2">"Could not migrate legacy archive folder </span><span class="si">{0}</span><span class="s2"> to </span><span class="si">{1}</span><span class="s2">: </span><span class="si">{2}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">old_folder</span><span class="p">,</span> <span class="n">new_folder</span><span class="p">,</span> <span class="n">error</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="get_dmarc_reports_from_mailbox">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.get_dmarc_reports_from_mailbox">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">get_dmarc_reports_from_mailbox</span><span class="p">(</span>
|
||||
@@ -2079,7 +2296,7 @@
|
||||
<span class="sd"> dns_retries (int): Number of times to retry DNS queries on timeout</span>
|
||||
<span class="sd"> or other transient errors</span>
|
||||
<span class="sd"> strip_attachment_payloads (bool): Remove attachment payloads from</span>
|
||||
<span class="sd"> forensic report results</span>
|
||||
<span class="sd"> failure report results</span>
|
||||
<span class="sd"> results (dict): Results from the previous run</span>
|
||||
<span class="sd"> batch_size (int): Number of messages to read and process before saving</span>
|
||||
<span class="sd"> (use 0 for no limit)</span>
|
||||
@@ -2090,7 +2307,7 @@
|
||||
<span class="sd"> normalize_timespan_threshold_hours (float): Normalize timespans beyond this</span>
|
||||
|
||||
<span class="sd"> Returns:</span>
|
||||
<span class="sd"> dict: Lists of ``aggregate_reports``, ``forensic_reports``, and ``smtp_tls_reports``</span>
|
||||
<span class="sd"> dict: Lists of ``aggregate_reports``, ``failure_reports``, and ``smtp_tls_reports``</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">if</span> <span class="n">delete</span> <span class="ow">and</span> <span class="n">test</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"delete and test options are mutually exclusive"</span><span class="p">)</span>
|
||||
@@ -2102,25 +2319,26 @@
|
||||
<span class="n">current_time</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">datetime</span><span class="p">,</span> <span class="n">date</span><span class="p">,</span> <span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
|
||||
<span class="n">aggregate_reports</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">AggregateReport</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="n">forensic_reports</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">ForensicReport</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="n">failure_reports</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">FailureReport</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="n">smtp_tls_reports</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">SMTPTLSReport</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="n">aggregate_report_msg_uids</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="n">forensic_report_msg_uids</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="n">failure_report_msg_uids</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="n">smtp_tls_msg_uids</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="n">aggregate_reports_folder</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">/Aggregate"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">archive_folder</span><span class="p">)</span>
|
||||
<span class="n">forensic_reports_folder</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">/Forensic"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">archive_folder</span><span class="p">)</span>
|
||||
<span class="n">failure_reports_folder</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">/Failure"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">archive_folder</span><span class="p">)</span>
|
||||
<span class="n">smtp_tls_reports_folder</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">/SMTP-TLS"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">archive_folder</span><span class="p">)</span>
|
||||
<span class="n">invalid_reports_folder</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">/Invalid"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">archive_folder</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">results</span><span class="p">:</span>
|
||||
<span class="n">aggregate_reports</span> <span class="o">=</span> <span class="n">results</span><span class="p">[</span><span class="s2">"aggregate_reports"</span><span class="p">]</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
<span class="n">forensic_reports</span> <span class="o">=</span> <span class="n">results</span><span class="p">[</span><span class="s2">"forensic_reports"</span><span class="p">]</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
<span class="n">failure_reports</span> <span class="o">=</span> <span class="n">results</span><span class="p">[</span><span class="s2">"failure_reports"</span><span class="p">]</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
<span class="n">smtp_tls_reports</span> <span class="o">=</span> <span class="n">results</span><span class="p">[</span><span class="s2">"smtp_tls_reports"</span><span class="p">]</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">test</span> <span class="ow">and</span> <span class="n">create_folders</span><span class="p">:</span>
|
||||
<span class="n">_migrate_forensic_archive_folder</span><span class="p">(</span><span class="n">connection</span><span class="p">,</span> <span class="n">archive_folder</span><span class="p">)</span>
|
||||
<span class="n">connection</span><span class="o">.</span><span class="n">create_folder</span><span class="p">(</span><span class="n">archive_folder</span><span class="p">)</span>
|
||||
<span class="n">connection</span><span class="o">.</span><span class="n">create_folder</span><span class="p">(</span><span class="n">aggregate_reports_folder</span><span class="p">)</span>
|
||||
<span class="n">connection</span><span class="o">.</span><span class="n">create_folder</span><span class="p">(</span><span class="n">forensic_reports_folder</span><span class="p">)</span>
|
||||
<span class="n">connection</span><span class="o">.</span><span class="n">create_folder</span><span class="p">(</span><span class="n">failure_reports_folder</span><span class="p">)</span>
|
||||
<span class="n">connection</span><span class="o">.</span><span class="n">create_folder</span><span class="p">(</span><span class="n">smtp_tls_reports_folder</span><span class="p">)</span>
|
||||
<span class="n">connection</span><span class="o">.</span><span class="n">create_folder</span><span class="p">(</span><span class="n">invalid_reports_folder</span><span class="p">)</span>
|
||||
|
||||
@@ -2226,9 +2444,9 @@
|
||||
<span class="sa">f</span><span class="s2">"Skipping duplicate aggregate report with ID: </span><span class="si">{</span><span class="n">report_id</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">aggregate_report_msg_uids</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">message_id</span><span class="p">)</span>
|
||||
<span class="k">elif</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">"report_type"</span><span class="p">]</span> <span class="o">==</span> <span class="s2">"forensic"</span><span class="p">:</span>
|
||||
<span class="n">forensic_reports</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">parsed_email</span><span class="p">[</span><span class="s2">"report"</span><span class="p">])</span>
|
||||
<span class="n">forensic_report_msg_uids</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">message_id</span><span class="p">)</span>
|
||||
<span class="k">elif</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">"report_type"</span><span class="p">]</span> <span class="o">==</span> <span class="s2">"failure"</span><span class="p">:</span>
|
||||
<span class="n">failure_reports</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">parsed_email</span><span class="p">[</span><span class="s2">"report"</span><span class="p">])</span>
|
||||
<span class="n">failure_report_msg_uids</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">message_id</span><span class="p">)</span>
|
||||
<span class="k">elif</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">"report_type"</span><span class="p">]</span> <span class="o">==</span> <span class="s2">"smtp_tls"</span><span class="p">:</span>
|
||||
<span class="n">smtp_tls_reports</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">parsed_email</span><span class="p">[</span><span class="s2">"report"</span><span class="p">])</span>
|
||||
<span class="n">smtp_tls_msg_uids</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">message_id</span><span class="p">)</span>
|
||||
@@ -2255,7 +2473,7 @@
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">test</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">delete</span><span class="p">:</span>
|
||||
<span class="n">processed_messages</span> <span class="o">=</span> <span class="p">(</span>
|
||||
<span class="n">aggregate_report_msg_uids</span> <span class="o">+</span> <span class="n">forensic_report_msg_uids</span> <span class="o">+</span> <span class="n">smtp_tls_msg_uids</span>
|
||||
<span class="n">aggregate_report_msg_uids</span> <span class="o">+</span> <span class="n">failure_report_msg_uids</span> <span class="o">+</span> <span class="n">smtp_tls_msg_uids</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">number_of_processed_msgs</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">processed_messages</span><span class="p">)</span>
|
||||
@@ -2295,24 +2513,24 @@
|
||||
<span class="n">message</span> <span class="o">=</span> <span class="s2">"Error moving message UID"</span>
|
||||
<span class="n">e</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2"> </span><span class="si">{1}</span><span class="s2">: </span><span class="si">{2}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">message</span><span class="p">,</span> <span class="n">msg_uid</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Mailbox error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="p">))</span>
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">forensic_report_msg_uids</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="n">message</span> <span class="o">=</span> <span class="s2">"Moving forensic report messages from"</span>
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">failure_report_msg_uids</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="n">message</span> <span class="o">=</span> <span class="s2">"Moving failure report messages from"</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
||||
<span class="s2">"</span><span class="si">{0}</span><span class="s2"> </span><span class="si">{1}</span><span class="s2"> to </span><span class="si">{2}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">message</span><span class="p">,</span> <span class="n">reports_folder</span><span class="p">,</span> <span class="n">forensic_reports_folder</span>
|
||||
<span class="n">message</span><span class="p">,</span> <span class="n">reports_folder</span><span class="p">,</span> <span class="n">failure_reports_folder</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">number_of_forensic_msgs</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">forensic_report_msg_uids</span><span class="p">)</span>
|
||||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">number_of_forensic_msgs</span><span class="p">):</span>
|
||||
<span class="n">msg_uid</span> <span class="o">=</span> <span class="n">forensic_report_msg_uids</span><span class="p">[</span><span class="n">i</span><span class="p">]</span>
|
||||
<span class="n">number_of_failure_msgs</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">failure_report_msg_uids</span><span class="p">)</span>
|
||||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">number_of_failure_msgs</span><span class="p">):</span>
|
||||
<span class="n">msg_uid</span> <span class="o">=</span> <span class="n">failure_report_msg_uids</span><span class="p">[</span><span class="n">i</span><span class="p">]</span>
|
||||
<span class="n">message</span> <span class="o">=</span> <span class="s2">"Moving message"</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
||||
<span class="s2">"</span><span class="si">{0}</span><span class="s2"> </span><span class="si">{1}</span><span class="s2"> of </span><span class="si">{2}</span><span class="s2">: UID </span><span class="si">{3}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">message</span><span class="p">,</span> <span class="n">i</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="n">number_of_forensic_msgs</span><span class="p">,</span> <span class="n">msg_uid</span>
|
||||
<span class="n">message</span><span class="p">,</span> <span class="n">i</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="n">number_of_failure_msgs</span><span class="p">,</span> <span class="n">msg_uid</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">connection</span><span class="o">.</span><span class="n">move_message</span><span class="p">(</span><span class="n">msg_uid</span><span class="p">,</span> <span class="n">forensic_reports_folder</span><span class="p">)</span>
|
||||
<span class="n">connection</span><span class="o">.</span><span class="n">move_message</span><span class="p">(</span><span class="n">msg_uid</span><span class="p">,</span> <span class="n">failure_reports_folder</span><span class="p">)</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">e</span> <span class="o">=</span> <span class="s2">"Error moving message UID </span><span class="si">{0}</span><span class="s2">: </span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">msg_uid</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Mailbox error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="p">))</span>
|
||||
@@ -2339,7 +2557,7 @@
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Mailbox error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="p">))</span>
|
||||
<span class="n">results</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s2">"aggregate_reports"</span><span class="p">:</span> <span class="n">aggregate_reports</span><span class="p">,</span>
|
||||
<span class="s2">"forensic_reports"</span><span class="p">:</span> <span class="n">forensic_reports</span><span class="p">,</span>
|
||||
<span class="s2">"failure_reports"</span><span class="p">:</span> <span class="n">failure_reports</span><span class="p">,</span>
|
||||
<span class="s2">"smtp_tls_reports"</span><span class="p">:</span> <span class="n">smtp_tls_reports</span><span class="p">,</span>
|
||||
<span class="p">}</span>
|
||||
|
||||
@@ -2428,7 +2646,7 @@
|
||||
<span class="sd"> dns_retries (int): Number of times to retry DNS queries on timeout</span>
|
||||
<span class="sd"> or other transient errors</span>
|
||||
<span class="sd"> strip_attachment_payloads (bool): Replace attachment payloads in</span>
|
||||
<span class="sd"> forensic report samples with None</span>
|
||||
<span class="sd"> failure report samples with None</span>
|
||||
<span class="sd"> batch_size (int): Number of messages to read and process before saving</span>
|
||||
<span class="sd"> since: Search for messages since certain time</span>
|
||||
<span class="sd"> normalize_timespan_threshold_hours (float): Normalize timespans beyond this</span>
|
||||
@@ -2470,34 +2688,50 @@
|
||||
|
||||
|
||||
|
||||
<div class="viewcode-block" id="append_json">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.append_json">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">append_json</span><span class="p">(</span>
|
||||
<span class="n">filename</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
||||
<span class="n">reports</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span>
|
||||
<span class="n">Sequence</span><span class="p">[</span><span class="n">AggregateReport</span><span class="p">],</span>
|
||||
<span class="n">Sequence</span><span class="p">[</span><span class="n">ForensicReport</span><span class="p">],</span>
|
||||
<span class="n">Sequence</span><span class="p">[</span><span class="n">FailureReport</span><span class="p">],</span>
|
||||
<span class="n">Sequence</span><span class="p">[</span><span class="n">SMTPTLSReport</span><span class="p">],</span>
|
||||
<span class="p">],</span>
|
||||
<span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="s2">"a+"</span><span class="p">,</span> <span class="n">newline</span><span class="o">=</span><span class="s2">"</span><span class="se">\n</span><span class="s2">"</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s2">"utf-8"</span><span class="p">)</span> <span class="k">as</span> <span class="n">output</span><span class="p">:</span>
|
||||
<span class="n">output_json</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">reports</span><span class="p">,</span> <span class="n">ensure_ascii</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">indent</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">output</span><span class="o">.</span><span class="n">seek</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">os</span><span class="o">.</span><span class="n">SEEK_END</span><span class="p">)</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">reports</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="c1"># not appending anything, don't do any dance to append it</span>
|
||||
<span class="c1"># correctly</span>
|
||||
<span class="k">return</span>
|
||||
<span class="n">output</span><span class="o">.</span><span class="n">seek</span><span class="p">(</span><span class="n">output</span><span class="o">.</span><span class="n">tell</span><span class="p">()</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span>
|
||||
<span class="n">last_char</span> <span class="o">=</span> <span class="n">output</span><span class="o">.</span><span class="n">read</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">last_char</span> <span class="o">==</span> <span class="s2">"]"</span><span class="p">:</span>
|
||||
<span class="c1"># remove the trailing "\n]", leading "[\n", and replace with</span>
|
||||
<span class="c1"># ",\n"</span>
|
||||
<span class="n">output</span><span class="o">.</span><span class="n">seek</span><span class="p">(</span><span class="n">output</span><span class="o">.</span><span class="n">tell</span><span class="p">()</span> <span class="o">-</span> <span class="mi">2</span><span class="p">)</span>
|
||||
<span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s2">",</span><span class="se">\n</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="n">output_json</span> <span class="o">=</span> <span class="n">output_json</span><span class="p">[</span><span class="mi">2</span><span class="p">:]</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">output</span><span class="o">.</span><span class="n">seek</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
|
||||
<span class="n">output</span><span class="o">.</span><span class="n">truncate</span><span class="p">()</span>
|
||||
<span class="w"> </span><span class="sd">"""Append ``reports`` to a JSON array on disk, creating the file</span>
|
||||
<span class="sd"> if needed.</span>
|
||||
|
||||
<span class="sd"> Reads the existing array (if the file exists and parses cleanly),</span>
|
||||
<span class="sd"> merges the new reports onto the end, and rewrites the file as a</span>
|
||||
<span class="sd"> single valid JSON array. An earlier version of this used an</span>
|
||||
<span class="sd"> ``open(..., "a+")`` + ``seek()`` + overwrite pattern, but Python's</span>
|
||||
<span class="sd"> documentation is explicit that on POSIX, ``a`` / ``a+`` writes</span>
|
||||
<span class="sd"> *always* go to EOF regardless of seek position — so the second</span>
|
||||
<span class="sd"> call onto an existing file produced ``[...],\\n[...]``-style</span>
|
||||
<span class="sd"> corrupted output. Read-merge-write is the only way to get a valid</span>
|
||||
<span class="sd"> JSON array out of repeated appends.</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">reports</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="c1"># Don't create an empty-array file for an empty input; if a</span>
|
||||
<span class="c1"># file already exists, leave it alone.</span>
|
||||
<span class="k">return</span>
|
||||
|
||||
<span class="n">existing</span><span class="p">:</span> <span class="nb">list</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isfile</span><span class="p">(</span><span class="n">filename</span><span class="p">)</span> <span class="ow">and</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">getsize</span><span class="p">(</span><span class="n">filename</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="s2">"r"</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s2">"utf-8"</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
|
||||
<span class="n">loaded</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">read</span><span class="p">())</span>
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">loaded</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
|
||||
<span class="n">existing</span> <span class="o">=</span> <span class="n">loaded</span>
|
||||
<span class="k">except</span> <span class="p">(</span><span class="n">json</span><span class="o">.</span><span class="n">JSONDecodeError</span><span class="p">,</span> <span class="ne">OSError</span><span class="p">):</span>
|
||||
<span class="c1"># Corrupted or unreadable: overwrite cleanly rather than</span>
|
||||
<span class="c1"># silently fail to record.</span>
|
||||
<span class="n">existing</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
|
||||
<span class="n">merged</span> <span class="o">=</span> <span class="n">existing</span> <span class="o">+</span> <span class="nb">list</span><span class="p">(</span><span class="n">reports</span><span class="p">)</span>
|
||||
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="s2">"w"</span><span class="p">,</span> <span class="n">newline</span><span class="o">=</span><span class="s2">"</span><span class="se">\n</span><span class="s2">"</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s2">"utf-8"</span><span class="p">)</span> <span class="k">as</span> <span class="n">output</span><span class="p">:</span>
|
||||
<span class="n">json</span><span class="o">.</span><span class="n">dump</span><span class="p">(</span><span class="n">merged</span><span class="p">,</span> <span class="n">output</span><span class="p">,</span> <span class="n">ensure_ascii</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">indent</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span></div>
|
||||
|
||||
<span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">output_json</span><span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">append_csv</span><span class="p">(</span><span class="n">filename</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">csv</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||
@@ -2519,10 +2753,10 @@
|
||||
<span class="o">*</span><span class="p">,</span>
|
||||
<span class="n">output_directory</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"output"</span><span class="p">,</span>
|
||||
<span class="n">aggregate_json_filename</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"aggregate.json"</span><span class="p">,</span>
|
||||
<span class="n">forensic_json_filename</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"forensic.json"</span><span class="p">,</span>
|
||||
<span class="n">failure_json_filename</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"failure.json"</span><span class="p">,</span>
|
||||
<span class="n">smtp_tls_json_filename</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"smtp_tls.json"</span><span class="p">,</span>
|
||||
<span class="n">aggregate_csv_filename</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"aggregate.csv"</span><span class="p">,</span>
|
||||
<span class="n">forensic_csv_filename</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"forensic.csv"</span><span class="p">,</span>
|
||||
<span class="n">failure_csv_filename</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"failure.csv"</span><span class="p">,</span>
|
||||
<span class="n">smtp_tls_csv_filename</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"smtp_tls.csv"</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
@@ -2532,15 +2766,15 @@
|
||||
<span class="sd"> results: Parsing results</span>
|
||||
<span class="sd"> output_directory (str): The path to the directory to save in</span>
|
||||
<span class="sd"> aggregate_json_filename (str): Filename for the aggregate JSON file</span>
|
||||
<span class="sd"> forensic_json_filename (str): Filename for the forensic JSON file</span>
|
||||
<span class="sd"> failure_json_filename (str): Filename for the failure JSON file</span>
|
||||
<span class="sd"> smtp_tls_json_filename (str): Filename for the SMTP TLS JSON file</span>
|
||||
<span class="sd"> aggregate_csv_filename (str): Filename for the aggregate CSV file</span>
|
||||
<span class="sd"> forensic_csv_filename (str): Filename for the forensic CSV file</span>
|
||||
<span class="sd"> failure_csv_filename (str): Filename for the failure CSV file</span>
|
||||
<span class="sd"> smtp_tls_csv_filename (str): Filename for the SMTP TLS CSV file</span>
|
||||
<span class="sd"> """</span>
|
||||
|
||||
<span class="n">aggregate_reports</span> <span class="o">=</span> <span class="n">results</span><span class="p">[</span><span class="s2">"aggregate_reports"</span><span class="p">]</span>
|
||||
<span class="n">forensic_reports</span> <span class="o">=</span> <span class="n">results</span><span class="p">[</span><span class="s2">"forensic_reports"</span><span class="p">]</span>
|
||||
<span class="n">failure_reports</span> <span class="o">=</span> <span class="n">results</span><span class="p">[</span><span class="s2">"failure_reports"</span><span class="p">]</span>
|
||||
<span class="n">smtp_tls_reports</span> <span class="o">=</span> <span class="n">results</span><span class="p">[</span><span class="s2">"smtp_tls_reports"</span><span class="p">]</span>
|
||||
<span class="n">output_directory</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">expanduser</span><span class="p">(</span><span class="n">output_directory</span><span class="p">)</span>
|
||||
|
||||
@@ -2559,13 +2793,11 @@
|
||||
<span class="n">parsed_aggregate_reports_to_csv</span><span class="p">(</span><span class="n">aggregate_reports</span><span class="p">),</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">append_json</span><span class="p">(</span>
|
||||
<span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">output_directory</span><span class="p">,</span> <span class="n">forensic_json_filename</span><span class="p">),</span> <span class="n">forensic_reports</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">append_json</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">output_directory</span><span class="p">,</span> <span class="n">failure_json_filename</span><span class="p">),</span> <span class="n">failure_reports</span><span class="p">)</span>
|
||||
|
||||
<span class="n">append_csv</span><span class="p">(</span>
|
||||
<span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">output_directory</span><span class="p">,</span> <span class="n">forensic_csv_filename</span><span class="p">),</span>
|
||||
<span class="n">parsed_forensic_reports_to_csv</span><span class="p">(</span><span class="n">forensic_reports</span><span class="p">),</span>
|
||||
<span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">output_directory</span><span class="p">,</span> <span class="n">failure_csv_filename</span><span class="p">),</span>
|
||||
<span class="n">parsed_failure_reports_to_csv</span><span class="p">(</span><span class="n">failure_reports</span><span class="p">),</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">append_json</span><span class="p">(</span>
|
||||
@@ -2582,10 +2814,10 @@
|
||||
<span class="n">os</span><span class="o">.</span><span class="n">makedirs</span><span class="p">(</span><span class="n">samples_directory</span><span class="p">)</span>
|
||||
|
||||
<span class="n">sample_filenames</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="k">for</span> <span class="n">forensic_report</span> <span class="ow">in</span> <span class="n">forensic_reports</span><span class="p">:</span>
|
||||
<span class="n">sample</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"sample"</span><span class="p">]</span>
|
||||
<span class="k">for</span> <span class="n">failure_report</span> <span class="ow">in</span> <span class="n">failure_reports</span><span class="p">:</span>
|
||||
<span class="n">sample</span> <span class="o">=</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"sample"</span><span class="p">]</span>
|
||||
<span class="n">message_count</span> <span class="o">=</span> <span class="mi">0</span>
|
||||
<span class="n">parsed_sample</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">]</span>
|
||||
<span class="n">parsed_sample</span> <span class="o">=</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">]</span>
|
||||
<span class="n">subject</span> <span class="o">=</span> <span class="p">(</span>
|
||||
<span class="n">parsed_sample</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"filename_safe_subject"</span><span class="p">)</span>
|
||||
<span class="ow">or</span> <span class="n">parsed_sample</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"subject"</span><span class="p">)</span>
|
||||
@@ -2726,6 +2958,12 @@
|
||||
<span class="n">plain_message</span><span class="o">=</span><span class="n">message</span><span class="p">,</span>
|
||||
<span class="p">)</span></div>
|
||||
|
||||
|
||||
|
||||
<span class="c1"># Backward-compatible aliases</span>
|
||||
<span class="n">parse_forensic_report</span> <span class="o">=</span> <span class="n">parse_failure_report</span>
|
||||
<span class="n">parsed_forensic_reports_to_csv_rows</span> <span class="o">=</span> <span class="n">parsed_failure_reports_to_csv_rows</span>
|
||||
<span class="n">parsed_forensic_reports_to_csv</span> <span class="o">=</span> <span class="n">parsed_failure_reports_to_csv</span>
|
||||
</pre></div>
|
||||
|
||||
</div>
|
||||
|
||||
@@ -5,14 +5,14 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc.elastic — parsedmarc 9.11.2 documentation</title>
|
||||
<title>parsedmarc.elastic — parsedmarc 10.0.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<script src="../../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../../_static/documentation_options.js?v=de4344a5"></script>
|
||||
<script src="../../_static/documentation_options.js?v=335988e4"></script>
|
||||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="../../_static/js/theme.js"></script>
|
||||
@@ -95,6 +95,7 @@
|
||||
<span class="n">InnerDoc</span><span class="p">,</span>
|
||||
<span class="n">Integer</span><span class="p">,</span>
|
||||
<span class="n">Ip</span><span class="p">,</span>
|
||||
<span class="n">Keyword</span><span class="p">,</span>
|
||||
<span class="n">Nested</span><span class="p">,</span>
|
||||
<span class="n">Object</span><span class="p">,</span>
|
||||
<span class="n">Search</span><span class="p">,</span>
|
||||
@@ -103,7 +104,7 @@
|
||||
<span class="p">)</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">elasticsearch_dsl.search</span><span class="w"> </span><span class="kn">import</span> <span class="n">Q</span>
|
||||
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc</span><span class="w"> </span><span class="kn">import</span> <span class="n">InvalidForensicReport</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc</span><span class="w"> </span><span class="kn">import</span> <span class="n">InvalidFailureReport</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.log</span><span class="w"> </span><span class="kn">import</span> <span class="n">logger</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">human_timestamp_to_datetime</span>
|
||||
|
||||
@@ -115,6 +116,18 @@
|
||||
|
||||
|
||||
|
||||
<span class="c1"># Mirror of the ``serverless`` flag passed to ``set_hosts``; consulted by</span>
|
||||
<span class="c1"># ``create_indexes`` to strip settings Elastic Cloud Serverless rejects.</span>
|
||||
<span class="c1"># Module-level state is consistent with the existing ``connections.create_connection``</span>
|
||||
<span class="c1"># global the rest of this module relies on — there is a single default ES</span>
|
||||
<span class="c1"># connection per process.</span>
|
||||
<span class="n">_SERVERLESS</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
|
||||
<span class="c1"># Index settings rejected by Elastic Cloud Serverless with HTTP 400. Other</span>
|
||||
<span class="c1"># settings (e.g. ``refresh_interval``) are accepted and pass through.</span>
|
||||
<span class="n">_SERVERLESS_REJECTED_SETTINGS</span> <span class="o">=</span> <span class="nb">frozenset</span><span class="p">({</span><span class="s2">"number_of_shards"</span><span class="p">,</span> <span class="s2">"number_of_replicas"</span><span class="p">})</span>
|
||||
|
||||
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_PolicyOverride</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="nb">type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">comment</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
@@ -128,18 +141,23 @@
|
||||
<span class="n">sp</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">pct</span> <span class="o">=</span> <span class="n">Integer</span><span class="p">()</span>
|
||||
<span class="n">fo</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">np</span> <span class="o">=</span> <span class="n">Keyword</span><span class="p">()</span>
|
||||
<span class="n">testing</span> <span class="o">=</span> <span class="n">Keyword</span><span class="p">()</span>
|
||||
<span class="n">discovery_method</span> <span class="o">=</span> <span class="n">Keyword</span><span class="p">()</span>
|
||||
|
||||
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_DKIMResult</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="n">domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">selector</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">result</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">human_result</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
|
||||
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_SPFResult</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="n">domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">scope</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">results</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">human_result</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
|
||||
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_AggregateReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
@@ -147,6 +165,7 @@
|
||||
<span class="n">name</span> <span class="o">=</span> <span class="s2">"dmarc_aggregate"</span>
|
||||
|
||||
<span class="n">xml_schema</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">xml_namespace</span> <span class="o">=</span> <span class="n">Keyword</span><span class="p">()</span>
|
||||
<span class="n">org_name</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">org_email</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">org_extra_contact_info</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
@@ -178,17 +197,45 @@
|
||||
<span class="n">envelope_to</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">dkim_results</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_DKIMResult</span><span class="p">)</span>
|
||||
<span class="n">spf_results</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SPFResult</span><span class="p">)</span>
|
||||
<span class="n">np</span> <span class="o">=</span> <span class="n">Keyword</span><span class="p">()</span>
|
||||
<span class="n">testing</span> <span class="o">=</span> <span class="n">Keyword</span><span class="p">()</span>
|
||||
<span class="n">discovery_method</span> <span class="o">=</span> <span class="n">Keyword</span><span class="p">()</span>
|
||||
<span class="n">generator</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_policy_override</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">type_</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">comment</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">policy_overrides</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_PolicyOverride</span><span class="p">(</span><span class="nb">type</span><span class="o">=</span><span class="n">type_</span><span class="p">,</span> <span class="n">comment</span><span class="o">=</span><span class="n">comment</span><span class="p">))</span> <span class="c1"># pyright: ignore[reportCallIssue]</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_dkim_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">domain</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">selector</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">result</span><span class="p">:</span> <span class="n">_DKIMResult</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_dkim_result</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="p">,</span>
|
||||
<span class="n">domain</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
||||
<span class="n">selector</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
||||
<span class="n">result</span><span class="p">:</span> <span class="n">_DKIMResult</span><span class="p">,</span>
|
||||
<span class="n">human_result</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">dkim_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="n">_DKIMResult</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span> <span class="n">selector</span><span class="o">=</span><span class="n">selector</span><span class="p">,</span> <span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">)</span>
|
||||
<span class="n">_DKIMResult</span><span class="p">(</span>
|
||||
<span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span>
|
||||
<span class="n">selector</span><span class="o">=</span><span class="n">selector</span><span class="p">,</span>
|
||||
<span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">,</span>
|
||||
<span class="n">human_result</span><span class="o">=</span><span class="n">human_result</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span> <span class="c1"># pyright: ignore[reportCallIssue]</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_spf_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">domain</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">scope</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">result</span><span class="p">:</span> <span class="n">_SPFResult</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">spf_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_SPFResult</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span> <span class="n">scope</span><span class="o">=</span><span class="n">scope</span><span class="p">,</span> <span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">))</span> <span class="c1"># pyright: ignore[reportCallIssue]</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_spf_result</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="p">,</span>
|
||||
<span class="n">domain</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
||||
<span class="n">scope</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
||||
<span class="n">result</span><span class="p">:</span> <span class="n">_SPFResult</span><span class="p">,</span>
|
||||
<span class="n">human_result</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">spf_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="n">_SPFResult</span><span class="p">(</span>
|
||||
<span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span>
|
||||
<span class="n">scope</span><span class="o">=</span><span class="n">scope</span><span class="p">,</span>
|
||||
<span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">,</span>
|
||||
<span class="n">human_result</span><span class="o">=</span><span class="n">human_result</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span> <span class="c1"># pyright: ignore[reportCallIssue]</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">save</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span> <span class="c1"># pyright: ignore[reportIncompatibleMethodOverride]</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">passed_dmarc</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
@@ -208,7 +255,7 @@
|
||||
<span class="n">sha256</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
|
||||
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_ForensicSampleDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_FailureSampleDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="n">raw</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">headers</span> <span class="o">=</span> <span class="n">Object</span><span class="p">()</span>
|
||||
<span class="n">headers_only</span> <span class="o">=</span> <span class="n">Boolean</span><span class="p">()</span>
|
||||
@@ -245,9 +292,9 @@
|
||||
<span class="p">)</span> <span class="c1"># pyright: ignore[reportCallIssue]</span>
|
||||
|
||||
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_ForensicReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_FailureReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">Index</span><span class="p">:</span>
|
||||
<span class="n">name</span> <span class="o">=</span> <span class="s2">"dmarc_forensic"</span>
|
||||
<span class="n">name</span> <span class="o">=</span> <span class="s2">"dmarc_failure"</span>
|
||||
|
||||
<span class="n">feedback_type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">user_agent</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
@@ -268,7 +315,7 @@
|
||||
<span class="n">source_auth_failures</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">dkim_domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">original_rcpt_to</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">sample</span> <span class="o">=</span> <span class="n">Object</span><span class="p">(</span><span class="n">_ForensicSampleDoc</span><span class="p">)</span>
|
||||
<span class="n">sample</span> <span class="o">=</span> <span class="n">Object</span><span class="p">(</span><span class="n">_FailureSampleDoc</span><span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_SMTPTLSFailureDetailsDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
@@ -369,6 +416,7 @@
|
||||
<span class="n">password</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">api_key</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">timeout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">60.0</span><span class="p">,</span>
|
||||
<span class="n">serverless</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Sets the Elasticsearch hosts to use</span>
|
||||
@@ -382,7 +430,14 @@
|
||||
<span class="sd"> password (str): The password to use for authentication</span>
|
||||
<span class="sd"> api_key (str): The Base64 encoded API key to use for authentication</span>
|
||||
<span class="sd"> timeout (float): Timeout in seconds</span>
|
||||
<span class="sd"> serverless (bool): Target an Elastic Cloud Serverless project. When True,</span>
|
||||
<span class="sd"> ``create_indexes`` strips ``number_of_shards`` / ``number_of_replicas``</span>
|
||||
<span class="sd"> from its settings (which Serverless rejects with HTTP 400) and passes</span>
|
||||
<span class="sd"> any other settings through unchanged.</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="c1"># Module-global; see the _SERVERLESS comment at the top of the module.</span>
|
||||
<span class="k">global</span> <span class="n">_SERVERLESS</span>
|
||||
<span class="n">_SERVERLESS</span> <span class="o">=</span> <span class="n">serverless</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">hosts</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
|
||||
<span class="n">hosts</span> <span class="o">=</span> <span class="p">[</span><span class="n">hosts</span><span class="p">]</span>
|
||||
<span class="n">conn_params</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"hosts"</span><span class="p">:</span> <span class="n">hosts</span><span class="p">,</span> <span class="s2">"timeout"</span><span class="p">:</span> <span class="n">timeout</span><span class="p">}</span>
|
||||
@@ -410,18 +465,28 @@
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> names (list): A list of index names</span>
|
||||
<span class="sd"> settings (dict): Index settings</span>
|
||||
|
||||
<span class="sd"> settings (dict): Index settings. In Serverless mode, keys in</span>
|
||||
<span class="sd"> ``_SERVERLESS_REJECTED_SETTINGS`` are filtered out and the</span>
|
||||
<span class="sd"> remaining keys are passed through; defaults are skipped entirely.</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">if</span> <span class="n">settings</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">effective_settings</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span>
|
||||
<span class="p">{}</span> <span class="k">if</span> <span class="n">_SERVERLESS</span> <span class="k">else</span> <span class="p">{</span><span class="s2">"number_of_shards"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span> <span class="s2">"number_of_replicas"</span><span class="p">:</span> <span class="mi">0</span><span class="p">}</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">elif</span> <span class="n">_SERVERLESS</span><span class="p">:</span>
|
||||
<span class="n">effective_settings</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="n">k</span><span class="p">:</span> <span class="n">v</span> <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">settings</span><span class="o">.</span><span class="n">items</span><span class="p">()</span> <span class="k">if</span> <span class="n">k</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">_SERVERLESS_REJECTED_SETTINGS</span>
|
||||
<span class="p">}</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">effective_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="n">settings</span><span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">names</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="n">Index</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">index</span><span class="o">.</span><span class="n">exists</span><span class="p">():</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Creating Elasticsearch index: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">name</span><span class="p">))</span>
|
||||
<span class="k">if</span> <span class="n">settings</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">index</span><span class="o">.</span><span class="n">settings</span><span class="p">(</span><span class="n">number_of_shards</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">number_of_replicas</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">index</span><span class="o">.</span><span class="n">settings</span><span class="p">(</span><span class="o">**</span><span class="n">settings</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">effective_settings</span><span class="p">:</span>
|
||||
<span class="n">index</span><span class="o">.</span><span class="n">settings</span><span class="p">(</span><span class="o">**</span><span class="n">effective_settings</span><span class="p">)</span>
|
||||
<span class="n">index</span><span class="o">.</span><span class="n">create</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="s2">"Elasticsearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
@@ -432,20 +497,20 @@
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.migrate_indexes">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">migrate_indexes</span><span class="p">(</span>
|
||||
<span class="n">aggregate_indexes</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">forensic_indexes</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">failure_indexes</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Updates index mappings</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> aggregate_indexes (list): A list of aggregate index names</span>
|
||||
<span class="sd"> forensic_indexes (list): A list of forensic index names</span>
|
||||
<span class="sd"> failure_indexes (list): A list of failure index names</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">version</span> <span class="o">=</span> <span class="mi">2</span>
|
||||
<span class="k">if</span> <span class="n">aggregate_indexes</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">aggregate_indexes</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="k">if</span> <span class="n">forensic_indexes</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">forensic_indexes</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="k">if</span> <span class="n">failure_indexes</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">failure_indexes</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="k">for</span> <span class="n">aggregate_index_name</span> <span class="ow">in</span> <span class="n">aggregate_indexes</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">Index</span><span class="p">(</span><span class="n">aggregate_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">exists</span><span class="p">():</span>
|
||||
<span class="k">continue</span>
|
||||
@@ -475,7 +540,7 @@
|
||||
<span class="n">reindex</span><span class="p">(</span><span class="n">connections</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(),</span> <span class="n">aggregate_index_name</span><span class="p">,</span> <span class="n">new_index_name</span><span class="p">)</span> <span class="c1"># pyright: ignore[reportArgumentType]</span>
|
||||
<span class="n">Index</span><span class="p">(</span><span class="n">aggregate_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">delete</span><span class="p">()</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">forensic_index</span> <span class="ow">in</span> <span class="n">forensic_indexes</span><span class="p">:</span>
|
||||
<span class="k">for</span> <span class="n">failure_index</span> <span class="ow">in</span> <span class="n">failure_indexes</span><span class="p">:</span>
|
||||
<span class="k">pass</span></div>
|
||||
|
||||
|
||||
@@ -494,7 +559,7 @@
|
||||
<span class="sd"> Saves a parsed DMARC aggregate report to Elasticsearch</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> aggregate_report (dict): A parsed forensic report</span>
|
||||
<span class="sd"> aggregate_report (dict): A parsed aggregate report</span>
|
||||
<span class="sd"> index_suffix (str): The suffix of the name of the index to save to</span>
|
||||
<span class="sd"> index_prefix (str): The prefix of the name of the index to save to</span>
|
||||
<span class="sd"> monthly_indexes (bool): Use monthly indexes instead of daily indexes</span>
|
||||
@@ -562,6 +627,9 @@
|
||||
<span class="n">sp</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"sp"</span><span class="p">],</span>
|
||||
<span class="n">pct</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"pct"</span><span class="p">],</span>
|
||||
<span class="n">fo</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"fo"</span><span class="p">],</span>
|
||||
<span class="n">np</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"np"</span><span class="p">),</span>
|
||||
<span class="n">testing</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"testing"</span><span class="p">),</span>
|
||||
<span class="n">discovery_method</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"discovery_method"</span><span class="p">),</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">record</span> <span class="ow">in</span> <span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"records"</span><span class="p">]:</span>
|
||||
@@ -578,6 +646,7 @@
|
||||
<span class="n">date_range</span> <span class="o">=</span> <span class="p">[</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">],</span> <span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">]]</span>
|
||||
<span class="n">agg_doc</span> <span class="o">=</span> <span class="n">_AggregateReportDoc</span><span class="p">(</span>
|
||||
<span class="n">xml_schema</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"xml_schema"</span><span class="p">],</span>
|
||||
<span class="n">xml_namespace</span><span class="o">=</span><span class="n">aggregate_report</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"xml_namespace"</span><span class="p">),</span>
|
||||
<span class="n">org_name</span><span class="o">=</span><span class="n">metadata</span><span class="p">[</span><span class="s2">"org_name"</span><span class="p">],</span>
|
||||
<span class="n">org_email</span><span class="o">=</span><span class="n">metadata</span><span class="p">[</span><span class="s2">"org_email"</span><span class="p">],</span>
|
||||
<span class="n">org_extra_contact_info</span><span class="o">=</span><span class="n">metadata</span><span class="p">[</span><span class="s2">"org_extra_contact_info"</span><span class="p">],</span>
|
||||
@@ -606,6 +675,12 @@
|
||||
<span class="n">header_from</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"header_from"</span><span class="p">],</span>
|
||||
<span class="n">envelope_from</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"envelope_from"</span><span class="p">],</span>
|
||||
<span class="n">envelope_to</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"envelope_to"</span><span class="p">],</span>
|
||||
<span class="n">np</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"np"</span><span class="p">),</span>
|
||||
<span class="n">testing</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"testing"</span><span class="p">),</span>
|
||||
<span class="n">discovery_method</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
|
||||
<span class="s2">"discovery_method"</span>
|
||||
<span class="p">),</span>
|
||||
<span class="n">generator</span><span class="o">=</span><span class="n">metadata</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"generator"</span><span class="p">),</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">override</span> <span class="ow">in</span> <span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"policy_override_reasons"</span><span class="p">]:</span>
|
||||
@@ -618,6 +693,7 @@
|
||||
<span class="n">domain</span><span class="o">=</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">],</span>
|
||||
<span class="n">selector</span><span class="o">=</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">"selector"</span><span class="p">],</span>
|
||||
<span class="n">result</span><span class="o">=</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">],</span>
|
||||
<span class="n">human_result</span><span class="o">=</span><span class="n">dkim_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"human_result"</span><span class="p">),</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">spf_result</span> <span class="ow">in</span> <span class="n">record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">]:</span>
|
||||
@@ -625,6 +701,7 @@
|
||||
<span class="n">domain</span><span class="o">=</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">],</span>
|
||||
<span class="n">scope</span><span class="o">=</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">"scope"</span><span class="p">],</span>
|
||||
<span class="n">result</span><span class="o">=</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">],</span>
|
||||
<span class="n">human_result</span><span class="o">=</span><span class="n">spf_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"human_result"</span><span class="p">),</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"dmarc_aggregate"</span>
|
||||
@@ -647,10 +724,10 @@
|
||||
|
||||
|
||||
|
||||
<div class="viewcode-block" id="save_forensic_report_to_elasticsearch">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.save_forensic_report_to_elasticsearch">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">save_forensic_report_to_elasticsearch</span><span class="p">(</span>
|
||||
<span class="n">forensic_report</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">],</span>
|
||||
<div class="viewcode-block" id="save_failure_report_to_elasticsearch">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.save_failure_report_to_elasticsearch">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">save_failure_report_to_elasticsearch</span><span class="p">(</span>
|
||||
<span class="n">failure_report</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">],</span>
|
||||
<span class="n">index_suffix</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">index_prefix</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">monthly_indexes</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
||||
@@ -658,10 +735,10 @@
|
||||
<span class="n">number_of_replicas</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">0</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Saves a parsed DMARC forensic report to Elasticsearch</span>
|
||||
<span class="sd"> Saves a parsed DMARC failure report to Elasticsearch</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> forensic_report (dict): A parsed forensic report</span>
|
||||
<span class="sd"> failure_report (dict): A parsed failure report</span>
|
||||
<span class="sd"> index_suffix (str): The suffix of the name of the index to save to</span>
|
||||
<span class="sd"> index_prefix (str): The prefix of the name of the index to save to</span>
|
||||
<span class="sd"> monthly_indexes (bool): Use monthly indexes instead of daily</span>
|
||||
@@ -674,26 +751,28 @@
|
||||
<span class="sd"> AlreadySaved</span>
|
||||
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Saving forensic report to Elasticsearch"</span><span class="p">)</span>
|
||||
<span class="n">forensic_report</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Saving failure report to Elasticsearch"</span><span class="p">)</span>
|
||||
<span class="n">failure_report</span> <span class="o">=</span> <span class="n">failure_report</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
<span class="n">sample_date</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="k">if</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"date"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">sample_date</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"date"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"date"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">sample_date</span> <span class="o">=</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"date"</span><span class="p">]</span>
|
||||
<span class="n">sample_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">sample_date</span><span class="p">)</span>
|
||||
<span class="n">original_headers</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"headers"</span><span class="p">]</span>
|
||||
<span class="n">original_headers</span> <span class="o">=</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"headers"</span><span class="p">]</span>
|
||||
<span class="n">headers</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
|
||||
<span class="k">for</span> <span class="n">original_header</span> <span class="ow">in</span> <span class="n">original_headers</span><span class="p">:</span>
|
||||
<span class="n">headers</span><span class="p">[</span><span class="n">original_header</span><span class="o">.</span><span class="n">lower</span><span class="p">()]</span> <span class="o">=</span> <span class="n">original_headers</span><span class="p">[</span><span class="n">original_header</span><span class="p">]</span>
|
||||
|
||||
<span class="n">arrival_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"arrival_date_utc"</span><span class="p">])</span>
|
||||
<span class="n">arrival_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"arrival_date_utc"</span><span class="p">])</span>
|
||||
<span class="n">arrival_date_epoch_milliseconds</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">arrival_date</span><span class="o">.</span><span class="n">timestamp</span><span class="p">()</span> <span class="o">*</span> <span class="mi">1000</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">index_suffix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"dmarc_forensic_</span><span class="si">{0}</span><span class="s2">*"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"dmarc_failure_</span><span class="si">{0}</span><span class="s2">*,dmarc_forensic_</span><span class="si">{0}</span><span class="s2">*"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"dmarc_forensic*"</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"dmarc_failure*,dmarc_forensic*"</span>
|
||||
<span class="k">if</span> <span class="n">index_prefix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">search_index</span><span class="p">)</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">","</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
|
||||
<span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">part</span><span class="p">)</span> <span class="k">for</span> <span class="n">part</span> <span class="ow">in</span> <span class="n">search_index</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">","</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="n">search_index</span><span class="p">)</span>
|
||||
<span class="n">q</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">arrival_date</span><span class="o">=</span><span class="n">arrival_date_epoch_milliseconds</span><span class="p">)))</span> <span class="c1"># pyright: ignore[reportArgumentType]</span>
|
||||
|
||||
@@ -734,67 +813,67 @@
|
||||
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">existing</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">AlreadySaved</span><span class="p">(</span>
|
||||
<span class="s2">"A forensic sample to </span><span class="si">{0}</span><span class="s2"> from </span><span class="si">{1}</span><span class="s2"> "</span>
|
||||
<span class="s2">"A failure sample to </span><span class="si">{0}</span><span class="s2"> from </span><span class="si">{1}</span><span class="s2"> "</span>
|
||||
<span class="s2">"with a subject of </span><span class="si">{2}</span><span class="s2"> and arrival date of </span><span class="si">{3}</span><span class="s2"> "</span>
|
||||
<span class="s2">"already exists in "</span>
|
||||
<span class="s2">"Elasticsearch"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">to_</span><span class="p">,</span> <span class="n">from_</span><span class="p">,</span> <span class="n">subject</span><span class="p">,</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"arrival_date_utc"</span><span class="p">]</span>
|
||||
<span class="n">to_</span><span class="p">,</span> <span class="n">from_</span><span class="p">,</span> <span class="n">subject</span><span class="p">,</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"arrival_date_utc"</span><span class="p">]</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">parsed_sample</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">]</span>
|
||||
<span class="n">sample</span> <span class="o">=</span> <span class="n">_ForensicSampleDoc</span><span class="p">(</span>
|
||||
<span class="n">raw</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"sample"</span><span class="p">],</span>
|
||||
<span class="n">parsed_sample</span> <span class="o">=</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">]</span>
|
||||
<span class="n">sample</span> <span class="o">=</span> <span class="n">_FailureSampleDoc</span><span class="p">(</span>
|
||||
<span class="n">raw</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"sample"</span><span class="p">],</span>
|
||||
<span class="n">headers</span><span class="o">=</span><span class="n">headers</span><span class="p">,</span>
|
||||
<span class="n">headers_only</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"sample_headers_only"</span><span class="p">],</span>
|
||||
<span class="n">headers_only</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"sample_headers_only"</span><span class="p">],</span>
|
||||
<span class="n">date</span><span class="o">=</span><span class="n">sample_date</span><span class="p">,</span>
|
||||
<span class="n">subject</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"subject"</span><span class="p">],</span>
|
||||
<span class="n">subject</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"subject"</span><span class="p">],</span>
|
||||
<span class="n">filename_safe_subject</span><span class="o">=</span><span class="n">parsed_sample</span><span class="p">[</span><span class="s2">"filename_safe_subject"</span><span class="p">],</span>
|
||||
<span class="n">body</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"body"</span><span class="p">],</span>
|
||||
<span class="n">body</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"body"</span><span class="p">],</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"to"</span><span class="p">]:</span>
|
||||
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"to"</span><span class="p">]:</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_to</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">])</span>
|
||||
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"reply_to"</span><span class="p">]:</span>
|
||||
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"reply_to"</span><span class="p">]:</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_reply_to</span><span class="p">(</span>
|
||||
<span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">]</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"cc"</span><span class="p">]:</span>
|
||||
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"cc"</span><span class="p">]:</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_cc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">])</span>
|
||||
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"bcc"</span><span class="p">]:</span>
|
||||
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"bcc"</span><span class="p">]:</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_bcc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">])</span>
|
||||
<span class="k">for</span> <span class="n">attachment</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"attachments"</span><span class="p">]:</span>
|
||||
<span class="k">for</span> <span class="n">attachment</span> <span class="ow">in</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"attachments"</span><span class="p">]:</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_attachment</span><span class="p">(</span>
|
||||
<span class="n">filename</span><span class="o">=</span><span class="n">attachment</span><span class="p">[</span><span class="s2">"filename"</span><span class="p">],</span>
|
||||
<span class="n">content_type</span><span class="o">=</span><span class="n">attachment</span><span class="p">[</span><span class="s2">"mail_content_type"</span><span class="p">],</span>
|
||||
<span class="n">sha256</span><span class="o">=</span><span class="n">attachment</span><span class="p">[</span><span class="s2">"sha256"</span><span class="p">],</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">forensic_doc</span> <span class="o">=</span> <span class="n">_ForensicReportDoc</span><span class="p">(</span>
|
||||
<span class="n">feedback_type</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"feedback_type"</span><span class="p">],</span>
|
||||
<span class="n">user_agent</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"user_agent"</span><span class="p">],</span>
|
||||
<span class="n">version</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"version"</span><span class="p">],</span>
|
||||
<span class="n">original_mail_from</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"original_mail_from"</span><span class="p">],</span>
|
||||
<span class="n">failure_doc</span> <span class="o">=</span> <span class="n">_FailureReportDoc</span><span class="p">(</span>
|
||||
<span class="n">feedback_type</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"feedback_type"</span><span class="p">],</span>
|
||||
<span class="n">user_agent</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"user_agent"</span><span class="p">],</span>
|
||||
<span class="n">version</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"version"</span><span class="p">],</span>
|
||||
<span class="n">original_mail_from</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"original_mail_from"</span><span class="p">],</span>
|
||||
<span class="n">arrival_date</span><span class="o">=</span><span class="n">arrival_date_epoch_milliseconds</span><span class="p">,</span>
|
||||
<span class="n">domain</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"reported_domain"</span><span class="p">],</span>
|
||||
<span class="n">original_envelope_id</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"original_envelope_id"</span><span class="p">],</span>
|
||||
<span class="n">authentication_results</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"authentication_results"</span><span class="p">],</span>
|
||||
<span class="n">delivery_results</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"delivery_result"</span><span class="p">],</span>
|
||||
<span class="n">source_ip_address</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"ip_address"</span><span class="p">],</span>
|
||||
<span class="n">source_country</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"country"</span><span class="p">],</span>
|
||||
<span class="n">source_reverse_dns</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"reverse_dns"</span><span class="p">],</span>
|
||||
<span class="n">source_base_domain</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"base_domain"</span><span class="p">],</span>
|
||||
<span class="n">source_asn</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"asn"</span><span class="p">],</span>
|
||||
<span class="n">source_as_name</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"as_name"</span><span class="p">],</span>
|
||||
<span class="n">source_as_domain</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"as_domain"</span><span class="p">],</span>
|
||||
<span class="n">authentication_mechanisms</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"authentication_mechanisms"</span><span class="p">],</span>
|
||||
<span class="n">auth_failure</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"auth_failure"</span><span class="p">],</span>
|
||||
<span class="n">dkim_domain</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"dkim_domain"</span><span class="p">],</span>
|
||||
<span class="n">original_rcpt_to</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"original_rcpt_to"</span><span class="p">],</span>
|
||||
<span class="n">domain</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"reported_domain"</span><span class="p">],</span>
|
||||
<span class="n">original_envelope_id</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"original_envelope_id"</span><span class="p">],</span>
|
||||
<span class="n">authentication_results</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"authentication_results"</span><span class="p">],</span>
|
||||
<span class="n">delivery_results</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"delivery_result"</span><span class="p">],</span>
|
||||
<span class="n">source_ip_address</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"ip_address"</span><span class="p">],</span>
|
||||
<span class="n">source_country</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"country"</span><span class="p">],</span>
|
||||
<span class="n">source_reverse_dns</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"reverse_dns"</span><span class="p">],</span>
|
||||
<span class="n">source_base_domain</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"base_domain"</span><span class="p">],</span>
|
||||
<span class="n">source_asn</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"asn"</span><span class="p">],</span>
|
||||
<span class="n">source_as_name</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"as_name"</span><span class="p">],</span>
|
||||
<span class="n">source_as_domain</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"as_domain"</span><span class="p">],</span>
|
||||
<span class="n">authentication_mechanisms</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"authentication_mechanisms"</span><span class="p">],</span>
|
||||
<span class="n">auth_failure</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"auth_failure"</span><span class="p">],</span>
|
||||
<span class="n">dkim_domain</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"dkim_domain"</span><span class="p">],</span>
|
||||
<span class="n">original_rcpt_to</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"original_rcpt_to"</span><span class="p">],</span>
|
||||
<span class="n">sample</span><span class="o">=</span><span class="n">sample</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"dmarc_forensic"</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"dmarc_failure"</span>
|
||||
<span class="k">if</span> <span class="n">index_suffix</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">_</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">index_prefix</span><span class="p">:</span>
|
||||
@@ -808,14 +887,14 @@
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span> <span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">create_indexes</span><span class="p">([</span><span class="n">index</span><span class="p">],</span> <span class="n">index_settings</span><span class="p">)</span>
|
||||
<span class="n">forensic_doc</span><span class="o">.</span><span class="n">meta</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">index</span> <span class="c1"># pyright: ignore[reportAttributeAccessIssue, reportOptionalMemberAccess]</span>
|
||||
<span class="n">failure_doc</span><span class="o">.</span><span class="n">meta</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">index</span> <span class="c1"># pyright: ignore[reportAttributeAccessIssue, reportOptionalMemberAccess]</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">forensic_doc</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
|
||||
<span class="n">failure_doc</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="s2">"Elasticsearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
||||
<span class="k">except</span> <span class="ne">KeyError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidForensicReport</span><span class="p">(</span>
|
||||
<span class="s2">"Forensic report missing required field: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidFailureReport</span><span class="p">(</span>
|
||||
<span class="s2">"Failure report missing required field: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="p">)</span></div>
|
||||
|
||||
|
||||
@@ -973,6 +1052,12 @@
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="s2">"Elasticsearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
|
||||
|
||||
|
||||
<span class="c1"># Backward-compatible aliases</span>
|
||||
<span class="n">_ForensicSampleDoc</span> <span class="o">=</span> <span class="n">_FailureSampleDoc</span>
|
||||
<span class="n">_ForensicReportDoc</span> <span class="o">=</span> <span class="n">_FailureReportDoc</span>
|
||||
<span class="n">save_forensic_report_to_elasticsearch</span> <span class="o">=</span> <span class="n">save_failure_report_to_elasticsearch</span>
|
||||
</pre></div>
|
||||
|
||||
</div>
|
||||
|
||||
@@ -5,14 +5,14 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc.opensearch — parsedmarc 9.11.2 documentation</title>
|
||||
<title>parsedmarc.opensearch — parsedmarc 10.0.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<script src="../../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../../_static/documentation_options.js?v=de4344a5"></script>
|
||||
<script src="../../_static/documentation_options.js?v=335988e4"></script>
|
||||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="../../_static/js/theme.js"></script>
|
||||
@@ -96,6 +96,7 @@
|
||||
<span class="n">InnerDoc</span><span class="p">,</span>
|
||||
<span class="n">Integer</span><span class="p">,</span>
|
||||
<span class="n">Ip</span><span class="p">,</span>
|
||||
<span class="n">Keyword</span><span class="p">,</span>
|
||||
<span class="n">Nested</span><span class="p">,</span>
|
||||
<span class="n">Object</span><span class="p">,</span>
|
||||
<span class="n">Q</span><span class="p">,</span>
|
||||
@@ -106,7 +107,7 @@
|
||||
<span class="p">)</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">opensearchpy.helpers</span><span class="w"> </span><span class="kn">import</span> <span class="n">reindex</span>
|
||||
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc</span><span class="w"> </span><span class="kn">import</span> <span class="n">InvalidForensicReport</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc</span><span class="w"> </span><span class="kn">import</span> <span class="n">InvalidFailureReport</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.log</span><span class="w"> </span><span class="kn">import</span> <span class="n">logger</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">human_timestamp_to_datetime</span>
|
||||
|
||||
@@ -131,18 +132,23 @@
|
||||
<span class="n">sp</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">pct</span> <span class="o">=</span> <span class="n">Integer</span><span class="p">()</span>
|
||||
<span class="n">fo</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">np</span> <span class="o">=</span> <span class="n">Keyword</span><span class="p">()</span>
|
||||
<span class="n">testing</span> <span class="o">=</span> <span class="n">Keyword</span><span class="p">()</span>
|
||||
<span class="n">discovery_method</span> <span class="o">=</span> <span class="n">Keyword</span><span class="p">()</span>
|
||||
|
||||
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_DKIMResult</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="n">domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">selector</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">result</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">human_result</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
|
||||
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_SPFResult</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="n">domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">scope</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">results</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">human_result</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
|
||||
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_AggregateReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
@@ -150,6 +156,7 @@
|
||||
<span class="n">name</span> <span class="o">=</span> <span class="s2">"dmarc_aggregate"</span>
|
||||
|
||||
<span class="n">xml_schema</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">xml_namespace</span> <span class="o">=</span> <span class="n">Keyword</span><span class="p">()</span>
|
||||
<span class="n">org_name</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">org_email</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">org_extra_contact_info</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
@@ -181,17 +188,45 @@
|
||||
<span class="n">envelope_to</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">dkim_results</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_DKIMResult</span><span class="p">)</span>
|
||||
<span class="n">spf_results</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SPFResult</span><span class="p">)</span>
|
||||
<span class="n">np</span> <span class="o">=</span> <span class="n">Keyword</span><span class="p">()</span>
|
||||
<span class="n">testing</span> <span class="o">=</span> <span class="n">Keyword</span><span class="p">()</span>
|
||||
<span class="n">discovery_method</span> <span class="o">=</span> <span class="n">Keyword</span><span class="p">()</span>
|
||||
<span class="n">generator</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_policy_override</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">type_</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">comment</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">policy_overrides</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_PolicyOverride</span><span class="p">(</span><span class="nb">type</span><span class="o">=</span><span class="n">type_</span><span class="p">,</span> <span class="n">comment</span><span class="o">=</span><span class="n">comment</span><span class="p">))</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_dkim_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">domain</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">selector</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">result</span><span class="p">:</span> <span class="n">_DKIMResult</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_dkim_result</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="p">,</span>
|
||||
<span class="n">domain</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
||||
<span class="n">selector</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
||||
<span class="n">result</span><span class="p">:</span> <span class="n">_DKIMResult</span><span class="p">,</span>
|
||||
<span class="n">human_result</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">dkim_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="n">_DKIMResult</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span> <span class="n">selector</span><span class="o">=</span><span class="n">selector</span><span class="p">,</span> <span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">)</span>
|
||||
<span class="n">_DKIMResult</span><span class="p">(</span>
|
||||
<span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span>
|
||||
<span class="n">selector</span><span class="o">=</span><span class="n">selector</span><span class="p">,</span>
|
||||
<span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">,</span>
|
||||
<span class="n">human_result</span><span class="o">=</span><span class="n">human_result</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_spf_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">domain</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">scope</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">result</span><span class="p">:</span> <span class="n">_SPFResult</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">spf_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_SPFResult</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span> <span class="n">scope</span><span class="o">=</span><span class="n">scope</span><span class="p">,</span> <span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">))</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_spf_result</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="p">,</span>
|
||||
<span class="n">domain</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
||||
<span class="n">scope</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
||||
<span class="n">result</span><span class="p">:</span> <span class="n">_SPFResult</span><span class="p">,</span>
|
||||
<span class="n">human_result</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">spf_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="n">_SPFResult</span><span class="p">(</span>
|
||||
<span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span>
|
||||
<span class="n">scope</span><span class="o">=</span><span class="n">scope</span><span class="p">,</span>
|
||||
<span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">,</span>
|
||||
<span class="n">human_result</span><span class="o">=</span><span class="n">human_result</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">save</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span> <span class="c1"># pyright: ignore[reportIncompatibleMethodOverride]</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">passed_dmarc</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
@@ -211,7 +246,7 @@
|
||||
<span class="n">sha256</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
|
||||
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_ForensicSampleDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_FailureSampleDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="n">raw</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">headers</span> <span class="o">=</span> <span class="n">Object</span><span class="p">()</span>
|
||||
<span class="n">headers_only</span> <span class="o">=</span> <span class="n">Boolean</span><span class="p">()</span>
|
||||
@@ -248,9 +283,9 @@
|
||||
<span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_ForensicReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_FailureReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">Index</span><span class="p">:</span>
|
||||
<span class="n">name</span> <span class="o">=</span> <span class="s2">"dmarc_forensic"</span>
|
||||
<span class="n">name</span> <span class="o">=</span> <span class="s2">"dmarc_failure"</span>
|
||||
|
||||
<span class="n">feedback_type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">user_agent</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
@@ -271,7 +306,7 @@
|
||||
<span class="n">source_auth_failures</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">dkim_domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">original_rcpt_to</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">sample</span> <span class="o">=</span> <span class="n">Object</span><span class="p">(</span><span class="n">_ForensicSampleDoc</span><span class="p">)</span>
|
||||
<span class="n">sample</span> <span class="o">=</span> <span class="n">Object</span><span class="p">(</span><span class="n">_FailureSampleDoc</span><span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_SMTPTLSFailureDetailsDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
@@ -462,20 +497,20 @@
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.migrate_indexes">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">migrate_indexes</span><span class="p">(</span>
|
||||
<span class="n">aggregate_indexes</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">forensic_indexes</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">failure_indexes</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Updates index mappings</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> aggregate_indexes (list): A list of aggregate index names</span>
|
||||
<span class="sd"> forensic_indexes (list): A list of forensic index names</span>
|
||||
<span class="sd"> failure_indexes (list): A list of failure index names</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">version</span> <span class="o">=</span> <span class="mi">2</span>
|
||||
<span class="k">if</span> <span class="n">aggregate_indexes</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">aggregate_indexes</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="k">if</span> <span class="n">forensic_indexes</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">forensic_indexes</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="k">if</span> <span class="n">failure_indexes</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">failure_indexes</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="k">for</span> <span class="n">aggregate_index_name</span> <span class="ow">in</span> <span class="n">aggregate_indexes</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">Index</span><span class="p">(</span><span class="n">aggregate_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">exists</span><span class="p">():</span>
|
||||
<span class="k">continue</span>
|
||||
@@ -505,7 +540,7 @@
|
||||
<span class="n">reindex</span><span class="p">(</span><span class="n">connections</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(),</span> <span class="n">aggregate_index_name</span><span class="p">,</span> <span class="n">new_index_name</span><span class="p">)</span>
|
||||
<span class="n">Index</span><span class="p">(</span><span class="n">aggregate_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">delete</span><span class="p">()</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">forensic_index</span> <span class="ow">in</span> <span class="n">forensic_indexes</span><span class="p">:</span>
|
||||
<span class="k">for</span> <span class="n">failure_index</span> <span class="ow">in</span> <span class="n">failure_indexes</span><span class="p">:</span>
|
||||
<span class="k">pass</span></div>
|
||||
|
||||
|
||||
@@ -524,7 +559,7 @@
|
||||
<span class="sd"> Saves a parsed DMARC aggregate report to OpenSearch</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> aggregate_report (dict): A parsed forensic report</span>
|
||||
<span class="sd"> aggregate_report (dict): A parsed aggregate report</span>
|
||||
<span class="sd"> index_suffix (str): The suffix of the name of the index to save to</span>
|
||||
<span class="sd"> index_prefix (str): The prefix of the name of the index to save to</span>
|
||||
<span class="sd"> monthly_indexes (bool): Use monthly indexes instead of daily indexes</span>
|
||||
@@ -592,6 +627,9 @@
|
||||
<span class="n">sp</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"sp"</span><span class="p">],</span>
|
||||
<span class="n">pct</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"pct"</span><span class="p">],</span>
|
||||
<span class="n">fo</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"fo"</span><span class="p">],</span>
|
||||
<span class="n">np</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"np"</span><span class="p">),</span>
|
||||
<span class="n">testing</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"testing"</span><span class="p">),</span>
|
||||
<span class="n">discovery_method</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"discovery_method"</span><span class="p">),</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">record</span> <span class="ow">in</span> <span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"records"</span><span class="p">]:</span>
|
||||
@@ -608,6 +646,7 @@
|
||||
<span class="n">date_range</span> <span class="o">=</span> <span class="p">[</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">],</span> <span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">]]</span>
|
||||
<span class="n">agg_doc</span> <span class="o">=</span> <span class="n">_AggregateReportDoc</span><span class="p">(</span>
|
||||
<span class="n">xml_schema</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"xml_schema"</span><span class="p">],</span>
|
||||
<span class="n">xml_namespace</span><span class="o">=</span><span class="n">aggregate_report</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"xml_namespace"</span><span class="p">),</span>
|
||||
<span class="n">org_name</span><span class="o">=</span><span class="n">metadata</span><span class="p">[</span><span class="s2">"org_name"</span><span class="p">],</span>
|
||||
<span class="n">org_email</span><span class="o">=</span><span class="n">metadata</span><span class="p">[</span><span class="s2">"org_email"</span><span class="p">],</span>
|
||||
<span class="n">org_extra_contact_info</span><span class="o">=</span><span class="n">metadata</span><span class="p">[</span><span class="s2">"org_extra_contact_info"</span><span class="p">],</span>
|
||||
@@ -636,6 +675,12 @@
|
||||
<span class="n">header_from</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"header_from"</span><span class="p">],</span>
|
||||
<span class="n">envelope_from</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"envelope_from"</span><span class="p">],</span>
|
||||
<span class="n">envelope_to</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"envelope_to"</span><span class="p">],</span>
|
||||
<span class="n">np</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"np"</span><span class="p">),</span>
|
||||
<span class="n">testing</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"testing"</span><span class="p">),</span>
|
||||
<span class="n">discovery_method</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
|
||||
<span class="s2">"discovery_method"</span>
|
||||
<span class="p">),</span>
|
||||
<span class="n">generator</span><span class="o">=</span><span class="n">metadata</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"generator"</span><span class="p">),</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">override</span> <span class="ow">in</span> <span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"policy_override_reasons"</span><span class="p">]:</span>
|
||||
@@ -648,6 +693,7 @@
|
||||
<span class="n">domain</span><span class="o">=</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">],</span>
|
||||
<span class="n">selector</span><span class="o">=</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">"selector"</span><span class="p">],</span>
|
||||
<span class="n">result</span><span class="o">=</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">],</span>
|
||||
<span class="n">human_result</span><span class="o">=</span><span class="n">dkim_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"human_result"</span><span class="p">),</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">spf_result</span> <span class="ow">in</span> <span class="n">record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">]:</span>
|
||||
@@ -655,6 +701,7 @@
|
||||
<span class="n">domain</span><span class="o">=</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">],</span>
|
||||
<span class="n">scope</span><span class="o">=</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">"scope"</span><span class="p">],</span>
|
||||
<span class="n">result</span><span class="o">=</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">],</span>
|
||||
<span class="n">human_result</span><span class="o">=</span><span class="n">spf_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"human_result"</span><span class="p">),</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"dmarc_aggregate"</span>
|
||||
@@ -677,10 +724,10 @@
|
||||
|
||||
|
||||
|
||||
<div class="viewcode-block" id="save_forensic_report_to_opensearch">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.save_forensic_report_to_opensearch">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">save_forensic_report_to_opensearch</span><span class="p">(</span>
|
||||
<span class="n">forensic_report</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">],</span>
|
||||
<div class="viewcode-block" id="save_failure_report_to_opensearch">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.save_failure_report_to_opensearch">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">save_failure_report_to_opensearch</span><span class="p">(</span>
|
||||
<span class="n">failure_report</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">],</span>
|
||||
<span class="n">index_suffix</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">index_prefix</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">monthly_indexes</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
||||
@@ -688,10 +735,10 @@
|
||||
<span class="n">number_of_replicas</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">0</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Saves a parsed DMARC forensic report to OpenSearch</span>
|
||||
<span class="sd"> Saves a parsed DMARC failure report to OpenSearch</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> forensic_report (dict): A parsed forensic report</span>
|
||||
<span class="sd"> failure_report (dict): A parsed failure report</span>
|
||||
<span class="sd"> index_suffix (str): The suffix of the name of the index to save to</span>
|
||||
<span class="sd"> index_prefix (str): The prefix of the name of the index to save to</span>
|
||||
<span class="sd"> monthly_indexes (bool): Use monthly indexes instead of daily</span>
|
||||
@@ -704,26 +751,28 @@
|
||||
<span class="sd"> AlreadySaved</span>
|
||||
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Saving forensic report to OpenSearch"</span><span class="p">)</span>
|
||||
<span class="n">forensic_report</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Saving failure report to OpenSearch"</span><span class="p">)</span>
|
||||
<span class="n">failure_report</span> <span class="o">=</span> <span class="n">failure_report</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
<span class="n">sample_date</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="k">if</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"date"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">sample_date</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"date"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"date"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">sample_date</span> <span class="o">=</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"date"</span><span class="p">]</span>
|
||||
<span class="n">sample_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">sample_date</span><span class="p">)</span>
|
||||
<span class="n">original_headers</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"headers"</span><span class="p">]</span>
|
||||
<span class="n">original_headers</span> <span class="o">=</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"headers"</span><span class="p">]</span>
|
||||
<span class="n">headers</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
|
||||
<span class="k">for</span> <span class="n">original_header</span> <span class="ow">in</span> <span class="n">original_headers</span><span class="p">:</span>
|
||||
<span class="n">headers</span><span class="p">[</span><span class="n">original_header</span><span class="o">.</span><span class="n">lower</span><span class="p">()]</span> <span class="o">=</span> <span class="n">original_headers</span><span class="p">[</span><span class="n">original_header</span><span class="p">]</span>
|
||||
|
||||
<span class="n">arrival_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"arrival_date_utc"</span><span class="p">])</span>
|
||||
<span class="n">arrival_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"arrival_date_utc"</span><span class="p">])</span>
|
||||
<span class="n">arrival_date_epoch_milliseconds</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">arrival_date</span><span class="o">.</span><span class="n">timestamp</span><span class="p">()</span> <span class="o">*</span> <span class="mi">1000</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">index_suffix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"dmarc_forensic_</span><span class="si">{0}</span><span class="s2">*"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"dmarc_failure_</span><span class="si">{0}</span><span class="s2">*,dmarc_forensic_</span><span class="si">{0}</span><span class="s2">*"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"dmarc_forensic*"</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"dmarc_failure*,dmarc_forensic*"</span>
|
||||
<span class="k">if</span> <span class="n">index_prefix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">search_index</span><span class="p">)</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">","</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
|
||||
<span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">part</span><span class="p">)</span> <span class="k">for</span> <span class="n">part</span> <span class="ow">in</span> <span class="n">search_index</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">","</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="n">search_index</span><span class="p">)</span>
|
||||
<span class="n">q</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">arrival_date</span><span class="o">=</span><span class="n">arrival_date_epoch_milliseconds</span><span class="p">)))</span>
|
||||
|
||||
@@ -764,67 +813,65 @@
|
||||
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">existing</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">AlreadySaved</span><span class="p">(</span>
|
||||
<span class="s2">"A forensic sample to </span><span class="si">{0}</span><span class="s2"> from </span><span class="si">{1}</span><span class="s2"> "</span>
|
||||
<span class="s2">"A failure sample to </span><span class="si">{0}</span><span class="s2"> from </span><span class="si">{1}</span><span class="s2"> "</span>
|
||||
<span class="s2">"with a subject of </span><span class="si">{2}</span><span class="s2"> and arrival date of </span><span class="si">{3}</span><span class="s2"> "</span>
|
||||
<span class="s2">"already exists in "</span>
|
||||
<span class="s2">"OpenSearch"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">to_</span><span class="p">,</span> <span class="n">from_</span><span class="p">,</span> <span class="n">subject</span><span class="p">,</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"arrival_date_utc"</span><span class="p">]</span>
|
||||
<span class="p">)</span>
|
||||
<span class="s2">"OpenSearch"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">to_</span><span class="p">,</span> <span class="n">from_</span><span class="p">,</span> <span class="n">subject</span><span class="p">,</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"arrival_date_utc"</span><span class="p">])</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">parsed_sample</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">]</span>
|
||||
<span class="n">sample</span> <span class="o">=</span> <span class="n">_ForensicSampleDoc</span><span class="p">(</span>
|
||||
<span class="n">raw</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"sample"</span><span class="p">],</span>
|
||||
<span class="n">parsed_sample</span> <span class="o">=</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">]</span>
|
||||
<span class="n">sample</span> <span class="o">=</span> <span class="n">_FailureSampleDoc</span><span class="p">(</span>
|
||||
<span class="n">raw</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"sample"</span><span class="p">],</span>
|
||||
<span class="n">headers</span><span class="o">=</span><span class="n">headers</span><span class="p">,</span>
|
||||
<span class="n">headers_only</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"sample_headers_only"</span><span class="p">],</span>
|
||||
<span class="n">headers_only</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"sample_headers_only"</span><span class="p">],</span>
|
||||
<span class="n">date</span><span class="o">=</span><span class="n">sample_date</span><span class="p">,</span>
|
||||
<span class="n">subject</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"subject"</span><span class="p">],</span>
|
||||
<span class="n">subject</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"subject"</span><span class="p">],</span>
|
||||
<span class="n">filename_safe_subject</span><span class="o">=</span><span class="n">parsed_sample</span><span class="p">[</span><span class="s2">"filename_safe_subject"</span><span class="p">],</span>
|
||||
<span class="n">body</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"body"</span><span class="p">],</span>
|
||||
<span class="n">body</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"body"</span><span class="p">],</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"to"</span><span class="p">]:</span>
|
||||
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"to"</span><span class="p">]:</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_to</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">])</span>
|
||||
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"reply_to"</span><span class="p">]:</span>
|
||||
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"reply_to"</span><span class="p">]:</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_reply_to</span><span class="p">(</span>
|
||||
<span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">]</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"cc"</span><span class="p">]:</span>
|
||||
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"cc"</span><span class="p">]:</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_cc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">])</span>
|
||||
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"bcc"</span><span class="p">]:</span>
|
||||
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"bcc"</span><span class="p">]:</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_bcc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">])</span>
|
||||
<span class="k">for</span> <span class="n">attachment</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"attachments"</span><span class="p">]:</span>
|
||||
<span class="k">for</span> <span class="n">attachment</span> <span class="ow">in</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"attachments"</span><span class="p">]:</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_attachment</span><span class="p">(</span>
|
||||
<span class="n">filename</span><span class="o">=</span><span class="n">attachment</span><span class="p">[</span><span class="s2">"filename"</span><span class="p">],</span>
|
||||
<span class="n">content_type</span><span class="o">=</span><span class="n">attachment</span><span class="p">[</span><span class="s2">"mail_content_type"</span><span class="p">],</span>
|
||||
<span class="n">sha256</span><span class="o">=</span><span class="n">attachment</span><span class="p">[</span><span class="s2">"sha256"</span><span class="p">],</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">forensic_doc</span> <span class="o">=</span> <span class="n">_ForensicReportDoc</span><span class="p">(</span>
|
||||
<span class="n">feedback_type</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"feedback_type"</span><span class="p">],</span>
|
||||
<span class="n">user_agent</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"user_agent"</span><span class="p">],</span>
|
||||
<span class="n">version</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"version"</span><span class="p">],</span>
|
||||
<span class="n">original_mail_from</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"original_mail_from"</span><span class="p">],</span>
|
||||
<span class="n">failure_doc</span> <span class="o">=</span> <span class="n">_FailureReportDoc</span><span class="p">(</span>
|
||||
<span class="n">feedback_type</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"feedback_type"</span><span class="p">],</span>
|
||||
<span class="n">user_agent</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"user_agent"</span><span class="p">],</span>
|
||||
<span class="n">version</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"version"</span><span class="p">],</span>
|
||||
<span class="n">original_mail_from</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"original_mail_from"</span><span class="p">],</span>
|
||||
<span class="n">arrival_date</span><span class="o">=</span><span class="n">arrival_date_epoch_milliseconds</span><span class="p">,</span>
|
||||
<span class="n">domain</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"reported_domain"</span><span class="p">],</span>
|
||||
<span class="n">original_envelope_id</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"original_envelope_id"</span><span class="p">],</span>
|
||||
<span class="n">authentication_results</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"authentication_results"</span><span class="p">],</span>
|
||||
<span class="n">delivery_results</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"delivery_result"</span><span class="p">],</span>
|
||||
<span class="n">source_ip_address</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"ip_address"</span><span class="p">],</span>
|
||||
<span class="n">source_country</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"country"</span><span class="p">],</span>
|
||||
<span class="n">source_reverse_dns</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"reverse_dns"</span><span class="p">],</span>
|
||||
<span class="n">source_base_domain</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"base_domain"</span><span class="p">],</span>
|
||||
<span class="n">source_asn</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"asn"</span><span class="p">],</span>
|
||||
<span class="n">source_as_name</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"as_name"</span><span class="p">],</span>
|
||||
<span class="n">source_as_domain</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"as_domain"</span><span class="p">],</span>
|
||||
<span class="n">authentication_mechanisms</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"authentication_mechanisms"</span><span class="p">],</span>
|
||||
<span class="n">auth_failure</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"auth_failure"</span><span class="p">],</span>
|
||||
<span class="n">dkim_domain</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"dkim_domain"</span><span class="p">],</span>
|
||||
<span class="n">original_rcpt_to</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"original_rcpt_to"</span><span class="p">],</span>
|
||||
<span class="n">domain</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"reported_domain"</span><span class="p">],</span>
|
||||
<span class="n">original_envelope_id</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"original_envelope_id"</span><span class="p">],</span>
|
||||
<span class="n">authentication_results</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"authentication_results"</span><span class="p">],</span>
|
||||
<span class="n">delivery_results</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"delivery_result"</span><span class="p">],</span>
|
||||
<span class="n">source_ip_address</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"ip_address"</span><span class="p">],</span>
|
||||
<span class="n">source_country</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"country"</span><span class="p">],</span>
|
||||
<span class="n">source_reverse_dns</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"reverse_dns"</span><span class="p">],</span>
|
||||
<span class="n">source_base_domain</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"base_domain"</span><span class="p">],</span>
|
||||
<span class="n">source_asn</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"asn"</span><span class="p">],</span>
|
||||
<span class="n">source_as_name</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"as_name"</span><span class="p">],</span>
|
||||
<span class="n">source_as_domain</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"as_domain"</span><span class="p">],</span>
|
||||
<span class="n">authentication_mechanisms</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"authentication_mechanisms"</span><span class="p">],</span>
|
||||
<span class="n">auth_failure</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"auth_failure"</span><span class="p">],</span>
|
||||
<span class="n">dkim_domain</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"dkim_domain"</span><span class="p">],</span>
|
||||
<span class="n">original_rcpt_to</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"original_rcpt_to"</span><span class="p">],</span>
|
||||
<span class="n">sample</span><span class="o">=</span><span class="n">sample</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"dmarc_forensic"</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"dmarc_failure"</span>
|
||||
<span class="k">if</span> <span class="n">index_suffix</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">_</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">index_prefix</span><span class="p">:</span>
|
||||
@@ -838,14 +885,14 @@
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span> <span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">create_indexes</span><span class="p">([</span><span class="n">index</span><span class="p">],</span> <span class="n">index_settings</span><span class="p">)</span>
|
||||
<span class="n">forensic_doc</span><span class="o">.</span><span class="n">meta</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">index</span>
|
||||
<span class="n">failure_doc</span><span class="o">.</span><span class="n">meta</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">index</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">forensic_doc</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
|
||||
<span class="n">failure_doc</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span><span class="s2">"OpenSearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
||||
<span class="k">except</span> <span class="ne">KeyError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidForensicReport</span><span class="p">(</span>
|
||||
<span class="s2">"Forensic report missing required field: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidFailureReport</span><span class="p">(</span>
|
||||
<span class="s2">"Failure report missing required field: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="p">)</span></div>
|
||||
|
||||
|
||||
@@ -1003,6 +1050,12 @@
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span><span class="s2">"OpenSearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
|
||||
|
||||
|
||||
<span class="c1"># Backward-compatible aliases</span>
|
||||
<span class="n">_ForensicSampleDoc</span> <span class="o">=</span> <span class="n">_FailureSampleDoc</span>
|
||||
<span class="n">_ForensicReportDoc</span> <span class="o">=</span> <span class="n">_FailureReportDoc</span>
|
||||
<span class="n">save_forensic_report_to_opensearch</span> <span class="o">=</span> <span class="n">save_failure_report_to_opensearch</span>
|
||||
</pre></div>
|
||||
|
||||
</div>
|
||||
|
||||
@@ -5,14 +5,14 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc.splunk — parsedmarc 9.11.2 documentation</title>
|
||||
<title>parsedmarc.splunk — parsedmarc 10.0.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<script src="../../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../../_static/documentation_options.js?v=de4344a5"></script>
|
||||
<script src="../../_static/documentation_options.js?v=335988e4"></script>
|
||||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="../../_static/js/theme.js"></script>
|
||||
@@ -229,30 +229,30 @@
|
||||
<span class="k">raise</span> <span class="n">SplunkError</span><span class="p">(</span><span class="n">response</span><span class="p">[</span><span class="s2">"text"</span><span class="p">])</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="HECClient.save_forensic_reports_to_splunk">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.splunk.HECClient.save_forensic_reports_to_splunk">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">save_forensic_reports_to_splunk</span><span class="p">(</span>
|
||||
<div class="viewcode-block" id="HECClient.save_failure_reports_to_splunk">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.splunk.HECClient.save_failure_reports_to_splunk">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">save_failure_reports_to_splunk</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="p">,</span>
|
||||
<span class="n">forensic_reports</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">list</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]],</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]],</span>
|
||||
<span class="n">failure_reports</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">list</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]],</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]],</span>
|
||||
<span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Saves forensic DMARC reports to Splunk</span>
|
||||
<span class="sd"> Saves failure DMARC reports to Splunk</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> forensic_reports (list): A list of forensic report dictionaries</span>
|
||||
<span class="sd"> failure_reports (list): A list of failure report dictionaries</span>
|
||||
<span class="sd"> to save in Splunk</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Saving forensic reports to Splunk"</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">forensic_reports</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
|
||||
<span class="n">forensic_reports</span> <span class="o">=</span> <span class="p">[</span><span class="n">forensic_reports</span><span class="p">]</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Saving failure reports to Splunk"</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">failure_reports</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
|
||||
<span class="n">failure_reports</span> <span class="o">=</span> <span class="p">[</span><span class="n">failure_reports</span><span class="p">]</span>
|
||||
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">forensic_reports</span><span class="p">)</span> <span class="o"><</span> <span class="mi">1</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">failure_reports</span><span class="p">)</span> <span class="o"><</span> <span class="mi">1</span><span class="p">:</span>
|
||||
<span class="k">return</span>
|
||||
|
||||
<span class="n">json_str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||
<span class="k">for</span> <span class="n">report</span> <span class="ow">in</span> <span class="n">forensic_reports</span><span class="p">:</span>
|
||||
<span class="k">for</span> <span class="n">report</span> <span class="ow">in</span> <span class="n">failure_reports</span><span class="p">:</span>
|
||||
<span class="n">data</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_common_data</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
<span class="n">data</span><span class="p">[</span><span class="s2">"sourcetype"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"dmarc:forensic"</span>
|
||||
<span class="n">data</span><span class="p">[</span><span class="s2">"sourcetype"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"dmarc:failure"</span>
|
||||
<span class="n">timestamp</span> <span class="o">=</span> <span class="n">human_timestamp_to_unix_timestamp</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">"arrival_date_utc"</span><span class="p">])</span>
|
||||
<span class="n">data</span><span class="p">[</span><span class="s2">"time"</span><span class="p">]</span> <span class="o">=</span> <span class="n">timestamp</span>
|
||||
<span class="n">data</span><span class="p">[</span><span class="s2">"event"</span><span class="p">]</span> <span class="o">=</span> <span class="n">report</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
@@ -320,6 +320,10 @@
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">close</span><span class="p">()</span></div>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<span class="c1"># Backward-compatible aliases</span>
|
||||
<span class="n">HECClient</span><span class="o">.</span><span class="n">save_forensic_reports_to_splunk</span> <span class="o">=</span> <span class="n">HECClient</span><span class="o">.</span><span class="n">save_failure_reports_to_splunk</span>
|
||||
</pre></div>
|
||||
|
||||
</div>
|
||||
|
||||
@@ -5,14 +5,14 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc.types — parsedmarc 9.11.2 documentation</title>
|
||||
<title>parsedmarc.types — parsedmarc 10.0.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<script src="../../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../../_static/documentation_options.js?v=de4344a5"></script>
|
||||
<script src="../../_static/documentation_options.js?v=335988e4"></script>
|
||||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="../../_static/js/theme.js"></script>
|
||||
@@ -90,7 +90,7 @@
|
||||
<span class="c1"># For optional keys, use total=False TypedDicts.</span>
|
||||
|
||||
|
||||
<span class="n">ReportType</span> <span class="o">=</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">"aggregate"</span><span class="p">,</span> <span class="s2">"forensic"</span><span class="p">,</span> <span class="s2">"smtp_tls"</span><span class="p">]</span>
|
||||
<span class="n">ReportType</span> <span class="o">=</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">"aggregate"</span><span class="p">,</span> <span class="s2">"failure"</span><span class="p">,</span> <span class="s2">"smtp_tls"</span><span class="p">]</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="AggregateReportMetadata">
|
||||
@@ -104,7 +104,8 @@
|
||||
<span class="n">end_date</span><span class="p">:</span> <span class="nb">str</span>
|
||||
<span class="n">timespan_requires_normalization</span><span class="p">:</span> <span class="nb">bool</span>
|
||||
<span class="n">original_timespan_seconds</span><span class="p">:</span> <span class="nb">int</span>
|
||||
<span class="n">errors</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span></div>
|
||||
<span class="n">errors</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
|
||||
<span class="n">generator</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span></div>
|
||||
|
||||
|
||||
|
||||
@@ -116,8 +117,11 @@
|
||||
<span class="n">aspf</span><span class="p">:</span> <span class="nb">str</span>
|
||||
<span class="n">p</span><span class="p">:</span> <span class="nb">str</span>
|
||||
<span class="n">sp</span><span class="p">:</span> <span class="nb">str</span>
|
||||
<span class="n">pct</span><span class="p">:</span> <span class="nb">str</span>
|
||||
<span class="n">fo</span><span class="p">:</span> <span class="nb">str</span></div>
|
||||
<span class="n">pct</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
|
||||
<span class="n">fo</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
|
||||
<span class="n">np</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
|
||||
<span class="n">testing</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
|
||||
<span class="n">discovery_method</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span></div>
|
||||
|
||||
|
||||
|
||||
@@ -167,7 +171,8 @@
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">AggregateAuthResultDKIM</span><span class="p">(</span><span class="n">TypedDict</span><span class="p">):</span>
|
||||
<span class="n">domain</span><span class="p">:</span> <span class="nb">str</span>
|
||||
<span class="n">result</span><span class="p">:</span> <span class="nb">str</span>
|
||||
<span class="n">selector</span><span class="p">:</span> <span class="nb">str</span></div>
|
||||
<span class="n">selector</span><span class="p">:</span> <span class="nb">str</span>
|
||||
<span class="n">human_result</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span></div>
|
||||
|
||||
|
||||
|
||||
@@ -176,7 +181,8 @@
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">AggregateAuthResultSPF</span><span class="p">(</span><span class="n">TypedDict</span><span class="p">):</span>
|
||||
<span class="n">domain</span><span class="p">:</span> <span class="nb">str</span>
|
||||
<span class="n">result</span><span class="p">:</span> <span class="nb">str</span>
|
||||
<span class="n">scope</span><span class="p">:</span> <span class="nb">str</span></div>
|
||||
<span class="n">scope</span><span class="p">:</span> <span class="nb">str</span>
|
||||
<span class="n">human_result</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span></div>
|
||||
|
||||
|
||||
|
||||
@@ -217,6 +223,7 @@
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.types.AggregateReport">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">AggregateReport</span><span class="p">(</span><span class="n">TypedDict</span><span class="p">):</span>
|
||||
<span class="n">xml_schema</span><span class="p">:</span> <span class="nb">str</span>
|
||||
<span class="n">xml_namespace</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
|
||||
<span class="n">report_metadata</span><span class="p">:</span> <span class="n">AggregateReportMetadata</span>
|
||||
<span class="n">policy_published</span><span class="p">:</span> <span class="n">AggregatePolicyPublished</span>
|
||||
<span class="n">records</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">AggregateRecord</span><span class="p">]</span></div>
|
||||
@@ -246,7 +253,7 @@
|
||||
<span class="s2">"ParsedEmail"</span><span class="p">,</span>
|
||||
<span class="p">{</span>
|
||||
<span class="c1"># This is a lightly-specified version of mailsuite/mailparser JSON.</span>
|
||||
<span class="c1"># It focuses on the fields parsedmarc uses in forensic handling.</span>
|
||||
<span class="c1"># It focuses on the fields parsedmarc uses in failure report handling.</span>
|
||||
<span class="s2">"headers"</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">],</span>
|
||||
<span class="s2">"subject"</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span>
|
||||
<span class="s2">"filename_safe_subject"</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span>
|
||||
@@ -265,9 +272,9 @@
|
||||
<span class="p">)</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="ForensicReport">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.types.ForensicReport">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">ForensicReport</span><span class="p">(</span><span class="n">TypedDict</span><span class="p">):</span>
|
||||
<div class="viewcode-block" id="FailureReport">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.types.FailureReport">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">FailureReport</span><span class="p">(</span><span class="n">TypedDict</span><span class="p">):</span>
|
||||
<span class="n">feedback_type</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
|
||||
<span class="n">user_agent</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
|
||||
<span class="n">version</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
|
||||
@@ -289,6 +296,10 @@
|
||||
|
||||
|
||||
|
||||
<span class="c1"># Backward-compatible alias</span>
|
||||
<span class="n">ForensicReport</span> <span class="o">=</span> <span class="n">FailureReport</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="SMTPTLSFailureDetails">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.types.SMTPTLSFailureDetails">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">SMTPTLSFailureDetails</span><span class="p">(</span><span class="n">TypedDict</span><span class="p">):</span>
|
||||
@@ -349,14 +360,18 @@
|
||||
|
||||
|
||||
|
||||
<div class="viewcode-block" id="ForensicParsedReport">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.types.ForensicParsedReport">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">ForensicParsedReport</span><span class="p">(</span><span class="n">TypedDict</span><span class="p">):</span>
|
||||
<span class="n">report_type</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">"forensic"</span><span class="p">]</span>
|
||||
<span class="n">report</span><span class="p">:</span> <span class="n">ForensicReport</span></div>
|
||||
<div class="viewcode-block" id="FailureParsedReport">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.types.FailureParsedReport">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">FailureParsedReport</span><span class="p">(</span><span class="n">TypedDict</span><span class="p">):</span>
|
||||
<span class="n">report_type</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">"failure"</span><span class="p">]</span>
|
||||
<span class="n">report</span><span class="p">:</span> <span class="n">FailureReport</span></div>
|
||||
|
||||
|
||||
|
||||
<span class="c1"># Backward-compatible alias</span>
|
||||
<span class="n">ForensicParsedReport</span> <span class="o">=</span> <span class="n">FailureParsedReport</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="SMTPTLSParsedReport">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.types.SMTPTLSParsedReport">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">SMTPTLSParsedReport</span><span class="p">(</span><span class="n">TypedDict</span><span class="p">):</span>
|
||||
@@ -365,14 +380,14 @@
|
||||
|
||||
|
||||
|
||||
<span class="n">ParsedReport</span> <span class="o">=</span> <span class="n">Union</span><span class="p">[</span><span class="n">AggregateParsedReport</span><span class="p">,</span> <span class="n">ForensicParsedReport</span><span class="p">,</span> <span class="n">SMTPTLSParsedReport</span><span class="p">]</span>
|
||||
<span class="n">ParsedReport</span> <span class="o">=</span> <span class="n">Union</span><span class="p">[</span><span class="n">AggregateParsedReport</span><span class="p">,</span> <span class="n">FailureParsedReport</span><span class="p">,</span> <span class="n">SMTPTLSParsedReport</span><span class="p">]</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="ParsingResults">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.types.ParsingResults">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">ParsingResults</span><span class="p">(</span><span class="n">TypedDict</span><span class="p">):</span>
|
||||
<span class="n">aggregate_reports</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">AggregateReport</span><span class="p">]</span>
|
||||
<span class="n">forensic_reports</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">ForensicReport</span><span class="p">]</span>
|
||||
<span class="n">failure_reports</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">FailureReport</span><span class="p">]</span>
|
||||
<span class="n">smtp_tls_reports</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">SMTPTLSReport</span><span class="p">]</span></div>
|
||||
|
||||
</pre></div>
|
||||
|
||||
@@ -5,14 +5,14 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc.utils — parsedmarc 9.11.2 documentation</title>
|
||||
<title>parsedmarc.utils — parsedmarc 10.0.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<script src="../../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../../_static/documentation_options.js?v=de4344a5"></script>
|
||||
<script src="../../_static/documentation_options.js?v=335988e4"></script>
|
||||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="../../_static/js/theme.js"></script>
|
||||
@@ -139,7 +139,7 @@
|
||||
|
||||
<span class="n">parenthesis_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">"\s*\(.*\)\s*"</span><span class="p">)</span>
|
||||
|
||||
<span class="n">null_file</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">devnull</span><span class="p">,</span> <span class="s2">"w"</span><span class="p">)</span>
|
||||
<span class="n">null_file</span> <span class="o">=</span> <span class="n">subprocess</span><span class="o">.</span><span class="n">DEVNULL</span>
|
||||
<span class="n">mailparser_logger</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s2">"mailparser"</span><span class="p">)</span>
|
||||
<span class="n">mailparser_logger</span><span class="o">.</span><span class="n">setLevel</span><span class="p">(</span><span class="n">logging</span><span class="o">.</span><span class="n">CRITICAL</span><span class="p">)</span>
|
||||
<span class="n">psl</span> <span class="o">=</span> <span class="n">publicsuffixlist</span><span class="o">.</span><span class="n">PublicSuffixList</span><span class="p">()</span>
|
||||
|
||||
Reference in New Issue
Block a user