mirror of
https://github.com/domainaware/parsedmarc.git
synced 2026-05-22 11:55:24 +00:00
Update docs
This commit is contained in:
Sean Whalen
+2
-2
@@ -5,14 +5,14 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Overview: module code — parsedmarc 9.11.2 documentation</title>
|
||||
<title>Overview: module code — parsedmarc 10.0.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="../_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<script src="../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../_static/documentation_options.js?v=de4344a5"></script>
|
||||
<script src="../_static/documentation_options.js?v=335988e4"></script>
|
||||
<script src="../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="../_static/js/theme.js"></script>
|
||||
|
||||
+367
-129
@@ -5,14 +5,14 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc — parsedmarc 9.11.2 documentation</title>
|
||||
<title>parsedmarc — parsedmarc 10.0.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="../_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<script src="../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../_static/documentation_options.js?v=de4344a5"></script>
|
||||
<script src="../_static/documentation_options.js?v=335988e4"></script>
|
||||
<script src="../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="../_static/js/theme.js"></script>
|
||||
@@ -134,7 +134,8 @@
|
||||
<span class="p">)</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.types</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
|
||||
<span class="n">AggregateReport</span><span class="p">,</span>
|
||||
<span class="n">ForensicReport</span><span class="p">,</span>
|
||||
<span class="n">FailureReport</span><span class="p">,</span>
|
||||
<span class="n">ForensicReport</span> <span class="k">as</span> <span class="n">ForensicReport</span><span class="p">,</span>
|
||||
<span class="n">ParsedReport</span><span class="p">,</span>
|
||||
<span class="n">ParsingResults</span><span class="p">,</span>
|
||||
<span class="n">SMTPTLSReport</span><span class="p">,</span>
|
||||
@@ -155,10 +156,33 @@
|
||||
<span class="n">xml_header_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">"^<\?xml .*?>"</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">MULTILINE</span><span class="p">)</span>
|
||||
<span class="n">xml_schema_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">"</??xs:schema.*>"</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">MULTILINE</span><span class="p">)</span>
|
||||
<span class="n">text_report_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">"\s*([a-zA-Z\s]+):\s(.+)"</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">MULTILINE</span><span class="p">)</span>
|
||||
<span class="c1"># Captures the value of any xmlns (default or prefixed) declaration so the</span>
|
||||
<span class="c1"># RFC 9990 namespace can be detected before xmltodict drops it.</span>
|
||||
<span class="n">xml_namespace_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span>
|
||||
<span class="w"> </span><span class="sa">r</span><span class="sd">"""xmlns(?::[a-zA-Z_][\w.-]*)?\s*=\s*["']([^"']+)["']"""</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="c1"># The XML namespace assigned to DMARC aggregate reports by RFC 9990.</span>
|
||||
<span class="n">RFC_9990_NAMESPACE</span> <span class="o">=</span> <span class="s2">"urn:ietf:params:xml:ns:dmarc-2.0"</span>
|
||||
|
||||
<span class="c1"># PolicyOverrideType enumeration from RFC 9990. Compared to RFC 7489,</span>
|
||||
<span class="c1"># `policy_test_mode` was added (emitted when t=y suppresses enforcement)</span>
|
||||
<span class="c1"># and `forwarded` / `sampled_out` were removed.</span>
|
||||
<span class="n">RFC_9990_POLICY_OVERRIDE_TYPES</span> <span class="o">=</span> <span class="nb">frozenset</span><span class="p">(</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s2">"local_policy"</span><span class="p">,</span>
|
||||
<span class="s2">"mailing_list"</span><span class="p">,</span>
|
||||
<span class="s2">"other"</span><span class="p">,</span>
|
||||
<span class="s2">"policy_test_mode"</span><span class="p">,</span>
|
||||
<span class="s2">"trusted_forwarder"</span><span class="p">,</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">RFC_7489_REMOVED_POLICY_OVERRIDE_TYPES</span> <span class="o">=</span> <span class="nb">frozenset</span><span class="p">({</span><span class="s2">"forwarded"</span><span class="p">,</span> <span class="s2">"sampled_out"</span><span class="p">})</span>
|
||||
|
||||
<span class="n">MAGIC_ZIP</span> <span class="o">=</span> <span class="sa">b</span><span class="s2">"</span><span class="se">\x50\x4b\x03\x04</span><span class="s2">"</span>
|
||||
<span class="n">MAGIC_GZIP</span> <span class="o">=</span> <span class="sa">b</span><span class="s2">"</span><span class="se">\x1f\x8b</span><span class="s2">"</span>
|
||||
<span class="n">MAGIC_XML</span> <span class="o">=</span> <span class="sa">b</span><span class="s2">"</span><span class="se">\x3c\x3f\x78\x6d\x6c\x20</span><span class="s2">"</span>
|
||||
<span class="n">MAGIC_XML_TAG</span> <span class="o">=</span> <span class="sa">b</span><span class="s2">"</span><span class="se">\x3c</span><span class="s2">"</span> <span class="c1"># '<' - XML starting with an element tag (no declaration)</span>
|
||||
<span class="n">MAGIC_JSON</span> <span class="o">=</span> <span class="sa">b</span><span class="s2">"</span><span class="se">\7</span><span class="s2">b"</span>
|
||||
|
||||
<span class="n">EMAIL_SAMPLE_CONTENT_TYPES</span> <span class="o">=</span> <span class="p">(</span>
|
||||
@@ -205,13 +229,37 @@
|
||||
|
||||
|
||||
|
||||
<div class="viewcode-block" id="InvalidForensicReport">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.InvalidForensicReport">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">InvalidForensicReport</span><span class="p">(</span><span class="n">InvalidDMARCReport</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Raised when an invalid DMARC forensic report is encountered"""</span></div>
|
||||
<div class="viewcode-block" id="InvalidFailureReport">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.InvalidFailureReport">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">InvalidFailureReport</span><span class="p">(</span><span class="n">InvalidDMARCReport</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Raised when an invalid DMARC failure report is encountered"""</span></div>
|
||||
|
||||
|
||||
|
||||
<span class="c1"># Backward-compatible alias</span>
|
||||
<span class="n">InvalidForensicReport</span> <span class="o">=</span> <span class="n">InvalidFailureReport</span>
|
||||
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">_text</span><span class="p">(</span><span class="n">value</span><span class="p">:</span> <span class="n">Any</span><span class="p">)</span> <span class="o">-></span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
|
||||
<span class="w"> </span><span class="sd">"""Unwrap a possibly-langAttrString value parsed by xmltodict.</span>
|
||||
|
||||
<span class="sd"> RFC 9990 changed several aggregate-report elements (extra_contact_info,</span>
|
||||
<span class="sd"> error, comment, human_result) to type ``langAttrString`` — an</span>
|
||||
<span class="sd"> xs:simpleContent string with an optional ``lang`` attribute. When the</span>
|
||||
<span class="sd"> attribute is present, xmltodict parses the element as</span>
|
||||
<span class="sd"> ``{"#text": "...", "@lang": "en"}`` instead of a plain string. Returns</span>
|
||||
<span class="sd"> the text payload for both shapes, ``None`` for unset values, and leaves</span>
|
||||
<span class="sd"> other scalar shapes untouched so callers can preserve whatever the</span>
|
||||
<span class="sd"> reporter sent.</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">if</span> <span class="n">value</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="kc">None</span>
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
|
||||
<span class="n">text</span> <span class="o">=</span> <span class="n">value</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"#text"</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="kc">None</span> <span class="k">if</span> <span class="n">text</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="nb">str</span><span class="p">(</span><span class="n">text</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="n">value</span>
|
||||
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">_bucket_interval_by_day</span><span class="p">(</span>
|
||||
<span class="n">begin</span><span class="p">:</span> <span class="n">datetime</span><span class="p">,</span>
|
||||
<span class="n">end</span><span class="p">:</span> <span class="n">datetime</span><span class="p">,</span>
|
||||
@@ -404,6 +452,7 @@
|
||||
<span class="n">nameservers</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">dns_timeout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="n">DEFAULT_DNS_TIMEOUT</span><span class="p">,</span>
|
||||
<span class="n">dns_retries</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="n">DEFAULT_DNS_MAX_RETRIES</span><span class="p">,</span>
|
||||
<span class="n">is_rfc_9990</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
||||
<span class="p">)</span> <span class="o">-></span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]:</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Converts a record from a DMARC aggregate report into a more consistent</span>
|
||||
@@ -453,8 +502,6 @@
|
||||
<span class="p">}</span>
|
||||
<span class="k">if</span> <span class="s2">"disposition"</span> <span class="ow">in</span> <span class="n">policy_evaluated</span><span class="p">:</span>
|
||||
<span class="n">new_policy_evaluated</span><span class="p">[</span><span class="s2">"disposition"</span><span class="p">]</span> <span class="o">=</span> <span class="n">policy_evaluated</span><span class="p">[</span><span class="s2">"disposition"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="n">new_policy_evaluated</span><span class="p">[</span><span class="s2">"disposition"</span><span class="p">]</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s2">"pass"</span><span class="p">:</span>
|
||||
<span class="n">new_policy_evaluated</span><span class="p">[</span><span class="s2">"disposition"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"none"</span>
|
||||
<span class="k">if</span> <span class="s2">"dkim"</span> <span class="ow">in</span> <span class="n">policy_evaluated</span><span class="p">:</span>
|
||||
<span class="n">new_policy_evaluated</span><span class="p">[</span><span class="s2">"dkim"</span><span class="p">]</span> <span class="o">=</span> <span class="n">policy_evaluated</span><span class="p">[</span><span class="s2">"dkim"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="s2">"spf"</span> <span class="ow">in</span> <span class="n">policy_evaluated</span><span class="p">:</span>
|
||||
@@ -479,8 +526,27 @@
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">reasons</span> <span class="o">=</span> <span class="p">[</span><span class="n">policy_evaluated</span><span class="p">[</span><span class="s2">"reason"</span><span class="p">]]</span>
|
||||
<span class="k">for</span> <span class="n">reason</span> <span class="ow">in</span> <span class="n">reasons</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="s2">"comment"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">reason</span><span class="p">:</span>
|
||||
<span class="n">reason</span><span class="p">[</span><span class="s2">"comment"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="c1"># `comment` is langAttrString in RFC 9990 — unwrap {"#text": ..., "@lang": ...}</span>
|
||||
<span class="n">reason</span><span class="p">[</span><span class="s2">"comment"</span><span class="p">]</span> <span class="o">=</span> <span class="n">_text</span><span class="p">(</span><span class="n">reason</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"comment"</span><span class="p">))</span>
|
||||
<span class="n">reason_type</span> <span class="o">=</span> <span class="n">reason</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"type"</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">is_rfc_9990</span> <span class="ow">and</span> <span class="n">reason_type</span> <span class="ow">in</span> <span class="n">RFC_7489_REMOVED_POLICY_OVERRIDE_TYPES</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
||||
<span class="s2">"Policy override reason type </span><span class="si">%r</span><span class="s2"> was removed in RFC 9990; "</span>
|
||||
<span class="s2">"expected one of </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span>
|
||||
<span class="n">reason_type</span><span class="p">,</span>
|
||||
<span class="nb">sorted</span><span class="p">(</span><span class="n">RFC_9990_POLICY_OVERRIDE_TYPES</span><span class="p">),</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">elif</span> <span class="p">(</span>
|
||||
<span class="n">is_rfc_9990</span>
|
||||
<span class="ow">and</span> <span class="n">reason_type</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
|
||||
<span class="ow">and</span> <span class="n">reason_type</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">RFC_9990_POLICY_OVERRIDE_TYPES</span>
|
||||
<span class="p">):</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
||||
<span class="s2">"Unknown policy override reason type </span><span class="si">%r</span><span class="s2"> per RFC 9990; "</span>
|
||||
<span class="s2">"expected one of </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span>
|
||||
<span class="n">reason_type</span><span class="p">,</span>
|
||||
<span class="nb">sorted</span><span class="p">(</span><span class="n">RFC_9990_POLICY_OVERRIDE_TYPES</span><span class="p">),</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">new_policy_evaluated</span><span class="p">[</span><span class="s2">"policy_override_reasons"</span><span class="p">]</span> <span class="o">=</span> <span class="n">reasons</span>
|
||||
<span class="n">new_record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">]</span> <span class="o">=</span> <span class="n">new_policy_evaluated</span>
|
||||
<span class="k">if</span> <span class="s2">"identities"</span> <span class="ow">in</span> <span class="n">record</span><span class="p">:</span>
|
||||
@@ -510,11 +576,18 @@
|
||||
<span class="k">if</span> <span class="s2">"selector"</span> <span class="ow">in</span> <span class="n">result</span> <span class="ow">and</span> <span class="n">result</span><span class="p">[</span><span class="s2">"selector"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">new_result</span><span class="p">[</span><span class="s2">"selector"</span><span class="p">]</span> <span class="o">=</span> <span class="n">result</span><span class="p">[</span><span class="s2">"selector"</span><span class="p">]</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">is_rfc_9990</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
||||
<span class="s2">"DKIM auth result for </span><span class="si">%r</span><span class="s2"> is missing the 'selector' "</span>
|
||||
<span class="s2">"element, which is REQUIRED by RFC 9990"</span><span class="p">,</span>
|
||||
<span class="n">result</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">],</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">new_result</span><span class="p">[</span><span class="s2">"selector"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"none"</span>
|
||||
<span class="k">if</span> <span class="s2">"result"</span> <span class="ow">in</span> <span class="n">result</span> <span class="ow">and</span> <span class="n">result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">new_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">]</span> <span class="o">=</span> <span class="n">result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">]</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">new_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"none"</span>
|
||||
<span class="n">new_result</span><span class="p">[</span><span class="s2">"human_result"</span><span class="p">]</span> <span class="o">=</span> <span class="n">_text</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"human_result"</span><span class="p">))</span>
|
||||
<span class="n">new_record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">][</span><span class="s2">"dkim"</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">new_result</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">auth_results</span><span class="p">[</span><span class="s2">"spf"</span><span class="p">],</span> <span class="nb">list</span><span class="p">):</span>
|
||||
@@ -530,6 +603,7 @@
|
||||
<span class="n">new_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">]</span> <span class="o">=</span> <span class="n">result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">]</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">new_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"none"</span>
|
||||
<span class="n">new_result</span><span class="p">[</span><span class="s2">"human_result"</span><span class="p">]</span> <span class="o">=</span> <span class="n">_text</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"human_result"</span><span class="p">))</span>
|
||||
<span class="n">new_record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">new_result</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="s2">"envelope_from"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">new_record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">]:</span>
|
||||
@@ -813,6 +887,21 @@
|
||||
<span class="c1"># Parse XML and recover from errors</span>
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">xml</span><span class="p">,</span> <span class="nb">bytes</span><span class="p">):</span>
|
||||
<span class="n">xml</span> <span class="o">=</span> <span class="n">xml</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="n">errors</span><span class="o">=</span><span class="s2">"ignore"</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># Detect the XML namespace before any rewriting strips it. The dmarc-2.0</span>
|
||||
<span class="c1"># namespace is one of the indicators for an RFC 9990 report but it is</span>
|
||||
<span class="c1"># NOT a reliable sole discriminator: the <version> element value is</span>
|
||||
<span class="c1"># ambiguous (RFC 9990's appendix sample uses <version>1.0</version></span>
|
||||
<span class="c1"># inside the dmarc-2.0 namespace), and real-world reporters frequently</span>
|
||||
<span class="c1"># emit RFC 9990-shaped reports without declaring the namespace at all.</span>
|
||||
<span class="c1"># The final `is_rfc_9990` decision is made post-parse so that</span>
|
||||
<span class="c1"># RFC 9990-only fields (np, testing, discovery_method, generator,</span>
|
||||
<span class="c1"># human_result) can also vote it in.</span>
|
||||
<span class="n">xml_namespace</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="n">namespace_match</span> <span class="o">=</span> <span class="n">xml_namespace_regex</span><span class="o">.</span><span class="n">search</span><span class="p">(</span><span class="n">xml</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">namespace_match</span><span class="p">:</span>
|
||||
<span class="n">xml_namespace</span> <span class="o">=</span> <span class="n">namespace_match</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
|
||||
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">xmltodict</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">xml</span><span class="p">)[</span><span class="s2">"feedback"</span><span class="p">]</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
@@ -836,16 +925,24 @@
|
||||
|
||||
<span class="n">report</span> <span class="o">=</span> <span class="n">xmltodict</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">xml</span><span class="p">)[</span><span class="s2">"feedback"</span><span class="p">]</span>
|
||||
<span class="n">report_metadata</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">]</span>
|
||||
<span class="c1"># <email> is xs:string in both RFC 7489 and RFC 9990, but defensive</span>
|
||||
<span class="c1"># parsing in the wild: some reporters emit it with an xml:lang or</span>
|
||||
<span class="c1"># similar attribute, which xmltodict turns into a dict.</span>
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">report_metadata</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"email"</span><span class="p">),</span> <span class="nb">dict</span><span class="p">):</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
||||
<span class="s2">"Discarding malformed <email> in report_metadata: </span><span class="si">%r</span><span class="s2">"</span><span class="p">,</span>
|
||||
<span class="n">report_metadata</span><span class="p">[</span><span class="s2">"email"</span><span class="p">],</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">report_metadata</span><span class="p">[</span><span class="s2">"email"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="n">unwrapped</span> <span class="o">=</span> <span class="n">_text</span><span class="p">(</span><span class="n">report_metadata</span><span class="p">[</span><span class="s2">"email"</span><span class="p">])</span>
|
||||
<span class="k">if</span> <span class="n">unwrapped</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
||||
<span class="s2">"Discarding malformed <email> in report_metadata: </span><span class="si">%r</span><span class="s2">"</span><span class="p">,</span>
|
||||
<span class="n">report_metadata</span><span class="p">[</span><span class="s2">"email"</span><span class="p">],</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">report_metadata</span><span class="p">[</span><span class="s2">"email"</span><span class="p">]</span> <span class="o">=</span> <span class="n">unwrapped</span>
|
||||
<span class="n">schema</span> <span class="o">=</span> <span class="s2">"draft"</span>
|
||||
<span class="k">if</span> <span class="s2">"version"</span> <span class="ow">in</span> <span class="n">report</span><span class="p">:</span>
|
||||
<span class="n">schema</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"version"</span><span class="p">]</span>
|
||||
<span class="n">new_report</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"xml_schema"</span><span class="p">:</span> <span class="n">schema</span><span class="p">}</span>
|
||||
<span class="n">new_report</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s2">"xml_schema"</span><span class="p">:</span> <span class="n">schema</span><span class="p">,</span>
|
||||
<span class="s2">"xml_namespace"</span><span class="p">:</span> <span class="n">xml_namespace</span><span class="p">,</span>
|
||||
<span class="p">}</span>
|
||||
<span class="n">new_report_metadata</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
|
||||
<span class="k">if</span> <span class="n">report_metadata</span><span class="p">[</span><span class="s2">"org_name"</span><span class="p">]</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">report_metadata</span><span class="p">[</span><span class="s2">"email"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
@@ -866,9 +963,9 @@
|
||||
<span class="p">)</span>
|
||||
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"org_name"</span><span class="p">]</span> <span class="o">=</span> <span class="n">org_name</span>
|
||||
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"org_email"</span><span class="p">]</span> <span class="o">=</span> <span class="n">report_metadata</span><span class="p">[</span><span class="s2">"email"</span><span class="p">]</span>
|
||||
<span class="n">extra</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="k">if</span> <span class="s2">"extra_contact_info"</span> <span class="ow">in</span> <span class="n">report_metadata</span><span class="p">:</span>
|
||||
<span class="n">extra</span> <span class="o">=</span> <span class="n">report_metadata</span><span class="p">[</span><span class="s2">"extra_contact_info"</span><span class="p">]</span>
|
||||
<span class="c1"># extra_contact_info is langAttrString in RFC 9990 (xs:string in</span>
|
||||
<span class="c1"># RFC 7489) — unwrap {"#text": ..., "@lang": ...} if present.</span>
|
||||
<span class="n">extra</span> <span class="o">=</span> <span class="n">_text</span><span class="p">(</span><span class="n">report_metadata</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"extra_contact_info"</span><span class="p">))</span>
|
||||
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"org_extra_contact_info"</span><span class="p">]</span> <span class="o">=</span> <span class="n">extra</span>
|
||||
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"report_id"</span><span class="p">]</span> <span class="o">=</span> <span class="n">report_metadata</span><span class="p">[</span><span class="s2">"report_id"</span><span class="p">]</span>
|
||||
<span class="n">report_id</span> <span class="o">=</span> <span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"report_id"</span><span class="p">]</span>
|
||||
@@ -896,17 +993,38 @@
|
||||
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="c1"># <error> is langAttrString in RFC 9990 (xs:string in RFC 7489) and</span>
|
||||
<span class="c1"># was cardinality-narrowed from "unbounded" to "1" in RFC 9990, but</span>
|
||||
<span class="c1"># the parser still accepts a list for backward compatibility with</span>
|
||||
<span class="c1"># RFC 7489 reports that carry multiple errors.</span>
|
||||
<span class="k">if</span> <span class="s2">"error"</span> <span class="ow">in</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">]:</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">][</span><span class="s2">"error"</span><span class="p">],</span> <span class="nb">list</span><span class="p">):</span>
|
||||
<span class="n">errors</span> <span class="o">=</span> <span class="p">[</span><span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">][</span><span class="s2">"error"</span><span class="p">]]</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">errors</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">][</span><span class="s2">"error"</span><span class="p">]</span>
|
||||
<span class="n">raw_errors</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">][</span><span class="s2">"error"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">raw_errors</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
|
||||
<span class="n">raw_errors</span> <span class="o">=</span> <span class="p">[</span><span class="n">raw_errors</span><span class="p">]</span>
|
||||
<span class="n">errors</span> <span class="o">=</span> <span class="p">[</span><span class="n">text</span> <span class="k">for</span> <span class="n">text</span> <span class="ow">in</span> <span class="p">(</span><span class="n">_text</span><span class="p">(</span><span class="n">e</span><span class="p">)</span> <span class="k">for</span> <span class="n">e</span> <span class="ow">in</span> <span class="n">raw_errors</span><span class="p">)</span> <span class="k">if</span> <span class="n">text</span><span class="p">]</span>
|
||||
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"errors"</span><span class="p">]</span> <span class="o">=</span> <span class="n">errors</span>
|
||||
<span class="c1"># <generator> is a plain xs:string in RFC 9990 but apply _text() so</span>
|
||||
<span class="c1"># a malformed reporter that decorates it with attributes still</span>
|
||||
<span class="c1"># yields a string instead of breaking downstream consumers.</span>
|
||||
<span class="n">generator</span> <span class="o">=</span> <span class="n">_text</span><span class="p">(</span><span class="n">report_metadata</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"generator"</span><span class="p">))</span>
|
||||
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"generator"</span><span class="p">]</span> <span class="o">=</span> <span class="n">generator</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">]</span> <span class="o">=</span> <span class="n">new_report_metadata</span>
|
||||
<span class="n">records</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="n">policy_published</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">policy_published</span><span class="p">)</span> <span class="ow">is</span> <span class="nb">list</span><span class="p">:</span>
|
||||
<span class="n">policy_published</span> <span class="o">=</span> <span class="n">policy_published</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
|
||||
|
||||
<span class="c1"># Final RFC 9990 detection: the dmarc-2.0 XML namespace OR any</span>
|
||||
<span class="c1"># RFC 9990-only field. Real-world reporters that follow the schema</span>
|
||||
<span class="c1"># without declaring the namespace still get RFC 9990-aware</span>
|
||||
<span class="c1"># warnings (missing DKIM selector, removed override-reason types,</span>
|
||||
<span class="c1"># etc.) and a truthful audit trail in `xml_namespace`.</span>
|
||||
<span class="n">rfc_9990_only_policy_fields</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"np"</span><span class="p">,</span> <span class="s2">"testing"</span><span class="p">,</span> <span class="s2">"discovery_method"</span><span class="p">}</span>
|
||||
<span class="n">is_rfc_9990</span> <span class="o">=</span> <span class="p">(</span>
|
||||
<span class="n">xml_namespace</span> <span class="o">==</span> <span class="n">RFC_9990_NAMESPACE</span>
|
||||
<span class="ow">or</span> <span class="s2">"generator"</span> <span class="ow">in</span> <span class="n">report_metadata</span>
|
||||
<span class="ow">or</span> <span class="nb">any</span><span class="p">(</span><span class="n">f</span> <span class="ow">in</span> <span class="n">policy_published</span> <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="n">rfc_9990_only_policy_fields</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">new_policy_published</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
|
||||
<span class="n">new_policy_published</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">]</span> <span class="o">=</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">]</span>
|
||||
<span class="n">adkim</span> <span class="o">=</span> <span class="s2">"r"</span>
|
||||
@@ -925,16 +1043,39 @@
|
||||
<span class="k">if</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"sp"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">sp</span> <span class="o">=</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"sp"</span><span class="p">]</span>
|
||||
<span class="n">new_policy_published</span><span class="p">[</span><span class="s2">"sp"</span><span class="p">]</span> <span class="o">=</span> <span class="n">sp</span>
|
||||
<span class="n">pct</span> <span class="o">=</span> <span class="s2">"100"</span>
|
||||
<span class="n">pct</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="k">if</span> <span class="s2">"pct"</span> <span class="ow">in</span> <span class="n">policy_published</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"pct"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">pct</span> <span class="o">=</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"pct"</span><span class="p">]</span>
|
||||
<span class="n">new_policy_published</span><span class="p">[</span><span class="s2">"pct"</span><span class="p">]</span> <span class="o">=</span> <span class="n">pct</span>
|
||||
<span class="n">fo</span> <span class="o">=</span> <span class="s2">"0"</span>
|
||||
<span class="n">fo</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="k">if</span> <span class="s2">"fo"</span> <span class="ow">in</span> <span class="n">policy_published</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"fo"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">fo</span> <span class="o">=</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"fo"</span><span class="p">]</span>
|
||||
<span class="n">new_policy_published</span><span class="p">[</span><span class="s2">"fo"</span><span class="p">]</span> <span class="o">=</span> <span class="n">fo</span>
|
||||
<span class="n">np_</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="k">if</span> <span class="s2">"np"</span> <span class="ow">in</span> <span class="n">policy_published</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"np"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">np_</span> <span class="o">=</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"np"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="n">np_</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">(</span><span class="s2">"none"</span><span class="p">,</span> <span class="s2">"quarantine"</span><span class="p">,</span> <span class="s2">"reject"</span><span class="p">):</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"Invalid np value: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">np_</span><span class="p">))</span>
|
||||
<span class="n">new_policy_published</span><span class="p">[</span><span class="s2">"np"</span><span class="p">]</span> <span class="o">=</span> <span class="n">np_</span>
|
||||
<span class="n">testing</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="k">if</span> <span class="s2">"testing"</span> <span class="ow">in</span> <span class="n">policy_published</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"testing"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">testing</span> <span class="o">=</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"testing"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="n">testing</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">(</span><span class="s2">"n"</span><span class="p">,</span> <span class="s2">"y"</span><span class="p">):</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"Invalid testing value: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">testing</span><span class="p">))</span>
|
||||
<span class="n">new_policy_published</span><span class="p">[</span><span class="s2">"testing"</span><span class="p">]</span> <span class="o">=</span> <span class="n">testing</span>
|
||||
<span class="n">discovery_method</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="k">if</span> <span class="s2">"discovery_method"</span> <span class="ow">in</span> <span class="n">policy_published</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"discovery_method"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">discovery_method</span> <span class="o">=</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"discovery_method"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="n">discovery_method</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">(</span><span class="s2">"psl"</span><span class="p">,</span> <span class="s2">"treewalk"</span><span class="p">):</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
||||
<span class="s2">"Invalid discovery_method value: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">discovery_method</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">new_policy_published</span><span class="p">[</span><span class="s2">"discovery_method"</span><span class="p">]</span> <span class="o">=</span> <span class="n">discovery_method</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">]</span> <span class="o">=</span> <span class="n">new_policy_published</span>
|
||||
|
||||
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">"record"</span><span class="p">])</span> <span class="ow">is</span> <span class="nb">list</span><span class="p">:</span>
|
||||
@@ -954,6 +1095,7 @@
|
||||
<span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
|
||||
<span class="n">dns_timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">,</span>
|
||||
<span class="n">dns_retries</span><span class="o">=</span><span class="n">retries</span><span class="p">,</span>
|
||||
<span class="n">is_rfc_9990</span><span class="o">=</span><span class="n">is_rfc_9990</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">_append_parsed_record</span><span class="p">(</span>
|
||||
<span class="n">parsed_record</span><span class="o">=</span><span class="n">report_record</span><span class="p">,</span>
|
||||
@@ -976,6 +1118,7 @@
|
||||
<span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
|
||||
<span class="n">dns_timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">,</span>
|
||||
<span class="n">dns_retries</span><span class="o">=</span><span class="n">retries</span><span class="p">,</span>
|
||||
<span class="n">is_rfc_9990</span><span class="o">=</span><span class="n">is_rfc_9990</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">_append_parsed_record</span><span class="p">(</span>
|
||||
<span class="n">parsed_record</span><span class="o">=</span><span class="n">report_record</span><span class="p">,</span>
|
||||
@@ -1072,6 +1215,7 @@
|
||||
<span class="p">)</span>
|
||||
<span class="k">elif</span> <span class="p">(</span>
|
||||
<span class="n">header</span><span class="p">[:</span> <span class="nb">len</span><span class="p">(</span><span class="n">MAGIC_XML</span><span class="p">)]</span> <span class="o">==</span> <span class="n">MAGIC_XML</span>
|
||||
<span class="ow">or</span> <span class="n">header</span><span class="p">[:</span> <span class="nb">len</span><span class="p">(</span><span class="n">MAGIC_XML_TAG</span><span class="p">)]</span> <span class="o">==</span> <span class="n">MAGIC_XML_TAG</span>
|
||||
<span class="ow">or</span> <span class="n">header</span><span class="p">[:</span> <span class="nb">len</span><span class="p">(</span><span class="n">MAGIC_JSON</span><span class="p">)]</span> <span class="o">==</span> <span class="n">MAGIC_JSON</span>
|
||||
<span class="p">):</span>
|
||||
<span class="n">report</span> <span class="o">=</span> <span class="n">file_object</span><span class="o">.</span><span class="n">read</span><span class="p">()</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="n">errors</span><span class="o">=</span><span class="s2">"ignore"</span><span class="p">)</span>
|
||||
@@ -1210,6 +1354,9 @@
|
||||
<span class="n">sp</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"sp"</span><span class="p">]</span>
|
||||
<span class="n">pct</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"pct"</span><span class="p">]</span>
|
||||
<span class="n">fo</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"fo"</span><span class="p">]</span>
|
||||
<span class="n">np_</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"np"</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||||
<span class="n">testing</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"testing"</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||||
<span class="n">discovery_method</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"discovery_method"</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||||
|
||||
<span class="n">report_dict</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
|
||||
<span class="n">xml_schema</span><span class="o">=</span><span class="n">xml_schema</span><span class="p">,</span>
|
||||
@@ -1226,8 +1373,11 @@
|
||||
<span class="n">aspf</span><span class="o">=</span><span class="n">aspf</span><span class="p">,</span>
|
||||
<span class="n">p</span><span class="o">=</span><span class="n">p</span><span class="p">,</span>
|
||||
<span class="n">sp</span><span class="o">=</span><span class="n">sp</span><span class="p">,</span>
|
||||
<span class="n">np</span><span class="o">=</span><span class="n">np_</span><span class="p">,</span>
|
||||
<span class="n">pct</span><span class="o">=</span><span class="n">pct</span><span class="p">,</span>
|
||||
<span class="n">fo</span><span class="o">=</span><span class="n">fo</span><span class="p">,</span>
|
||||
<span class="n">testing</span><span class="o">=</span><span class="n">testing</span><span class="p">,</span>
|
||||
<span class="n">discovery_method</span><span class="o">=</span><span class="n">discovery_method</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">record</span> <span class="ow">in</span> <span class="n">report</span><span class="p">[</span><span class="s2">"records"</span><span class="p">]:</span>
|
||||
@@ -1329,8 +1479,11 @@
|
||||
<span class="s2">"aspf"</span><span class="p">,</span>
|
||||
<span class="s2">"p"</span><span class="p">,</span>
|
||||
<span class="s2">"sp"</span><span class="p">,</span>
|
||||
<span class="s2">"np"</span><span class="p">,</span>
|
||||
<span class="s2">"pct"</span><span class="p">,</span>
|
||||
<span class="s2">"fo"</span><span class="p">,</span>
|
||||
<span class="s2">"testing"</span><span class="p">,</span>
|
||||
<span class="s2">"discovery_method"</span><span class="p">,</span>
|
||||
<span class="s2">"source_ip_address"</span><span class="p">,</span>
|
||||
<span class="s2">"source_country"</span><span class="p">,</span>
|
||||
<span class="s2">"source_reverse_dns"</span><span class="p">,</span>
|
||||
@@ -1372,9 +1525,9 @@
|
||||
|
||||
|
||||
|
||||
<div class="viewcode-block" id="parse_forensic_report">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.parse_forensic_report">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">parse_forensic_report</span><span class="p">(</span>
|
||||
<div class="viewcode-block" id="parse_failure_report">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.parse_failure_report">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">parse_failure_report</span><span class="p">(</span>
|
||||
<span class="n">feedback_report</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
||||
<span class="n">sample</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
||||
<span class="n">msg_date</span><span class="p">:</span> <span class="n">datetime</span><span class="p">,</span>
|
||||
@@ -1388,9 +1541,9 @@
|
||||
<span class="n">dns_timeout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="n">DEFAULT_DNS_TIMEOUT</span><span class="p">,</span>
|
||||
<span class="n">dns_retries</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="n">DEFAULT_DNS_MAX_RETRIES</span><span class="p">,</span>
|
||||
<span class="n">strip_attachment_payloads</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
||||
<span class="p">)</span> <span class="o">-></span> <span class="n">ForensicReport</span><span class="p">:</span>
|
||||
<span class="p">)</span> <span class="o">-></span> <span class="n">FailureReport</span><span class="p">:</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Converts a DMARC forensic report and sample to a dict</span>
|
||||
<span class="sd"> Converts a DMARC failure report and sample to a dict</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> feedback_report (str): A message's feedback report as a string</span>
|
||||
@@ -1407,7 +1560,7 @@
|
||||
<span class="sd"> dns_retries (int): Number of times to retry DNS queries on timeout</span>
|
||||
<span class="sd"> or other transient errors</span>
|
||||
<span class="sd"> strip_attachment_payloads (bool): Remove attachment payloads from</span>
|
||||
<span class="sd"> forensic report results</span>
|
||||
<span class="sd"> failure report results</span>
|
||||
|
||||
<span class="sd"> Returns:</span>
|
||||
<span class="sd"> dict: A parsed report and sample</span>
|
||||
@@ -1423,7 +1576,7 @@
|
||||
|
||||
<span class="k">if</span> <span class="s2">"arrival_date"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_report</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">msg_date</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidForensicReport</span><span class="p">(</span><span class="s2">"Forensic sample is not a valid email"</span><span class="p">)</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidFailureReport</span><span class="p">(</span><span class="s2">"Failure sample is not a valid email"</span><span class="p">)</span>
|
||||
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"arrival_date"</span><span class="p">]</span> <span class="o">=</span> <span class="n">msg_date</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span>
|
||||
|
||||
<span class="k">if</span> <span class="s2">"version"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_report</span><span class="p">:</span>
|
||||
@@ -1465,21 +1618,37 @@
|
||||
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">]</span> <span class="o">=</span> <span class="n">parsed_report_source</span>
|
||||
<span class="k">del</span> <span class="n">parsed_report</span><span class="p">[</span><span class="s2">"source_ip"</span><span class="p">]</span>
|
||||
|
||||
<span class="c1"># Identity-Alignment is REQUIRED per RFC 9991 §4. Default silently for</span>
|
||||
<span class="c1"># backward compatibility with pre-9991 reporters, but log so the</span>
|
||||
<span class="c1"># offending reporter is visible. Values are CFWS-separated per the</span>
|
||||
<span class="c1"># ABNF, so each mechanism is stripped after splitting.</span>
|
||||
<span class="k">if</span> <span class="s2">"identity_alignment"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_report</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
||||
<span class="s2">"Failure report missing required 'Identity-Alignment' "</span>
|
||||
<span class="s2">"field (RFC 9991 §4); defaulting to no aligned mechanisms"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"authentication_mechanisms"</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="k">elif</span> <span class="n">parsed_report</span><span class="p">[</span><span class="s2">"identity_alignment"</span><span class="p">]</span> <span class="o">==</span> <span class="s2">"none"</span><span class="p">:</span>
|
||||
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"authentication_mechanisms"</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="k">del</span> <span class="n">parsed_report</span><span class="p">[</span><span class="s2">"identity_alignment"</span><span class="p">]</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">auth_mechanisms</span> <span class="o">=</span> <span class="n">parsed_report</span><span class="p">[</span><span class="s2">"identity_alignment"</span><span class="p">]</span>
|
||||
<span class="n">auth_mechanisms</span> <span class="o">=</span> <span class="n">auth_mechanisms</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">","</span><span class="p">)</span>
|
||||
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"authentication_mechanisms"</span><span class="p">]</span> <span class="o">=</span> <span class="n">auth_mechanisms</span>
|
||||
<span class="n">raw_alignment</span> <span class="o">=</span> <span class="n">parsed_report</span><span class="p">[</span><span class="s2">"identity_alignment"</span><span class="p">]</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
|
||||
<span class="k">if</span> <span class="n">raw_alignment</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s2">"none"</span><span class="p">:</span>
|
||||
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"authentication_mechanisms"</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"authentication_mechanisms"</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span>
|
||||
<span class="n">m</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span> <span class="k">for</span> <span class="n">m</span> <span class="ow">in</span> <span class="n">raw_alignment</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">","</span><span class="p">)</span> <span class="k">if</span> <span class="n">m</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
|
||||
<span class="p">]</span>
|
||||
<span class="k">del</span> <span class="n">parsed_report</span><span class="p">[</span><span class="s2">"identity_alignment"</span><span class="p">]</span>
|
||||
|
||||
<span class="c1"># Auth-Failure is REQUIRED per RFC 9991 §4. Comma-separated per ABNF</span>
|
||||
<span class="c1"># so strip each token.</span>
|
||||
<span class="k">if</span> <span class="s2">"auth_failure"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_report</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
||||
<span class="s2">"Failure report missing required 'Auth-Failure' field "</span>
|
||||
<span class="s2">"(RFC 9991 §4); defaulting to 'dmarc'"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"auth_failure"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"dmarc"</span>
|
||||
<span class="n">auth_failure</span> <span class="o">=</span> <span class="n">parsed_report</span><span class="p">[</span><span class="s2">"auth_failure"</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">","</span><span class="p">)</span>
|
||||
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"auth_failure"</span><span class="p">]</span> <span class="o">=</span> <span class="n">auth_failure</span>
|
||||
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"auth_failure"</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span>
|
||||
<span class="n">f</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span> <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="n">parsed_report</span><span class="p">[</span><span class="s2">"auth_failure"</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">","</span><span class="p">)</span> <span class="k">if</span> <span class="n">f</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
|
||||
<span class="p">]</span>
|
||||
|
||||
<span class="n">optional_fields</span> <span class="o">=</span> <span class="p">[</span>
|
||||
<span class="s2">"original_envelope_id"</span><span class="p">,</span>
|
||||
@@ -1510,30 +1679,30 @@
|
||||
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"sample"</span><span class="p">]</span> <span class="o">=</span> <span class="n">sample</span>
|
||||
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">]</span> <span class="o">=</span> <span class="n">parsed_sample</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">cast</span><span class="p">(</span><span class="n">ForensicReport</span><span class="p">,</span> <span class="n">parsed_report</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="n">cast</span><span class="p">(</span><span class="n">FailureReport</span><span class="p">,</span> <span class="n">parsed_report</span><span class="p">)</span>
|
||||
|
||||
<span class="k">except</span> <span class="ne">KeyError</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidForensicReport</span><span class="p">(</span><span class="s2">"Missing value: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidFailureReport</span><span class="p">(</span><span class="s2">"Missing value: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
||||
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidForensicReport</span><span class="p">(</span><span class="s2">"Unexpected error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
<span class="k">raise</span> <span class="n">InvalidFailureReport</span><span class="p">(</span><span class="s2">"Unexpected error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
|
||||
|
||||
|
||||
<div class="viewcode-block" id="parsed_forensic_reports_to_csv_rows">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.parsed_forensic_reports_to_csv_rows">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">parsed_forensic_reports_to_csv_rows</span><span class="p">(</span>
|
||||
<span class="n">reports</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">ForensicReport</span><span class="p">,</span> <span class="nb">list</span><span class="p">[</span><span class="n">ForensicReport</span><span class="p">]],</span>
|
||||
<div class="viewcode-block" id="parsed_failure_reports_to_csv_rows">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.parsed_failure_reports_to_csv_rows">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">parsed_failure_reports_to_csv_rows</span><span class="p">(</span>
|
||||
<span class="n">reports</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">FailureReport</span><span class="p">,</span> <span class="nb">list</span><span class="p">[</span><span class="n">FailureReport</span><span class="p">]],</span>
|
||||
<span class="p">)</span> <span class="o">-></span> <span class="nb">list</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]]:</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Converts one or more parsed forensic reports to a list of dicts in flat CSV</span>
|
||||
<span class="sd"> Converts one or more parsed failure reports to a list of dicts in flat CSV</span>
|
||||
<span class="sd"> format</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> reports: A parsed forensic report or list of parsed forensic reports</span>
|
||||
<span class="sd"> reports: A parsed failure report or list of parsed failure reports</span>
|
||||
|
||||
<span class="sd"> Returns:</span>
|
||||
<span class="sd"> list: Parsed forensic report data as a list of dicts in flat CSV format</span>
|
||||
<span class="sd"> list: Parsed failure report data as a list of dicts in flat CSV format</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">reports</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
|
||||
<span class="n">reports</span> <span class="o">=</span> <span class="p">[</span><span class="n">reports</span><span class="p">]</span>
|
||||
@@ -1564,20 +1733,20 @@
|
||||
|
||||
|
||||
|
||||
<div class="viewcode-block" id="parsed_forensic_reports_to_csv">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.parsed_forensic_reports_to_csv">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">parsed_forensic_reports_to_csv</span><span class="p">(</span>
|
||||
<span class="n">reports</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">ForensicReport</span><span class="p">,</span> <span class="nb">list</span><span class="p">[</span><span class="n">ForensicReport</span><span class="p">]],</span>
|
||||
<div class="viewcode-block" id="parsed_failure_reports_to_csv">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.parsed_failure_reports_to_csv">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">parsed_failure_reports_to_csv</span><span class="p">(</span>
|
||||
<span class="n">reports</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">FailureReport</span><span class="p">,</span> <span class="nb">list</span><span class="p">[</span><span class="n">FailureReport</span><span class="p">]],</span>
|
||||
<span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Converts one or more parsed forensic reports to flat CSV format, including</span>
|
||||
<span class="sd"> Converts one or more parsed failure reports to flat CSV format, including</span>
|
||||
<span class="sd"> headers</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> reports: A parsed forensic report or list of parsed forensic reports</span>
|
||||
<span class="sd"> reports: A parsed failure report or list of parsed failure reports</span>
|
||||
|
||||
<span class="sd"> Returns:</span>
|
||||
<span class="sd"> str: Parsed forensic report data in flat CSV format, including headers</span>
|
||||
<span class="sd"> str: Parsed failure report data in flat CSV format, including headers</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">fields</span> <span class="o">=</span> <span class="p">[</span>
|
||||
<span class="s2">"feedback_type"</span><span class="p">,</span>
|
||||
@@ -1612,7 +1781,7 @@
|
||||
<span class="n">csv_writer</span> <span class="o">=</span> <span class="n">DictWriter</span><span class="p">(</span><span class="n">csv_file</span><span class="p">,</span> <span class="n">fieldnames</span><span class="o">=</span><span class="n">fields</span><span class="p">)</span>
|
||||
<span class="n">csv_writer</span><span class="o">.</span><span class="n">writeheader</span><span class="p">()</span>
|
||||
|
||||
<span class="n">rows</span> <span class="o">=</span> <span class="n">parsed_forensic_reports_to_csv_rows</span><span class="p">(</span><span class="n">reports</span><span class="p">)</span>
|
||||
<span class="n">rows</span> <span class="o">=</span> <span class="n">parsed_failure_reports_to_csv_rows</span><span class="p">(</span><span class="n">reports</span><span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="n">rows</span><span class="p">:</span>
|
||||
<span class="n">new_row</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
|
||||
@@ -1656,13 +1825,13 @@
|
||||
<span class="sd"> dns_retries (int): Number of times to retry DNS queries on timeout</span>
|
||||
<span class="sd"> or other transient errors</span>
|
||||
<span class="sd"> strip_attachment_payloads (bool): Remove attachment payloads from</span>
|
||||
<span class="sd"> forensic report results</span>
|
||||
<span class="sd"> failure report results</span>
|
||||
<span class="sd"> keep_alive (callable): keep alive function</span>
|
||||
<span class="sd"> normalize_timespan_threshold_hours (float): Normalize timespans beyond this</span>
|
||||
|
||||
<span class="sd"> Returns:</span>
|
||||
<span class="sd"> dict:</span>
|
||||
<span class="sd"> * ``report_type``: ``aggregate`` or ``forensic``</span>
|
||||
<span class="sd"> * ``report_type``: ``aggregate`` or ``failure``</span>
|
||||
<span class="sd"> * ``report``: The parsed report</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">result</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">ParsedReport</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
@@ -1806,7 +1975,7 @@
|
||||
|
||||
<span class="k">if</span> <span class="n">feedback_report</span> <span class="ow">and</span> <span class="n">sample</span><span class="p">:</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">forensic_report</span> <span class="o">=</span> <span class="n">parse_forensic_report</span><span class="p">(</span>
|
||||
<span class="n">failure_report</span> <span class="o">=</span> <span class="n">parse_failure_report</span><span class="p">(</span>
|
||||
<span class="n">feedback_report</span><span class="p">,</span>
|
||||
<span class="n">sample</span><span class="p">,</span>
|
||||
<span class="n">msg_date</span><span class="p">,</span>
|
||||
@@ -1820,17 +1989,17 @@
|
||||
<span class="n">dns_retries</span><span class="o">=</span><span class="n">dns_retries</span><span class="p">,</span>
|
||||
<span class="n">strip_attachment_payloads</span><span class="o">=</span><span class="n">strip_attachment_payloads</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">except</span> <span class="n">InvalidForensicReport</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">except</span> <span class="n">InvalidFailureReport</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">error</span> <span class="o">=</span> <span class="p">(</span>
|
||||
<span class="s1">'Message with subject "</span><span class="si">{0}</span><span class="s1">" '</span>
|
||||
<span class="s2">"is not a valid "</span>
|
||||
<span class="s2">"forensic DMARC report: </span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">subject</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
|
||||
<span class="s2">"failure DMARC report: </span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">subject</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidForensicReport</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidFailureReport</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidForensicReport</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidFailureReport</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
|
||||
<span class="n">result</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"report_type"</span><span class="p">:</span> <span class="s2">"forensic"</span><span class="p">,</span> <span class="s2">"report"</span><span class="p">:</span> <span class="n">forensic_report</span><span class="p">}</span>
|
||||
<span class="n">result</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"report_type"</span><span class="p">:</span> <span class="s2">"failure"</span><span class="p">,</span> <span class="s2">"report"</span><span class="p">:</span> <span class="n">failure_report</span><span class="p">}</span>
|
||||
<span class="k">return</span> <span class="n">result</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">result</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
@@ -1858,7 +2027,7 @@
|
||||
<span class="n">keep_alive</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Callable</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">normalize_timespan_threshold_hours</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mi">24</span><span class="p">,</span>
|
||||
<span class="p">)</span> <span class="o">-></span> <span class="n">ParsedReport</span><span class="p">:</span>
|
||||
<span class="w"> </span><span class="sd">"""Parses a DMARC aggregate or forensic file at the given path, a</span>
|
||||
<span class="w"> </span><span class="sd">"""Parses a DMARC aggregate or failure file at the given path, a</span>
|
||||
<span class="sd"> file-like object. or bytes</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
@@ -1870,7 +2039,7 @@
|
||||
<span class="sd"> dns_retries (int): Number of times to retry DNS queries on timeout</span>
|
||||
<span class="sd"> or other transient errors</span>
|
||||
<span class="sd"> strip_attachment_payloads (bool): Remove attachment payloads from</span>
|
||||
<span class="sd"> forensic report results</span>
|
||||
<span class="sd"> failure report results</span>
|
||||
<span class="sd"> ip_db_path (str): Path to a MMDB file from IPinfo, MaxMind, or DBIP</span>
|
||||
<span class="sd"> always_use_local_files (bool): Do not download files</span>
|
||||
<span class="sd"> reverse_dns_map_path (str): Path to a reverse DNS map</span>
|
||||
@@ -1969,7 +2138,7 @@
|
||||
<span class="sd"> dns_retries (int): Number of times to retry DNS queries on timeout</span>
|
||||
<span class="sd"> or other transient errors</span>
|
||||
<span class="sd"> strip_attachment_payloads (bool): Remove attachment payloads from</span>
|
||||
<span class="sd"> forensic report results</span>
|
||||
<span class="sd"> failure report results</span>
|
||||
<span class="sd"> always_use_local_files (bool): Do not download files</span>
|
||||
<span class="sd"> reverse_dns_map_path (str): Path to a reverse DNS map file</span>
|
||||
<span class="sd"> reverse_dns_map_url (str): URL to a reverse DNS map file</span>
|
||||
@@ -1978,11 +2147,11 @@
|
||||
<span class="sd"> normalize_timespan_threshold_hours (float): Normalize timespans beyond this</span>
|
||||
|
||||
<span class="sd"> Returns:</span>
|
||||
<span class="sd"> dict: Lists of ``aggregate_reports``, ``forensic_reports``, and ``smtp_tls_reports``</span>
|
||||
<span class="sd"> dict: Lists of ``aggregate_reports``, ``failure_reports``, and ``smtp_tls_reports``</span>
|
||||
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">aggregate_reports</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">AggregateReport</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="n">forensic_reports</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">ForensicReport</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="n">failure_reports</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">FailureReport</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="n">smtp_tls_reports</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">SMTPTLSReport</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">mbox</span> <span class="o">=</span> <span class="n">mailbox</span><span class="o">.</span><span class="n">mbox</span><span class="p">(</span><span class="n">input_</span><span class="p">)</span>
|
||||
@@ -2020,8 +2189,8 @@
|
||||
<span class="s2">"Skipping duplicate aggregate report "</span>
|
||||
<span class="sa">f</span><span class="s2">"from </span><span class="si">{</span><span class="n">report_org</span><span class="si">}</span><span class="s2"> with ID: </span><span class="si">{</span><span class="n">report_id</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">elif</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">"report_type"</span><span class="p">]</span> <span class="o">==</span> <span class="s2">"forensic"</span><span class="p">:</span>
|
||||
<span class="n">forensic_reports</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">parsed_email</span><span class="p">[</span><span class="s2">"report"</span><span class="p">])</span>
|
||||
<span class="k">elif</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">"report_type"</span><span class="p">]</span> <span class="o">==</span> <span class="s2">"failure"</span><span class="p">:</span>
|
||||
<span class="n">failure_reports</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">parsed_email</span><span class="p">[</span><span class="s2">"report"</span><span class="p">])</span>
|
||||
<span class="k">elif</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">"report_type"</span><span class="p">]</span> <span class="o">==</span> <span class="s2">"smtp_tls"</span><span class="p">:</span>
|
||||
<span class="n">smtp_tls_reports</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">parsed_email</span><span class="p">[</span><span class="s2">"report"</span><span class="p">])</span>
|
||||
<span class="k">except</span> <span class="n">InvalidDMARCReport</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
||||
@@ -2030,12 +2199,60 @@
|
||||
<span class="k">raise</span> <span class="n">InvalidDMARCReport</span><span class="p">(</span><span class="s2">"Mailbox </span><span class="si">{0}</span><span class="s2"> does not exist"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">input_</span><span class="p">))</span>
|
||||
<span class="k">return</span> <span class="p">{</span>
|
||||
<span class="s2">"aggregate_reports"</span><span class="p">:</span> <span class="n">aggregate_reports</span><span class="p">,</span>
|
||||
<span class="s2">"forensic_reports"</span><span class="p">:</span> <span class="n">forensic_reports</span><span class="p">,</span>
|
||||
<span class="s2">"failure_reports"</span><span class="p">:</span> <span class="n">failure_reports</span><span class="p">,</span>
|
||||
<span class="s2">"smtp_tls_reports"</span><span class="p">:</span> <span class="n">smtp_tls_reports</span><span class="p">,</span>
|
||||
<span class="p">}</span></div>
|
||||
|
||||
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">_migrate_forensic_archive_folder</span><span class="p">(</span>
|
||||
<span class="n">connection</span><span class="p">:</span> <span class="n">MailboxConnection</span><span class="p">,</span> <span class="n">archive_folder</span><span class="p">:</span> <span class="nb">str</span>
|
||||
<span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="w"> </span><span class="sd">"""Consolidate a pre-rename ``<archive>/Forensic`` subfolder into</span>
|
||||
<span class="sd"> ``<archive>/Failure``.</span>
|
||||
|
||||
<span class="sd"> Before failure reports were renamed from "forensic" reports, they were</span>
|
||||
<span class="sd"> archived under ``<archive_folder>/Forensic``; they now go to</span>
|
||||
<span class="sd"> ``<archive_folder>/Failure``. This best-effort, run-on-startup migration</span>
|
||||
<span class="sd"> moves any pre-existing legacy archive into the new location so reports</span>
|
||||
<span class="sd"> filed before and after the rename live in the same folder.</span>
|
||||
|
||||
<span class="sd"> It is a no-op when there is no legacy ``Forensic`` folder (the common</span>
|
||||
<span class="sd"> case), and never raises: a mailbox that cannot be reorganized is logged</span>
|
||||
<span class="sd"> and skipped, consistent with the rest of parsedmarc's mailbox handling</span>
|
||||
<span class="sd"> (warn, don't crash). Uses the folder-management API added in mailsuite</span>
|
||||
<span class="sd"> 2.1.0 (``folder_exists`` / ``rename_folder`` / ``merge_folders``).</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">old_folder</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">/Forensic"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">archive_folder</span><span class="p">)</span>
|
||||
<span class="n">new_folder</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">/Failure"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">archive_folder</span><span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">connection</span><span class="o">.</span><span class="n">folder_exists</span><span class="p">(</span><span class="n">old_folder</span><span class="p">):</span>
|
||||
<span class="k">return</span>
|
||||
<span class="k">if</span> <span class="n">connection</span><span class="o">.</span><span class="n">folder_exists</span><span class="p">(</span><span class="n">new_folder</span><span class="p">):</span>
|
||||
<span class="c1"># Both exist (e.g. a partial earlier migration, or a manually</span>
|
||||
<span class="c1"># created Failure folder): move the legacy folder's messages into</span>
|
||||
<span class="c1"># the new one and drop the now-empty legacy folder.</span>
|
||||
<span class="n">connection</span><span class="o">.</span><span class="n">merge_folders</span><span class="p">(</span><span class="n">old_folder</span><span class="p">,</span> <span class="n">new_folder</span><span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
|
||||
<span class="s2">"Merged legacy archive folder </span><span class="si">{0}</span><span class="s2"> into </span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">old_folder</span><span class="p">,</span> <span class="n">new_folder</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">connection</span><span class="o">.</span><span class="n">rename_folder</span><span class="p">(</span><span class="n">old_folder</span><span class="p">,</span> <span class="n">new_folder</span><span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
|
||||
<span class="s2">"Renamed legacy archive folder </span><span class="si">{0}</span><span class="s2"> to </span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">old_folder</span><span class="p">,</span> <span class="n">new_folder</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
||||
<span class="s2">"Could not migrate legacy archive folder </span><span class="si">{0}</span><span class="s2"> to </span><span class="si">{1}</span><span class="s2">: </span><span class="si">{2}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">old_folder</span><span class="p">,</span> <span class="n">new_folder</span><span class="p">,</span> <span class="n">error</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="get_dmarc_reports_from_mailbox">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.get_dmarc_reports_from_mailbox">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">get_dmarc_reports_from_mailbox</span><span class="p">(</span>
|
||||
@@ -2079,7 +2296,7 @@
|
||||
<span class="sd"> dns_retries (int): Number of times to retry DNS queries on timeout</span>
|
||||
<span class="sd"> or other transient errors</span>
|
||||
<span class="sd"> strip_attachment_payloads (bool): Remove attachment payloads from</span>
|
||||
<span class="sd"> forensic report results</span>
|
||||
<span class="sd"> failure report results</span>
|
||||
<span class="sd"> results (dict): Results from the previous run</span>
|
||||
<span class="sd"> batch_size (int): Number of messages to read and process before saving</span>
|
||||
<span class="sd"> (use 0 for no limit)</span>
|
||||
@@ -2090,7 +2307,7 @@
|
||||
<span class="sd"> normalize_timespan_threshold_hours (float): Normalize timespans beyond this</span>
|
||||
|
||||
<span class="sd"> Returns:</span>
|
||||
<span class="sd"> dict: Lists of ``aggregate_reports``, ``forensic_reports``, and ``smtp_tls_reports``</span>
|
||||
<span class="sd"> dict: Lists of ``aggregate_reports``, ``failure_reports``, and ``smtp_tls_reports``</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">if</span> <span class="n">delete</span> <span class="ow">and</span> <span class="n">test</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"delete and test options are mutually exclusive"</span><span class="p">)</span>
|
||||
@@ -2102,25 +2319,26 @@
|
||||
<span class="n">current_time</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">datetime</span><span class="p">,</span> <span class="n">date</span><span class="p">,</span> <span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
|
||||
<span class="n">aggregate_reports</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">AggregateReport</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="n">forensic_reports</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">ForensicReport</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="n">failure_reports</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">FailureReport</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="n">smtp_tls_reports</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">SMTPTLSReport</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="n">aggregate_report_msg_uids</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="n">forensic_report_msg_uids</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="n">failure_report_msg_uids</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="n">smtp_tls_msg_uids</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="n">aggregate_reports_folder</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">/Aggregate"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">archive_folder</span><span class="p">)</span>
|
||||
<span class="n">forensic_reports_folder</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">/Forensic"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">archive_folder</span><span class="p">)</span>
|
||||
<span class="n">failure_reports_folder</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">/Failure"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">archive_folder</span><span class="p">)</span>
|
||||
<span class="n">smtp_tls_reports_folder</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">/SMTP-TLS"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">archive_folder</span><span class="p">)</span>
|
||||
<span class="n">invalid_reports_folder</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">/Invalid"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">archive_folder</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">results</span><span class="p">:</span>
|
||||
<span class="n">aggregate_reports</span> <span class="o">=</span> <span class="n">results</span><span class="p">[</span><span class="s2">"aggregate_reports"</span><span class="p">]</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
<span class="n">forensic_reports</span> <span class="o">=</span> <span class="n">results</span><span class="p">[</span><span class="s2">"forensic_reports"</span><span class="p">]</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
<span class="n">failure_reports</span> <span class="o">=</span> <span class="n">results</span><span class="p">[</span><span class="s2">"failure_reports"</span><span class="p">]</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
<span class="n">smtp_tls_reports</span> <span class="o">=</span> <span class="n">results</span><span class="p">[</span><span class="s2">"smtp_tls_reports"</span><span class="p">]</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">test</span> <span class="ow">and</span> <span class="n">create_folders</span><span class="p">:</span>
|
||||
<span class="n">_migrate_forensic_archive_folder</span><span class="p">(</span><span class="n">connection</span><span class="p">,</span> <span class="n">archive_folder</span><span class="p">)</span>
|
||||
<span class="n">connection</span><span class="o">.</span><span class="n">create_folder</span><span class="p">(</span><span class="n">archive_folder</span><span class="p">)</span>
|
||||
<span class="n">connection</span><span class="o">.</span><span class="n">create_folder</span><span class="p">(</span><span class="n">aggregate_reports_folder</span><span class="p">)</span>
|
||||
<span class="n">connection</span><span class="o">.</span><span class="n">create_folder</span><span class="p">(</span><span class="n">forensic_reports_folder</span><span class="p">)</span>
|
||||
<span class="n">connection</span><span class="o">.</span><span class="n">create_folder</span><span class="p">(</span><span class="n">failure_reports_folder</span><span class="p">)</span>
|
||||
<span class="n">connection</span><span class="o">.</span><span class="n">create_folder</span><span class="p">(</span><span class="n">smtp_tls_reports_folder</span><span class="p">)</span>
|
||||
<span class="n">connection</span><span class="o">.</span><span class="n">create_folder</span><span class="p">(</span><span class="n">invalid_reports_folder</span><span class="p">)</span>
|
||||
|
||||
@@ -2226,9 +2444,9 @@
|
||||
<span class="sa">f</span><span class="s2">"Skipping duplicate aggregate report with ID: </span><span class="si">{</span><span class="n">report_id</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">aggregate_report_msg_uids</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">message_id</span><span class="p">)</span>
|
||||
<span class="k">elif</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">"report_type"</span><span class="p">]</span> <span class="o">==</span> <span class="s2">"forensic"</span><span class="p">:</span>
|
||||
<span class="n">forensic_reports</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">parsed_email</span><span class="p">[</span><span class="s2">"report"</span><span class="p">])</span>
|
||||
<span class="n">forensic_report_msg_uids</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">message_id</span><span class="p">)</span>
|
||||
<span class="k">elif</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">"report_type"</span><span class="p">]</span> <span class="o">==</span> <span class="s2">"failure"</span><span class="p">:</span>
|
||||
<span class="n">failure_reports</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">parsed_email</span><span class="p">[</span><span class="s2">"report"</span><span class="p">])</span>
|
||||
<span class="n">failure_report_msg_uids</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">message_id</span><span class="p">)</span>
|
||||
<span class="k">elif</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">"report_type"</span><span class="p">]</span> <span class="o">==</span> <span class="s2">"smtp_tls"</span><span class="p">:</span>
|
||||
<span class="n">smtp_tls_reports</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">parsed_email</span><span class="p">[</span><span class="s2">"report"</span><span class="p">])</span>
|
||||
<span class="n">smtp_tls_msg_uids</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">message_id</span><span class="p">)</span>
|
||||
@@ -2255,7 +2473,7 @@
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">test</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">delete</span><span class="p">:</span>
|
||||
<span class="n">processed_messages</span> <span class="o">=</span> <span class="p">(</span>
|
||||
<span class="n">aggregate_report_msg_uids</span> <span class="o">+</span> <span class="n">forensic_report_msg_uids</span> <span class="o">+</span> <span class="n">smtp_tls_msg_uids</span>
|
||||
<span class="n">aggregate_report_msg_uids</span> <span class="o">+</span> <span class="n">failure_report_msg_uids</span> <span class="o">+</span> <span class="n">smtp_tls_msg_uids</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">number_of_processed_msgs</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">processed_messages</span><span class="p">)</span>
|
||||
@@ -2295,24 +2513,24 @@
|
||||
<span class="n">message</span> <span class="o">=</span> <span class="s2">"Error moving message UID"</span>
|
||||
<span class="n">e</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2"> </span><span class="si">{1}</span><span class="s2">: </span><span class="si">{2}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">message</span><span class="p">,</span> <span class="n">msg_uid</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Mailbox error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="p">))</span>
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">forensic_report_msg_uids</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="n">message</span> <span class="o">=</span> <span class="s2">"Moving forensic report messages from"</span>
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">failure_report_msg_uids</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="n">message</span> <span class="o">=</span> <span class="s2">"Moving failure report messages from"</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
||||
<span class="s2">"</span><span class="si">{0}</span><span class="s2"> </span><span class="si">{1}</span><span class="s2"> to </span><span class="si">{2}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">message</span><span class="p">,</span> <span class="n">reports_folder</span><span class="p">,</span> <span class="n">forensic_reports_folder</span>
|
||||
<span class="n">message</span><span class="p">,</span> <span class="n">reports_folder</span><span class="p">,</span> <span class="n">failure_reports_folder</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">number_of_forensic_msgs</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">forensic_report_msg_uids</span><span class="p">)</span>
|
||||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">number_of_forensic_msgs</span><span class="p">):</span>
|
||||
<span class="n">msg_uid</span> <span class="o">=</span> <span class="n">forensic_report_msg_uids</span><span class="p">[</span><span class="n">i</span><span class="p">]</span>
|
||||
<span class="n">number_of_failure_msgs</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">failure_report_msg_uids</span><span class="p">)</span>
|
||||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">number_of_failure_msgs</span><span class="p">):</span>
|
||||
<span class="n">msg_uid</span> <span class="o">=</span> <span class="n">failure_report_msg_uids</span><span class="p">[</span><span class="n">i</span><span class="p">]</span>
|
||||
<span class="n">message</span> <span class="o">=</span> <span class="s2">"Moving message"</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
||||
<span class="s2">"</span><span class="si">{0}</span><span class="s2"> </span><span class="si">{1}</span><span class="s2"> of </span><span class="si">{2}</span><span class="s2">: UID </span><span class="si">{3}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">message</span><span class="p">,</span> <span class="n">i</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="n">number_of_forensic_msgs</span><span class="p">,</span> <span class="n">msg_uid</span>
|
||||
<span class="n">message</span><span class="p">,</span> <span class="n">i</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="n">number_of_failure_msgs</span><span class="p">,</span> <span class="n">msg_uid</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">connection</span><span class="o">.</span><span class="n">move_message</span><span class="p">(</span><span class="n">msg_uid</span><span class="p">,</span> <span class="n">forensic_reports_folder</span><span class="p">)</span>
|
||||
<span class="n">connection</span><span class="o">.</span><span class="n">move_message</span><span class="p">(</span><span class="n">msg_uid</span><span class="p">,</span> <span class="n">failure_reports_folder</span><span class="p">)</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">e</span> <span class="o">=</span> <span class="s2">"Error moving message UID </span><span class="si">{0}</span><span class="s2">: </span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">msg_uid</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Mailbox error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="p">))</span>
|
||||
@@ -2339,7 +2557,7 @@
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Mailbox error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="p">))</span>
|
||||
<span class="n">results</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s2">"aggregate_reports"</span><span class="p">:</span> <span class="n">aggregate_reports</span><span class="p">,</span>
|
||||
<span class="s2">"forensic_reports"</span><span class="p">:</span> <span class="n">forensic_reports</span><span class="p">,</span>
|
||||
<span class="s2">"failure_reports"</span><span class="p">:</span> <span class="n">failure_reports</span><span class="p">,</span>
|
||||
<span class="s2">"smtp_tls_reports"</span><span class="p">:</span> <span class="n">smtp_tls_reports</span><span class="p">,</span>
|
||||
<span class="p">}</span>
|
||||
|
||||
@@ -2428,7 +2646,7 @@
|
||||
<span class="sd"> dns_retries (int): Number of times to retry DNS queries on timeout</span>
|
||||
<span class="sd"> or other transient errors</span>
|
||||
<span class="sd"> strip_attachment_payloads (bool): Replace attachment payloads in</span>
|
||||
<span class="sd"> forensic report samples with None</span>
|
||||
<span class="sd"> failure report samples with None</span>
|
||||
<span class="sd"> batch_size (int): Number of messages to read and process before saving</span>
|
||||
<span class="sd"> since: Search for messages since certain time</span>
|
||||
<span class="sd"> normalize_timespan_threshold_hours (float): Normalize timespans beyond this</span>
|
||||
@@ -2470,34 +2688,50 @@
|
||||
|
||||
|
||||
|
||||
<div class="viewcode-block" id="append_json">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.append_json">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">append_json</span><span class="p">(</span>
|
||||
<span class="n">filename</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
||||
<span class="n">reports</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span>
|
||||
<span class="n">Sequence</span><span class="p">[</span><span class="n">AggregateReport</span><span class="p">],</span>
|
||||
<span class="n">Sequence</span><span class="p">[</span><span class="n">ForensicReport</span><span class="p">],</span>
|
||||
<span class="n">Sequence</span><span class="p">[</span><span class="n">FailureReport</span><span class="p">],</span>
|
||||
<span class="n">Sequence</span><span class="p">[</span><span class="n">SMTPTLSReport</span><span class="p">],</span>
|
||||
<span class="p">],</span>
|
||||
<span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="s2">"a+"</span><span class="p">,</span> <span class="n">newline</span><span class="o">=</span><span class="s2">"</span><span class="se">\n</span><span class="s2">"</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s2">"utf-8"</span><span class="p">)</span> <span class="k">as</span> <span class="n">output</span><span class="p">:</span>
|
||||
<span class="n">output_json</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">reports</span><span class="p">,</span> <span class="n">ensure_ascii</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">indent</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">output</span><span class="o">.</span><span class="n">seek</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">os</span><span class="o">.</span><span class="n">SEEK_END</span><span class="p">)</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">reports</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="c1"># not appending anything, don't do any dance to append it</span>
|
||||
<span class="c1"># correctly</span>
|
||||
<span class="k">return</span>
|
||||
<span class="n">output</span><span class="o">.</span><span class="n">seek</span><span class="p">(</span><span class="n">output</span><span class="o">.</span><span class="n">tell</span><span class="p">()</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span>
|
||||
<span class="n">last_char</span> <span class="o">=</span> <span class="n">output</span><span class="o">.</span><span class="n">read</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">last_char</span> <span class="o">==</span> <span class="s2">"]"</span><span class="p">:</span>
|
||||
<span class="c1"># remove the trailing "\n]", leading "[\n", and replace with</span>
|
||||
<span class="c1"># ",\n"</span>
|
||||
<span class="n">output</span><span class="o">.</span><span class="n">seek</span><span class="p">(</span><span class="n">output</span><span class="o">.</span><span class="n">tell</span><span class="p">()</span> <span class="o">-</span> <span class="mi">2</span><span class="p">)</span>
|
||||
<span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s2">",</span><span class="se">\n</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="n">output_json</span> <span class="o">=</span> <span class="n">output_json</span><span class="p">[</span><span class="mi">2</span><span class="p">:]</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">output</span><span class="o">.</span><span class="n">seek</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
|
||||
<span class="n">output</span><span class="o">.</span><span class="n">truncate</span><span class="p">()</span>
|
||||
<span class="w"> </span><span class="sd">"""Append ``reports`` to a JSON array on disk, creating the file</span>
|
||||
<span class="sd"> if needed.</span>
|
||||
|
||||
<span class="sd"> Reads the existing array (if the file exists and parses cleanly),</span>
|
||||
<span class="sd"> merges the new reports onto the end, and rewrites the file as a</span>
|
||||
<span class="sd"> single valid JSON array. An earlier version of this used an</span>
|
||||
<span class="sd"> ``open(..., "a+")`` + ``seek()`` + overwrite pattern, but Python's</span>
|
||||
<span class="sd"> documentation is explicit that on POSIX, ``a`` / ``a+`` writes</span>
|
||||
<span class="sd"> *always* go to EOF regardless of seek position — so the second</span>
|
||||
<span class="sd"> call onto an existing file produced ``[...],\\n[...]``-style</span>
|
||||
<span class="sd"> corrupted output. Read-merge-write is the only way to get a valid</span>
|
||||
<span class="sd"> JSON array out of repeated appends.</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">reports</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="c1"># Don't create an empty-array file for an empty input; if a</span>
|
||||
<span class="c1"># file already exists, leave it alone.</span>
|
||||
<span class="k">return</span>
|
||||
|
||||
<span class="n">existing</span><span class="p">:</span> <span class="nb">list</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isfile</span><span class="p">(</span><span class="n">filename</span><span class="p">)</span> <span class="ow">and</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">getsize</span><span class="p">(</span><span class="n">filename</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="s2">"r"</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s2">"utf-8"</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
|
||||
<span class="n">loaded</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">read</span><span class="p">())</span>
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">loaded</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
|
||||
<span class="n">existing</span> <span class="o">=</span> <span class="n">loaded</span>
|
||||
<span class="k">except</span> <span class="p">(</span><span class="n">json</span><span class="o">.</span><span class="n">JSONDecodeError</span><span class="p">,</span> <span class="ne">OSError</span><span class="p">):</span>
|
||||
<span class="c1"># Corrupted or unreadable: overwrite cleanly rather than</span>
|
||||
<span class="c1"># silently fail to record.</span>
|
||||
<span class="n">existing</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
|
||||
<span class="n">merged</span> <span class="o">=</span> <span class="n">existing</span> <span class="o">+</span> <span class="nb">list</span><span class="p">(</span><span class="n">reports</span><span class="p">)</span>
|
||||
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="s2">"w"</span><span class="p">,</span> <span class="n">newline</span><span class="o">=</span><span class="s2">"</span><span class="se">\n</span><span class="s2">"</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s2">"utf-8"</span><span class="p">)</span> <span class="k">as</span> <span class="n">output</span><span class="p">:</span>
|
||||
<span class="n">json</span><span class="o">.</span><span class="n">dump</span><span class="p">(</span><span class="n">merged</span><span class="p">,</span> <span class="n">output</span><span class="p">,</span> <span class="n">ensure_ascii</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">indent</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span></div>
|
||||
|
||||
<span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">output_json</span><span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">append_csv</span><span class="p">(</span><span class="n">filename</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">csv</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||
@@ -2519,10 +2753,10 @@
|
||||
<span class="o">*</span><span class="p">,</span>
|
||||
<span class="n">output_directory</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"output"</span><span class="p">,</span>
|
||||
<span class="n">aggregate_json_filename</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"aggregate.json"</span><span class="p">,</span>
|
||||
<span class="n">forensic_json_filename</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"forensic.json"</span><span class="p">,</span>
|
||||
<span class="n">failure_json_filename</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"failure.json"</span><span class="p">,</span>
|
||||
<span class="n">smtp_tls_json_filename</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"smtp_tls.json"</span><span class="p">,</span>
|
||||
<span class="n">aggregate_csv_filename</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"aggregate.csv"</span><span class="p">,</span>
|
||||
<span class="n">forensic_csv_filename</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"forensic.csv"</span><span class="p">,</span>
|
||||
<span class="n">failure_csv_filename</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"failure.csv"</span><span class="p">,</span>
|
||||
<span class="n">smtp_tls_csv_filename</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"smtp_tls.csv"</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
@@ -2532,15 +2766,15 @@
|
||||
<span class="sd"> results: Parsing results</span>
|
||||
<span class="sd"> output_directory (str): The path to the directory to save in</span>
|
||||
<span class="sd"> aggregate_json_filename (str): Filename for the aggregate JSON file</span>
|
||||
<span class="sd"> forensic_json_filename (str): Filename for the forensic JSON file</span>
|
||||
<span class="sd"> failure_json_filename (str): Filename for the failure JSON file</span>
|
||||
<span class="sd"> smtp_tls_json_filename (str): Filename for the SMTP TLS JSON file</span>
|
||||
<span class="sd"> aggregate_csv_filename (str): Filename for the aggregate CSV file</span>
|
||||
<span class="sd"> forensic_csv_filename (str): Filename for the forensic CSV file</span>
|
||||
<span class="sd"> failure_csv_filename (str): Filename for the failure CSV file</span>
|
||||
<span class="sd"> smtp_tls_csv_filename (str): Filename for the SMTP TLS CSV file</span>
|
||||
<span class="sd"> """</span>
|
||||
|
||||
<span class="n">aggregate_reports</span> <span class="o">=</span> <span class="n">results</span><span class="p">[</span><span class="s2">"aggregate_reports"</span><span class="p">]</span>
|
||||
<span class="n">forensic_reports</span> <span class="o">=</span> <span class="n">results</span><span class="p">[</span><span class="s2">"forensic_reports"</span><span class="p">]</span>
|
||||
<span class="n">failure_reports</span> <span class="o">=</span> <span class="n">results</span><span class="p">[</span><span class="s2">"failure_reports"</span><span class="p">]</span>
|
||||
<span class="n">smtp_tls_reports</span> <span class="o">=</span> <span class="n">results</span><span class="p">[</span><span class="s2">"smtp_tls_reports"</span><span class="p">]</span>
|
||||
<span class="n">output_directory</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">expanduser</span><span class="p">(</span><span class="n">output_directory</span><span class="p">)</span>
|
||||
|
||||
@@ -2559,13 +2793,11 @@
|
||||
<span class="n">parsed_aggregate_reports_to_csv</span><span class="p">(</span><span class="n">aggregate_reports</span><span class="p">),</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">append_json</span><span class="p">(</span>
|
||||
<span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">output_directory</span><span class="p">,</span> <span class="n">forensic_json_filename</span><span class="p">),</span> <span class="n">forensic_reports</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">append_json</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">output_directory</span><span class="p">,</span> <span class="n">failure_json_filename</span><span class="p">),</span> <span class="n">failure_reports</span><span class="p">)</span>
|
||||
|
||||
<span class="n">append_csv</span><span class="p">(</span>
|
||||
<span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">output_directory</span><span class="p">,</span> <span class="n">forensic_csv_filename</span><span class="p">),</span>
|
||||
<span class="n">parsed_forensic_reports_to_csv</span><span class="p">(</span><span class="n">forensic_reports</span><span class="p">),</span>
|
||||
<span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">output_directory</span><span class="p">,</span> <span class="n">failure_csv_filename</span><span class="p">),</span>
|
||||
<span class="n">parsed_failure_reports_to_csv</span><span class="p">(</span><span class="n">failure_reports</span><span class="p">),</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">append_json</span><span class="p">(</span>
|
||||
@@ -2582,10 +2814,10 @@
|
||||
<span class="n">os</span><span class="o">.</span><span class="n">makedirs</span><span class="p">(</span><span class="n">samples_directory</span><span class="p">)</span>
|
||||
|
||||
<span class="n">sample_filenames</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="k">for</span> <span class="n">forensic_report</span> <span class="ow">in</span> <span class="n">forensic_reports</span><span class="p">:</span>
|
||||
<span class="n">sample</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"sample"</span><span class="p">]</span>
|
||||
<span class="k">for</span> <span class="n">failure_report</span> <span class="ow">in</span> <span class="n">failure_reports</span><span class="p">:</span>
|
||||
<span class="n">sample</span> <span class="o">=</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"sample"</span><span class="p">]</span>
|
||||
<span class="n">message_count</span> <span class="o">=</span> <span class="mi">0</span>
|
||||
<span class="n">parsed_sample</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">]</span>
|
||||
<span class="n">parsed_sample</span> <span class="o">=</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">]</span>
|
||||
<span class="n">subject</span> <span class="o">=</span> <span class="p">(</span>
|
||||
<span class="n">parsed_sample</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"filename_safe_subject"</span><span class="p">)</span>
|
||||
<span class="ow">or</span> <span class="n">parsed_sample</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"subject"</span><span class="p">)</span>
|
||||
@@ -2726,6 +2958,12 @@
|
||||
<span class="n">plain_message</span><span class="o">=</span><span class="n">message</span><span class="p">,</span>
|
||||
<span class="p">)</span></div>
|
||||
|
||||
|
||||
|
||||
<span class="c1"># Backward-compatible aliases</span>
|
||||
<span class="n">parse_forensic_report</span> <span class="o">=</span> <span class="n">parse_failure_report</span>
|
||||
<span class="n">parsed_forensic_reports_to_csv_rows</span> <span class="o">=</span> <span class="n">parsed_failure_reports_to_csv_rows</span>
|
||||
<span class="n">parsed_forensic_reports_to_csv</span> <span class="o">=</span> <span class="n">parsed_failure_reports_to_csv</span>
|
||||
</pre></div>
|
||||
|
||||
</div>
|
||||
|
||||
@@ -5,14 +5,14 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc.elastic — parsedmarc 9.11.2 documentation</title>
|
||||
<title>parsedmarc.elastic — parsedmarc 10.0.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<script src="../../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../../_static/documentation_options.js?v=de4344a5"></script>
|
||||
<script src="../../_static/documentation_options.js?v=335988e4"></script>
|
||||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="../../_static/js/theme.js"></script>
|
||||
@@ -95,6 +95,7 @@
|
||||
<span class="n">InnerDoc</span><span class="p">,</span>
|
||||
<span class="n">Integer</span><span class="p">,</span>
|
||||
<span class="n">Ip</span><span class="p">,</span>
|
||||
<span class="n">Keyword</span><span class="p">,</span>
|
||||
<span class="n">Nested</span><span class="p">,</span>
|
||||
<span class="n">Object</span><span class="p">,</span>
|
||||
<span class="n">Search</span><span class="p">,</span>
|
||||
@@ -103,7 +104,7 @@
|
||||
<span class="p">)</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">elasticsearch_dsl.search</span><span class="w"> </span><span class="kn">import</span> <span class="n">Q</span>
|
||||
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc</span><span class="w"> </span><span class="kn">import</span> <span class="n">InvalidForensicReport</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc</span><span class="w"> </span><span class="kn">import</span> <span class="n">InvalidFailureReport</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.log</span><span class="w"> </span><span class="kn">import</span> <span class="n">logger</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">human_timestamp_to_datetime</span>
|
||||
|
||||
@@ -115,6 +116,18 @@
|
||||
|
||||
|
||||
|
||||
<span class="c1"># Mirror of the ``serverless`` flag passed to ``set_hosts``; consulted by</span>
|
||||
<span class="c1"># ``create_indexes`` to strip settings Elastic Cloud Serverless rejects.</span>
|
||||
<span class="c1"># Module-level state is consistent with the existing ``connections.create_connection``</span>
|
||||
<span class="c1"># global the rest of this module relies on — there is a single default ES</span>
|
||||
<span class="c1"># connection per process.</span>
|
||||
<span class="n">_SERVERLESS</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
|
||||
<span class="c1"># Index settings rejected by Elastic Cloud Serverless with HTTP 400. Other</span>
|
||||
<span class="c1"># settings (e.g. ``refresh_interval``) are accepted and pass through.</span>
|
||||
<span class="n">_SERVERLESS_REJECTED_SETTINGS</span> <span class="o">=</span> <span class="nb">frozenset</span><span class="p">({</span><span class="s2">"number_of_shards"</span><span class="p">,</span> <span class="s2">"number_of_replicas"</span><span class="p">})</span>
|
||||
|
||||
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_PolicyOverride</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="nb">type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">comment</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
@@ -128,18 +141,23 @@
|
||||
<span class="n">sp</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">pct</span> <span class="o">=</span> <span class="n">Integer</span><span class="p">()</span>
|
||||
<span class="n">fo</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">np</span> <span class="o">=</span> <span class="n">Keyword</span><span class="p">()</span>
|
||||
<span class="n">testing</span> <span class="o">=</span> <span class="n">Keyword</span><span class="p">()</span>
|
||||
<span class="n">discovery_method</span> <span class="o">=</span> <span class="n">Keyword</span><span class="p">()</span>
|
||||
|
||||
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_DKIMResult</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="n">domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">selector</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">result</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">human_result</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
|
||||
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_SPFResult</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="n">domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">scope</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">results</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">human_result</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
|
||||
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_AggregateReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
@@ -147,6 +165,7 @@
|
||||
<span class="n">name</span> <span class="o">=</span> <span class="s2">"dmarc_aggregate"</span>
|
||||
|
||||
<span class="n">xml_schema</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">xml_namespace</span> <span class="o">=</span> <span class="n">Keyword</span><span class="p">()</span>
|
||||
<span class="n">org_name</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">org_email</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">org_extra_contact_info</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
@@ -178,17 +197,45 @@
|
||||
<span class="n">envelope_to</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">dkim_results</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_DKIMResult</span><span class="p">)</span>
|
||||
<span class="n">spf_results</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SPFResult</span><span class="p">)</span>
|
||||
<span class="n">np</span> <span class="o">=</span> <span class="n">Keyword</span><span class="p">()</span>
|
||||
<span class="n">testing</span> <span class="o">=</span> <span class="n">Keyword</span><span class="p">()</span>
|
||||
<span class="n">discovery_method</span> <span class="o">=</span> <span class="n">Keyword</span><span class="p">()</span>
|
||||
<span class="n">generator</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_policy_override</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">type_</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">comment</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">policy_overrides</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_PolicyOverride</span><span class="p">(</span><span class="nb">type</span><span class="o">=</span><span class="n">type_</span><span class="p">,</span> <span class="n">comment</span><span class="o">=</span><span class="n">comment</span><span class="p">))</span> <span class="c1"># pyright: ignore[reportCallIssue]</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_dkim_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">domain</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">selector</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">result</span><span class="p">:</span> <span class="n">_DKIMResult</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_dkim_result</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="p">,</span>
|
||||
<span class="n">domain</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
||||
<span class="n">selector</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
||||
<span class="n">result</span><span class="p">:</span> <span class="n">_DKIMResult</span><span class="p">,</span>
|
||||
<span class="n">human_result</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">dkim_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="n">_DKIMResult</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span> <span class="n">selector</span><span class="o">=</span><span class="n">selector</span><span class="p">,</span> <span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">)</span>
|
||||
<span class="n">_DKIMResult</span><span class="p">(</span>
|
||||
<span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span>
|
||||
<span class="n">selector</span><span class="o">=</span><span class="n">selector</span><span class="p">,</span>
|
||||
<span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">,</span>
|
||||
<span class="n">human_result</span><span class="o">=</span><span class="n">human_result</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span> <span class="c1"># pyright: ignore[reportCallIssue]</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_spf_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">domain</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">scope</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">result</span><span class="p">:</span> <span class="n">_SPFResult</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">spf_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_SPFResult</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span> <span class="n">scope</span><span class="o">=</span><span class="n">scope</span><span class="p">,</span> <span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">))</span> <span class="c1"># pyright: ignore[reportCallIssue]</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_spf_result</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="p">,</span>
|
||||
<span class="n">domain</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
||||
<span class="n">scope</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
||||
<span class="n">result</span><span class="p">:</span> <span class="n">_SPFResult</span><span class="p">,</span>
|
||||
<span class="n">human_result</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">spf_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="n">_SPFResult</span><span class="p">(</span>
|
||||
<span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span>
|
||||
<span class="n">scope</span><span class="o">=</span><span class="n">scope</span><span class="p">,</span>
|
||||
<span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">,</span>
|
||||
<span class="n">human_result</span><span class="o">=</span><span class="n">human_result</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span> <span class="c1"># pyright: ignore[reportCallIssue]</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">save</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span> <span class="c1"># pyright: ignore[reportIncompatibleMethodOverride]</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">passed_dmarc</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
@@ -208,7 +255,7 @@
|
||||
<span class="n">sha256</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
|
||||
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_ForensicSampleDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_FailureSampleDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="n">raw</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">headers</span> <span class="o">=</span> <span class="n">Object</span><span class="p">()</span>
|
||||
<span class="n">headers_only</span> <span class="o">=</span> <span class="n">Boolean</span><span class="p">()</span>
|
||||
@@ -245,9 +292,9 @@
|
||||
<span class="p">)</span> <span class="c1"># pyright: ignore[reportCallIssue]</span>
|
||||
|
||||
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_ForensicReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_FailureReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">Index</span><span class="p">:</span>
|
||||
<span class="n">name</span> <span class="o">=</span> <span class="s2">"dmarc_forensic"</span>
|
||||
<span class="n">name</span> <span class="o">=</span> <span class="s2">"dmarc_failure"</span>
|
||||
|
||||
<span class="n">feedback_type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">user_agent</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
@@ -268,7 +315,7 @@
|
||||
<span class="n">source_auth_failures</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">dkim_domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">original_rcpt_to</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">sample</span> <span class="o">=</span> <span class="n">Object</span><span class="p">(</span><span class="n">_ForensicSampleDoc</span><span class="p">)</span>
|
||||
<span class="n">sample</span> <span class="o">=</span> <span class="n">Object</span><span class="p">(</span><span class="n">_FailureSampleDoc</span><span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_SMTPTLSFailureDetailsDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
@@ -369,6 +416,7 @@
|
||||
<span class="n">password</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">api_key</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">timeout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">60.0</span><span class="p">,</span>
|
||||
<span class="n">serverless</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Sets the Elasticsearch hosts to use</span>
|
||||
@@ -382,7 +430,14 @@
|
||||
<span class="sd"> password (str): The password to use for authentication</span>
|
||||
<span class="sd"> api_key (str): The Base64 encoded API key to use for authentication</span>
|
||||
<span class="sd"> timeout (float): Timeout in seconds</span>
|
||||
<span class="sd"> serverless (bool): Target an Elastic Cloud Serverless project. When True,</span>
|
||||
<span class="sd"> ``create_indexes`` strips ``number_of_shards`` / ``number_of_replicas``</span>
|
||||
<span class="sd"> from its settings (which Serverless rejects with HTTP 400) and passes</span>
|
||||
<span class="sd"> any other settings through unchanged.</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="c1"># Module-global; see the _SERVERLESS comment at the top of the module.</span>
|
||||
<span class="k">global</span> <span class="n">_SERVERLESS</span>
|
||||
<span class="n">_SERVERLESS</span> <span class="o">=</span> <span class="n">serverless</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">hosts</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
|
||||
<span class="n">hosts</span> <span class="o">=</span> <span class="p">[</span><span class="n">hosts</span><span class="p">]</span>
|
||||
<span class="n">conn_params</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"hosts"</span><span class="p">:</span> <span class="n">hosts</span><span class="p">,</span> <span class="s2">"timeout"</span><span class="p">:</span> <span class="n">timeout</span><span class="p">}</span>
|
||||
@@ -410,18 +465,28 @@
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> names (list): A list of index names</span>
|
||||
<span class="sd"> settings (dict): Index settings</span>
|
||||
|
||||
<span class="sd"> settings (dict): Index settings. In Serverless mode, keys in</span>
|
||||
<span class="sd"> ``_SERVERLESS_REJECTED_SETTINGS`` are filtered out and the</span>
|
||||
<span class="sd"> remaining keys are passed through; defaults are skipped entirely.</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">if</span> <span class="n">settings</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">effective_settings</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span>
|
||||
<span class="p">{}</span> <span class="k">if</span> <span class="n">_SERVERLESS</span> <span class="k">else</span> <span class="p">{</span><span class="s2">"number_of_shards"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span> <span class="s2">"number_of_replicas"</span><span class="p">:</span> <span class="mi">0</span><span class="p">}</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">elif</span> <span class="n">_SERVERLESS</span><span class="p">:</span>
|
||||
<span class="n">effective_settings</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="n">k</span><span class="p">:</span> <span class="n">v</span> <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">settings</span><span class="o">.</span><span class="n">items</span><span class="p">()</span> <span class="k">if</span> <span class="n">k</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">_SERVERLESS_REJECTED_SETTINGS</span>
|
||||
<span class="p">}</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">effective_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="n">settings</span><span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">names</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="n">Index</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">index</span><span class="o">.</span><span class="n">exists</span><span class="p">():</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Creating Elasticsearch index: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">name</span><span class="p">))</span>
|
||||
<span class="k">if</span> <span class="n">settings</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">index</span><span class="o">.</span><span class="n">settings</span><span class="p">(</span><span class="n">number_of_shards</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">number_of_replicas</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">index</span><span class="o">.</span><span class="n">settings</span><span class="p">(</span><span class="o">**</span><span class="n">settings</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">effective_settings</span><span class="p">:</span>
|
||||
<span class="n">index</span><span class="o">.</span><span class="n">settings</span><span class="p">(</span><span class="o">**</span><span class="n">effective_settings</span><span class="p">)</span>
|
||||
<span class="n">index</span><span class="o">.</span><span class="n">create</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="s2">"Elasticsearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
@@ -432,20 +497,20 @@
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.migrate_indexes">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">migrate_indexes</span><span class="p">(</span>
|
||||
<span class="n">aggregate_indexes</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">forensic_indexes</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">failure_indexes</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Updates index mappings</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> aggregate_indexes (list): A list of aggregate index names</span>
|
||||
<span class="sd"> forensic_indexes (list): A list of forensic index names</span>
|
||||
<span class="sd"> failure_indexes (list): A list of failure index names</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">version</span> <span class="o">=</span> <span class="mi">2</span>
|
||||
<span class="k">if</span> <span class="n">aggregate_indexes</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">aggregate_indexes</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="k">if</span> <span class="n">forensic_indexes</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">forensic_indexes</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="k">if</span> <span class="n">failure_indexes</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">failure_indexes</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="k">for</span> <span class="n">aggregate_index_name</span> <span class="ow">in</span> <span class="n">aggregate_indexes</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">Index</span><span class="p">(</span><span class="n">aggregate_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">exists</span><span class="p">():</span>
|
||||
<span class="k">continue</span>
|
||||
@@ -475,7 +540,7 @@
|
||||
<span class="n">reindex</span><span class="p">(</span><span class="n">connections</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(),</span> <span class="n">aggregate_index_name</span><span class="p">,</span> <span class="n">new_index_name</span><span class="p">)</span> <span class="c1"># pyright: ignore[reportArgumentType]</span>
|
||||
<span class="n">Index</span><span class="p">(</span><span class="n">aggregate_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">delete</span><span class="p">()</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">forensic_index</span> <span class="ow">in</span> <span class="n">forensic_indexes</span><span class="p">:</span>
|
||||
<span class="k">for</span> <span class="n">failure_index</span> <span class="ow">in</span> <span class="n">failure_indexes</span><span class="p">:</span>
|
||||
<span class="k">pass</span></div>
|
||||
|
||||
|
||||
@@ -494,7 +559,7 @@
|
||||
<span class="sd"> Saves a parsed DMARC aggregate report to Elasticsearch</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> aggregate_report (dict): A parsed forensic report</span>
|
||||
<span class="sd"> aggregate_report (dict): A parsed aggregate report</span>
|
||||
<span class="sd"> index_suffix (str): The suffix of the name of the index to save to</span>
|
||||
<span class="sd"> index_prefix (str): The prefix of the name of the index to save to</span>
|
||||
<span class="sd"> monthly_indexes (bool): Use monthly indexes instead of daily indexes</span>
|
||||
@@ -562,6 +627,9 @@
|
||||
<span class="n">sp</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"sp"</span><span class="p">],</span>
|
||||
<span class="n">pct</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"pct"</span><span class="p">],</span>
|
||||
<span class="n">fo</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"fo"</span><span class="p">],</span>
|
||||
<span class="n">np</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"np"</span><span class="p">),</span>
|
||||
<span class="n">testing</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"testing"</span><span class="p">),</span>
|
||||
<span class="n">discovery_method</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"discovery_method"</span><span class="p">),</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">record</span> <span class="ow">in</span> <span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"records"</span><span class="p">]:</span>
|
||||
@@ -578,6 +646,7 @@
|
||||
<span class="n">date_range</span> <span class="o">=</span> <span class="p">[</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">],</span> <span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">]]</span>
|
||||
<span class="n">agg_doc</span> <span class="o">=</span> <span class="n">_AggregateReportDoc</span><span class="p">(</span>
|
||||
<span class="n">xml_schema</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"xml_schema"</span><span class="p">],</span>
|
||||
<span class="n">xml_namespace</span><span class="o">=</span><span class="n">aggregate_report</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"xml_namespace"</span><span class="p">),</span>
|
||||
<span class="n">org_name</span><span class="o">=</span><span class="n">metadata</span><span class="p">[</span><span class="s2">"org_name"</span><span class="p">],</span>
|
||||
<span class="n">org_email</span><span class="o">=</span><span class="n">metadata</span><span class="p">[</span><span class="s2">"org_email"</span><span class="p">],</span>
|
||||
<span class="n">org_extra_contact_info</span><span class="o">=</span><span class="n">metadata</span><span class="p">[</span><span class="s2">"org_extra_contact_info"</span><span class="p">],</span>
|
||||
@@ -606,6 +675,12 @@
|
||||
<span class="n">header_from</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"header_from"</span><span class="p">],</span>
|
||||
<span class="n">envelope_from</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"envelope_from"</span><span class="p">],</span>
|
||||
<span class="n">envelope_to</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"envelope_to"</span><span class="p">],</span>
|
||||
<span class="n">np</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"np"</span><span class="p">),</span>
|
||||
<span class="n">testing</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"testing"</span><span class="p">),</span>
|
||||
<span class="n">discovery_method</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
|
||||
<span class="s2">"discovery_method"</span>
|
||||
<span class="p">),</span>
|
||||
<span class="n">generator</span><span class="o">=</span><span class="n">metadata</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"generator"</span><span class="p">),</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">override</span> <span class="ow">in</span> <span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"policy_override_reasons"</span><span class="p">]:</span>
|
||||
@@ -618,6 +693,7 @@
|
||||
<span class="n">domain</span><span class="o">=</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">],</span>
|
||||
<span class="n">selector</span><span class="o">=</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">"selector"</span><span class="p">],</span>
|
||||
<span class="n">result</span><span class="o">=</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">],</span>
|
||||
<span class="n">human_result</span><span class="o">=</span><span class="n">dkim_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"human_result"</span><span class="p">),</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">spf_result</span> <span class="ow">in</span> <span class="n">record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">]:</span>
|
||||
@@ -625,6 +701,7 @@
|
||||
<span class="n">domain</span><span class="o">=</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">],</span>
|
||||
<span class="n">scope</span><span class="o">=</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">"scope"</span><span class="p">],</span>
|
||||
<span class="n">result</span><span class="o">=</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">],</span>
|
||||
<span class="n">human_result</span><span class="o">=</span><span class="n">spf_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"human_result"</span><span class="p">),</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"dmarc_aggregate"</span>
|
||||
@@ -647,10 +724,10 @@
|
||||
|
||||
|
||||
|
||||
<div class="viewcode-block" id="save_forensic_report_to_elasticsearch">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.save_forensic_report_to_elasticsearch">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">save_forensic_report_to_elasticsearch</span><span class="p">(</span>
|
||||
<span class="n">forensic_report</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">],</span>
|
||||
<div class="viewcode-block" id="save_failure_report_to_elasticsearch">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.save_failure_report_to_elasticsearch">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">save_failure_report_to_elasticsearch</span><span class="p">(</span>
|
||||
<span class="n">failure_report</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">],</span>
|
||||
<span class="n">index_suffix</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">index_prefix</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">monthly_indexes</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
||||
@@ -658,10 +735,10 @@
|
||||
<span class="n">number_of_replicas</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">0</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Saves a parsed DMARC forensic report to Elasticsearch</span>
|
||||
<span class="sd"> Saves a parsed DMARC failure report to Elasticsearch</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> forensic_report (dict): A parsed forensic report</span>
|
||||
<span class="sd"> failure_report (dict): A parsed failure report</span>
|
||||
<span class="sd"> index_suffix (str): The suffix of the name of the index to save to</span>
|
||||
<span class="sd"> index_prefix (str): The prefix of the name of the index to save to</span>
|
||||
<span class="sd"> monthly_indexes (bool): Use monthly indexes instead of daily</span>
|
||||
@@ -674,26 +751,28 @@
|
||||
<span class="sd"> AlreadySaved</span>
|
||||
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Saving forensic report to Elasticsearch"</span><span class="p">)</span>
|
||||
<span class="n">forensic_report</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Saving failure report to Elasticsearch"</span><span class="p">)</span>
|
||||
<span class="n">failure_report</span> <span class="o">=</span> <span class="n">failure_report</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
<span class="n">sample_date</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="k">if</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"date"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">sample_date</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"date"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"date"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">sample_date</span> <span class="o">=</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"date"</span><span class="p">]</span>
|
||||
<span class="n">sample_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">sample_date</span><span class="p">)</span>
|
||||
<span class="n">original_headers</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"headers"</span><span class="p">]</span>
|
||||
<span class="n">original_headers</span> <span class="o">=</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"headers"</span><span class="p">]</span>
|
||||
<span class="n">headers</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
|
||||
<span class="k">for</span> <span class="n">original_header</span> <span class="ow">in</span> <span class="n">original_headers</span><span class="p">:</span>
|
||||
<span class="n">headers</span><span class="p">[</span><span class="n">original_header</span><span class="o">.</span><span class="n">lower</span><span class="p">()]</span> <span class="o">=</span> <span class="n">original_headers</span><span class="p">[</span><span class="n">original_header</span><span class="p">]</span>
|
||||
|
||||
<span class="n">arrival_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"arrival_date_utc"</span><span class="p">])</span>
|
||||
<span class="n">arrival_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"arrival_date_utc"</span><span class="p">])</span>
|
||||
<span class="n">arrival_date_epoch_milliseconds</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">arrival_date</span><span class="o">.</span><span class="n">timestamp</span><span class="p">()</span> <span class="o">*</span> <span class="mi">1000</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">index_suffix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"dmarc_forensic_</span><span class="si">{0}</span><span class="s2">*"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"dmarc_failure_</span><span class="si">{0}</span><span class="s2">*,dmarc_forensic_</span><span class="si">{0}</span><span class="s2">*"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"dmarc_forensic*"</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"dmarc_failure*,dmarc_forensic*"</span>
|
||||
<span class="k">if</span> <span class="n">index_prefix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">search_index</span><span class="p">)</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">","</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
|
||||
<span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">part</span><span class="p">)</span> <span class="k">for</span> <span class="n">part</span> <span class="ow">in</span> <span class="n">search_index</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">","</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="n">search_index</span><span class="p">)</span>
|
||||
<span class="n">q</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">arrival_date</span><span class="o">=</span><span class="n">arrival_date_epoch_milliseconds</span><span class="p">)))</span> <span class="c1"># pyright: ignore[reportArgumentType]</span>
|
||||
|
||||
@@ -734,67 +813,67 @@
|
||||
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">existing</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">AlreadySaved</span><span class="p">(</span>
|
||||
<span class="s2">"A forensic sample to </span><span class="si">{0}</span><span class="s2"> from </span><span class="si">{1}</span><span class="s2"> "</span>
|
||||
<span class="s2">"A failure sample to </span><span class="si">{0}</span><span class="s2"> from </span><span class="si">{1}</span><span class="s2"> "</span>
|
||||
<span class="s2">"with a subject of </span><span class="si">{2}</span><span class="s2"> and arrival date of </span><span class="si">{3}</span><span class="s2"> "</span>
|
||||
<span class="s2">"already exists in "</span>
|
||||
<span class="s2">"Elasticsearch"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">to_</span><span class="p">,</span> <span class="n">from_</span><span class="p">,</span> <span class="n">subject</span><span class="p">,</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"arrival_date_utc"</span><span class="p">]</span>
|
||||
<span class="n">to_</span><span class="p">,</span> <span class="n">from_</span><span class="p">,</span> <span class="n">subject</span><span class="p">,</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"arrival_date_utc"</span><span class="p">]</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">parsed_sample</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">]</span>
|
||||
<span class="n">sample</span> <span class="o">=</span> <span class="n">_ForensicSampleDoc</span><span class="p">(</span>
|
||||
<span class="n">raw</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"sample"</span><span class="p">],</span>
|
||||
<span class="n">parsed_sample</span> <span class="o">=</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">]</span>
|
||||
<span class="n">sample</span> <span class="o">=</span> <span class="n">_FailureSampleDoc</span><span class="p">(</span>
|
||||
<span class="n">raw</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"sample"</span><span class="p">],</span>
|
||||
<span class="n">headers</span><span class="o">=</span><span class="n">headers</span><span class="p">,</span>
|
||||
<span class="n">headers_only</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"sample_headers_only"</span><span class="p">],</span>
|
||||
<span class="n">headers_only</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"sample_headers_only"</span><span class="p">],</span>
|
||||
<span class="n">date</span><span class="o">=</span><span class="n">sample_date</span><span class="p">,</span>
|
||||
<span class="n">subject</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"subject"</span><span class="p">],</span>
|
||||
<span class="n">subject</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"subject"</span><span class="p">],</span>
|
||||
<span class="n">filename_safe_subject</span><span class="o">=</span><span class="n">parsed_sample</span><span class="p">[</span><span class="s2">"filename_safe_subject"</span><span class="p">],</span>
|
||||
<span class="n">body</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"body"</span><span class="p">],</span>
|
||||
<span class="n">body</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"body"</span><span class="p">],</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"to"</span><span class="p">]:</span>
|
||||
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"to"</span><span class="p">]:</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_to</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">])</span>
|
||||
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"reply_to"</span><span class="p">]:</span>
|
||||
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"reply_to"</span><span class="p">]:</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_reply_to</span><span class="p">(</span>
|
||||
<span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">]</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"cc"</span><span class="p">]:</span>
|
||||
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"cc"</span><span class="p">]:</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_cc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">])</span>
|
||||
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"bcc"</span><span class="p">]:</span>
|
||||
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"bcc"</span><span class="p">]:</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_bcc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">])</span>
|
||||
<span class="k">for</span> <span class="n">attachment</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"attachments"</span><span class="p">]:</span>
|
||||
<span class="k">for</span> <span class="n">attachment</span> <span class="ow">in</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"attachments"</span><span class="p">]:</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_attachment</span><span class="p">(</span>
|
||||
<span class="n">filename</span><span class="o">=</span><span class="n">attachment</span><span class="p">[</span><span class="s2">"filename"</span><span class="p">],</span>
|
||||
<span class="n">content_type</span><span class="o">=</span><span class="n">attachment</span><span class="p">[</span><span class="s2">"mail_content_type"</span><span class="p">],</span>
|
||||
<span class="n">sha256</span><span class="o">=</span><span class="n">attachment</span><span class="p">[</span><span class="s2">"sha256"</span><span class="p">],</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">forensic_doc</span> <span class="o">=</span> <span class="n">_ForensicReportDoc</span><span class="p">(</span>
|
||||
<span class="n">feedback_type</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"feedback_type"</span><span class="p">],</span>
|
||||
<span class="n">user_agent</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"user_agent"</span><span class="p">],</span>
|
||||
<span class="n">version</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"version"</span><span class="p">],</span>
|
||||
<span class="n">original_mail_from</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"original_mail_from"</span><span class="p">],</span>
|
||||
<span class="n">failure_doc</span> <span class="o">=</span> <span class="n">_FailureReportDoc</span><span class="p">(</span>
|
||||
<span class="n">feedback_type</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"feedback_type"</span><span class="p">],</span>
|
||||
<span class="n">user_agent</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"user_agent"</span><span class="p">],</span>
|
||||
<span class="n">version</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"version"</span><span class="p">],</span>
|
||||
<span class="n">original_mail_from</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"original_mail_from"</span><span class="p">],</span>
|
||||
<span class="n">arrival_date</span><span class="o">=</span><span class="n">arrival_date_epoch_milliseconds</span><span class="p">,</span>
|
||||
<span class="n">domain</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"reported_domain"</span><span class="p">],</span>
|
||||
<span class="n">original_envelope_id</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"original_envelope_id"</span><span class="p">],</span>
|
||||
<span class="n">authentication_results</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"authentication_results"</span><span class="p">],</span>
|
||||
<span class="n">delivery_results</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"delivery_result"</span><span class="p">],</span>
|
||||
<span class="n">source_ip_address</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"ip_address"</span><span class="p">],</span>
|
||||
<span class="n">source_country</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"country"</span><span class="p">],</span>
|
||||
<span class="n">source_reverse_dns</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"reverse_dns"</span><span class="p">],</span>
|
||||
<span class="n">source_base_domain</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"base_domain"</span><span class="p">],</span>
|
||||
<span class="n">source_asn</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"asn"</span><span class="p">],</span>
|
||||
<span class="n">source_as_name</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"as_name"</span><span class="p">],</span>
|
||||
<span class="n">source_as_domain</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"as_domain"</span><span class="p">],</span>
|
||||
<span class="n">authentication_mechanisms</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"authentication_mechanisms"</span><span class="p">],</span>
|
||||
<span class="n">auth_failure</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"auth_failure"</span><span class="p">],</span>
|
||||
<span class="n">dkim_domain</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"dkim_domain"</span><span class="p">],</span>
|
||||
<span class="n">original_rcpt_to</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"original_rcpt_to"</span><span class="p">],</span>
|
||||
<span class="n">domain</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"reported_domain"</span><span class="p">],</span>
|
||||
<span class="n">original_envelope_id</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"original_envelope_id"</span><span class="p">],</span>
|
||||
<span class="n">authentication_results</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"authentication_results"</span><span class="p">],</span>
|
||||
<span class="n">delivery_results</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"delivery_result"</span><span class="p">],</span>
|
||||
<span class="n">source_ip_address</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"ip_address"</span><span class="p">],</span>
|
||||
<span class="n">source_country</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"country"</span><span class="p">],</span>
|
||||
<span class="n">source_reverse_dns</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"reverse_dns"</span><span class="p">],</span>
|
||||
<span class="n">source_base_domain</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"base_domain"</span><span class="p">],</span>
|
||||
<span class="n">source_asn</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"asn"</span><span class="p">],</span>
|
||||
<span class="n">source_as_name</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"as_name"</span><span class="p">],</span>
|
||||
<span class="n">source_as_domain</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"as_domain"</span><span class="p">],</span>
|
||||
<span class="n">authentication_mechanisms</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"authentication_mechanisms"</span><span class="p">],</span>
|
||||
<span class="n">auth_failure</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"auth_failure"</span><span class="p">],</span>
|
||||
<span class="n">dkim_domain</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"dkim_domain"</span><span class="p">],</span>
|
||||
<span class="n">original_rcpt_to</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"original_rcpt_to"</span><span class="p">],</span>
|
||||
<span class="n">sample</span><span class="o">=</span><span class="n">sample</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"dmarc_forensic"</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"dmarc_failure"</span>
|
||||
<span class="k">if</span> <span class="n">index_suffix</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">_</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">index_prefix</span><span class="p">:</span>
|
||||
@@ -808,14 +887,14 @@
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span> <span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">create_indexes</span><span class="p">([</span><span class="n">index</span><span class="p">],</span> <span class="n">index_settings</span><span class="p">)</span>
|
||||
<span class="n">forensic_doc</span><span class="o">.</span><span class="n">meta</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">index</span> <span class="c1"># pyright: ignore[reportAttributeAccessIssue, reportOptionalMemberAccess]</span>
|
||||
<span class="n">failure_doc</span><span class="o">.</span><span class="n">meta</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">index</span> <span class="c1"># pyright: ignore[reportAttributeAccessIssue, reportOptionalMemberAccess]</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">forensic_doc</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
|
||||
<span class="n">failure_doc</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="s2">"Elasticsearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
||||
<span class="k">except</span> <span class="ne">KeyError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidForensicReport</span><span class="p">(</span>
|
||||
<span class="s2">"Forensic report missing required field: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidFailureReport</span><span class="p">(</span>
|
||||
<span class="s2">"Failure report missing required field: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="p">)</span></div>
|
||||
|
||||
|
||||
@@ -973,6 +1052,12 @@
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="s2">"Elasticsearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
|
||||
|
||||
|
||||
<span class="c1"># Backward-compatible aliases</span>
|
||||
<span class="n">_ForensicSampleDoc</span> <span class="o">=</span> <span class="n">_FailureSampleDoc</span>
|
||||
<span class="n">_ForensicReportDoc</span> <span class="o">=</span> <span class="n">_FailureReportDoc</span>
|
||||
<span class="n">save_forensic_report_to_elasticsearch</span> <span class="o">=</span> <span class="n">save_failure_report_to_elasticsearch</span>
|
||||
</pre></div>
|
||||
|
||||
</div>
|
||||
|
||||
@@ -5,14 +5,14 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc.opensearch — parsedmarc 9.11.2 documentation</title>
|
||||
<title>parsedmarc.opensearch — parsedmarc 10.0.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<script src="../../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../../_static/documentation_options.js?v=de4344a5"></script>
|
||||
<script src="../../_static/documentation_options.js?v=335988e4"></script>
|
||||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="../../_static/js/theme.js"></script>
|
||||
@@ -96,6 +96,7 @@
|
||||
<span class="n">InnerDoc</span><span class="p">,</span>
|
||||
<span class="n">Integer</span><span class="p">,</span>
|
||||
<span class="n">Ip</span><span class="p">,</span>
|
||||
<span class="n">Keyword</span><span class="p">,</span>
|
||||
<span class="n">Nested</span><span class="p">,</span>
|
||||
<span class="n">Object</span><span class="p">,</span>
|
||||
<span class="n">Q</span><span class="p">,</span>
|
||||
@@ -106,7 +107,7 @@
|
||||
<span class="p">)</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">opensearchpy.helpers</span><span class="w"> </span><span class="kn">import</span> <span class="n">reindex</span>
|
||||
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc</span><span class="w"> </span><span class="kn">import</span> <span class="n">InvalidForensicReport</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc</span><span class="w"> </span><span class="kn">import</span> <span class="n">InvalidFailureReport</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.log</span><span class="w"> </span><span class="kn">import</span> <span class="n">logger</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">human_timestamp_to_datetime</span>
|
||||
|
||||
@@ -131,18 +132,23 @@
|
||||
<span class="n">sp</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">pct</span> <span class="o">=</span> <span class="n">Integer</span><span class="p">()</span>
|
||||
<span class="n">fo</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">np</span> <span class="o">=</span> <span class="n">Keyword</span><span class="p">()</span>
|
||||
<span class="n">testing</span> <span class="o">=</span> <span class="n">Keyword</span><span class="p">()</span>
|
||||
<span class="n">discovery_method</span> <span class="o">=</span> <span class="n">Keyword</span><span class="p">()</span>
|
||||
|
||||
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_DKIMResult</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="n">domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">selector</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">result</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">human_result</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
|
||||
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_SPFResult</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="n">domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">scope</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">results</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">human_result</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
|
||||
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_AggregateReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
@@ -150,6 +156,7 @@
|
||||
<span class="n">name</span> <span class="o">=</span> <span class="s2">"dmarc_aggregate"</span>
|
||||
|
||||
<span class="n">xml_schema</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">xml_namespace</span> <span class="o">=</span> <span class="n">Keyword</span><span class="p">()</span>
|
||||
<span class="n">org_name</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">org_email</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">org_extra_contact_info</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
@@ -181,17 +188,45 @@
|
||||
<span class="n">envelope_to</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">dkim_results</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_DKIMResult</span><span class="p">)</span>
|
||||
<span class="n">spf_results</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SPFResult</span><span class="p">)</span>
|
||||
<span class="n">np</span> <span class="o">=</span> <span class="n">Keyword</span><span class="p">()</span>
|
||||
<span class="n">testing</span> <span class="o">=</span> <span class="n">Keyword</span><span class="p">()</span>
|
||||
<span class="n">discovery_method</span> <span class="o">=</span> <span class="n">Keyword</span><span class="p">()</span>
|
||||
<span class="n">generator</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_policy_override</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">type_</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">comment</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">policy_overrides</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_PolicyOverride</span><span class="p">(</span><span class="nb">type</span><span class="o">=</span><span class="n">type_</span><span class="p">,</span> <span class="n">comment</span><span class="o">=</span><span class="n">comment</span><span class="p">))</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_dkim_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">domain</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">selector</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">result</span><span class="p">:</span> <span class="n">_DKIMResult</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_dkim_result</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="p">,</span>
|
||||
<span class="n">domain</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
||||
<span class="n">selector</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
||||
<span class="n">result</span><span class="p">:</span> <span class="n">_DKIMResult</span><span class="p">,</span>
|
||||
<span class="n">human_result</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">dkim_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="n">_DKIMResult</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span> <span class="n">selector</span><span class="o">=</span><span class="n">selector</span><span class="p">,</span> <span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">)</span>
|
||||
<span class="n">_DKIMResult</span><span class="p">(</span>
|
||||
<span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span>
|
||||
<span class="n">selector</span><span class="o">=</span><span class="n">selector</span><span class="p">,</span>
|
||||
<span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">,</span>
|
||||
<span class="n">human_result</span><span class="o">=</span><span class="n">human_result</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_spf_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">domain</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">scope</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">result</span><span class="p">:</span> <span class="n">_SPFResult</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">spf_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_SPFResult</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span> <span class="n">scope</span><span class="o">=</span><span class="n">scope</span><span class="p">,</span> <span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">))</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_spf_result</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="p">,</span>
|
||||
<span class="n">domain</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
||||
<span class="n">scope</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
||||
<span class="n">result</span><span class="p">:</span> <span class="n">_SPFResult</span><span class="p">,</span>
|
||||
<span class="n">human_result</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">spf_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="n">_SPFResult</span><span class="p">(</span>
|
||||
<span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span>
|
||||
<span class="n">scope</span><span class="o">=</span><span class="n">scope</span><span class="p">,</span>
|
||||
<span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">,</span>
|
||||
<span class="n">human_result</span><span class="o">=</span><span class="n">human_result</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">save</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span> <span class="c1"># pyright: ignore[reportIncompatibleMethodOverride]</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">passed_dmarc</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
@@ -211,7 +246,7 @@
|
||||
<span class="n">sha256</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
|
||||
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_ForensicSampleDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_FailureSampleDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="n">raw</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">headers</span> <span class="o">=</span> <span class="n">Object</span><span class="p">()</span>
|
||||
<span class="n">headers_only</span> <span class="o">=</span> <span class="n">Boolean</span><span class="p">()</span>
|
||||
@@ -248,9 +283,9 @@
|
||||
<span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_ForensicReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_FailureReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">Index</span><span class="p">:</span>
|
||||
<span class="n">name</span> <span class="o">=</span> <span class="s2">"dmarc_forensic"</span>
|
||||
<span class="n">name</span> <span class="o">=</span> <span class="s2">"dmarc_failure"</span>
|
||||
|
||||
<span class="n">feedback_type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">user_agent</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
@@ -271,7 +306,7 @@
|
||||
<span class="n">source_auth_failures</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">dkim_domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">original_rcpt_to</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">sample</span> <span class="o">=</span> <span class="n">Object</span><span class="p">(</span><span class="n">_ForensicSampleDoc</span><span class="p">)</span>
|
||||
<span class="n">sample</span> <span class="o">=</span> <span class="n">Object</span><span class="p">(</span><span class="n">_FailureSampleDoc</span><span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_SMTPTLSFailureDetailsDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
@@ -462,20 +497,20 @@
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.migrate_indexes">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">migrate_indexes</span><span class="p">(</span>
|
||||
<span class="n">aggregate_indexes</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">forensic_indexes</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">failure_indexes</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Updates index mappings</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> aggregate_indexes (list): A list of aggregate index names</span>
|
||||
<span class="sd"> forensic_indexes (list): A list of forensic index names</span>
|
||||
<span class="sd"> failure_indexes (list): A list of failure index names</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">version</span> <span class="o">=</span> <span class="mi">2</span>
|
||||
<span class="k">if</span> <span class="n">aggregate_indexes</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">aggregate_indexes</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="k">if</span> <span class="n">forensic_indexes</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">forensic_indexes</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="k">if</span> <span class="n">failure_indexes</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">failure_indexes</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="k">for</span> <span class="n">aggregate_index_name</span> <span class="ow">in</span> <span class="n">aggregate_indexes</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">Index</span><span class="p">(</span><span class="n">aggregate_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">exists</span><span class="p">():</span>
|
||||
<span class="k">continue</span>
|
||||
@@ -505,7 +540,7 @@
|
||||
<span class="n">reindex</span><span class="p">(</span><span class="n">connections</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(),</span> <span class="n">aggregate_index_name</span><span class="p">,</span> <span class="n">new_index_name</span><span class="p">)</span>
|
||||
<span class="n">Index</span><span class="p">(</span><span class="n">aggregate_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">delete</span><span class="p">()</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">forensic_index</span> <span class="ow">in</span> <span class="n">forensic_indexes</span><span class="p">:</span>
|
||||
<span class="k">for</span> <span class="n">failure_index</span> <span class="ow">in</span> <span class="n">failure_indexes</span><span class="p">:</span>
|
||||
<span class="k">pass</span></div>
|
||||
|
||||
|
||||
@@ -524,7 +559,7 @@
|
||||
<span class="sd"> Saves a parsed DMARC aggregate report to OpenSearch</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> aggregate_report (dict): A parsed forensic report</span>
|
||||
<span class="sd"> aggregate_report (dict): A parsed aggregate report</span>
|
||||
<span class="sd"> index_suffix (str): The suffix of the name of the index to save to</span>
|
||||
<span class="sd"> index_prefix (str): The prefix of the name of the index to save to</span>
|
||||
<span class="sd"> monthly_indexes (bool): Use monthly indexes instead of daily indexes</span>
|
||||
@@ -592,6 +627,9 @@
|
||||
<span class="n">sp</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"sp"</span><span class="p">],</span>
|
||||
<span class="n">pct</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"pct"</span><span class="p">],</span>
|
||||
<span class="n">fo</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"fo"</span><span class="p">],</span>
|
||||
<span class="n">np</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"np"</span><span class="p">),</span>
|
||||
<span class="n">testing</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"testing"</span><span class="p">),</span>
|
||||
<span class="n">discovery_method</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"discovery_method"</span><span class="p">),</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">record</span> <span class="ow">in</span> <span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"records"</span><span class="p">]:</span>
|
||||
@@ -608,6 +646,7 @@
|
||||
<span class="n">date_range</span> <span class="o">=</span> <span class="p">[</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">],</span> <span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">]]</span>
|
||||
<span class="n">agg_doc</span> <span class="o">=</span> <span class="n">_AggregateReportDoc</span><span class="p">(</span>
|
||||
<span class="n">xml_schema</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"xml_schema"</span><span class="p">],</span>
|
||||
<span class="n">xml_namespace</span><span class="o">=</span><span class="n">aggregate_report</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"xml_namespace"</span><span class="p">),</span>
|
||||
<span class="n">org_name</span><span class="o">=</span><span class="n">metadata</span><span class="p">[</span><span class="s2">"org_name"</span><span class="p">],</span>
|
||||
<span class="n">org_email</span><span class="o">=</span><span class="n">metadata</span><span class="p">[</span><span class="s2">"org_email"</span><span class="p">],</span>
|
||||
<span class="n">org_extra_contact_info</span><span class="o">=</span><span class="n">metadata</span><span class="p">[</span><span class="s2">"org_extra_contact_info"</span><span class="p">],</span>
|
||||
@@ -636,6 +675,12 @@
|
||||
<span class="n">header_from</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"header_from"</span><span class="p">],</span>
|
||||
<span class="n">envelope_from</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"envelope_from"</span><span class="p">],</span>
|
||||
<span class="n">envelope_to</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"envelope_to"</span><span class="p">],</span>
|
||||
<span class="n">np</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"np"</span><span class="p">),</span>
|
||||
<span class="n">testing</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"testing"</span><span class="p">),</span>
|
||||
<span class="n">discovery_method</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
|
||||
<span class="s2">"discovery_method"</span>
|
||||
<span class="p">),</span>
|
||||
<span class="n">generator</span><span class="o">=</span><span class="n">metadata</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"generator"</span><span class="p">),</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">override</span> <span class="ow">in</span> <span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"policy_override_reasons"</span><span class="p">]:</span>
|
||||
@@ -648,6 +693,7 @@
|
||||
<span class="n">domain</span><span class="o">=</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">],</span>
|
||||
<span class="n">selector</span><span class="o">=</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">"selector"</span><span class="p">],</span>
|
||||
<span class="n">result</span><span class="o">=</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">],</span>
|
||||
<span class="n">human_result</span><span class="o">=</span><span class="n">dkim_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"human_result"</span><span class="p">),</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">spf_result</span> <span class="ow">in</span> <span class="n">record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">]:</span>
|
||||
@@ -655,6 +701,7 @@
|
||||
<span class="n">domain</span><span class="o">=</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">],</span>
|
||||
<span class="n">scope</span><span class="o">=</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">"scope"</span><span class="p">],</span>
|
||||
<span class="n">result</span><span class="o">=</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">],</span>
|
||||
<span class="n">human_result</span><span class="o">=</span><span class="n">spf_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"human_result"</span><span class="p">),</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"dmarc_aggregate"</span>
|
||||
@@ -677,10 +724,10 @@
|
||||
|
||||
|
||||
|
||||
<div class="viewcode-block" id="save_forensic_report_to_opensearch">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.save_forensic_report_to_opensearch">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">save_forensic_report_to_opensearch</span><span class="p">(</span>
|
||||
<span class="n">forensic_report</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">],</span>
|
||||
<div class="viewcode-block" id="save_failure_report_to_opensearch">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.save_failure_report_to_opensearch">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">save_failure_report_to_opensearch</span><span class="p">(</span>
|
||||
<span class="n">failure_report</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">],</span>
|
||||
<span class="n">index_suffix</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">index_prefix</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">monthly_indexes</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
||||
@@ -688,10 +735,10 @@
|
||||
<span class="n">number_of_replicas</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">0</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Saves a parsed DMARC forensic report to OpenSearch</span>
|
||||
<span class="sd"> Saves a parsed DMARC failure report to OpenSearch</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> forensic_report (dict): A parsed forensic report</span>
|
||||
<span class="sd"> failure_report (dict): A parsed failure report</span>
|
||||
<span class="sd"> index_suffix (str): The suffix of the name of the index to save to</span>
|
||||
<span class="sd"> index_prefix (str): The prefix of the name of the index to save to</span>
|
||||
<span class="sd"> monthly_indexes (bool): Use monthly indexes instead of daily</span>
|
||||
@@ -704,26 +751,28 @@
|
||||
<span class="sd"> AlreadySaved</span>
|
||||
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Saving forensic report to OpenSearch"</span><span class="p">)</span>
|
||||
<span class="n">forensic_report</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Saving failure report to OpenSearch"</span><span class="p">)</span>
|
||||
<span class="n">failure_report</span> <span class="o">=</span> <span class="n">failure_report</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
<span class="n">sample_date</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="k">if</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"date"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">sample_date</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"date"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"date"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">sample_date</span> <span class="o">=</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"date"</span><span class="p">]</span>
|
||||
<span class="n">sample_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">sample_date</span><span class="p">)</span>
|
||||
<span class="n">original_headers</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"headers"</span><span class="p">]</span>
|
||||
<span class="n">original_headers</span> <span class="o">=</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"headers"</span><span class="p">]</span>
|
||||
<span class="n">headers</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
|
||||
<span class="k">for</span> <span class="n">original_header</span> <span class="ow">in</span> <span class="n">original_headers</span><span class="p">:</span>
|
||||
<span class="n">headers</span><span class="p">[</span><span class="n">original_header</span><span class="o">.</span><span class="n">lower</span><span class="p">()]</span> <span class="o">=</span> <span class="n">original_headers</span><span class="p">[</span><span class="n">original_header</span><span class="p">]</span>
|
||||
|
||||
<span class="n">arrival_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"arrival_date_utc"</span><span class="p">])</span>
|
||||
<span class="n">arrival_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"arrival_date_utc"</span><span class="p">])</span>
|
||||
<span class="n">arrival_date_epoch_milliseconds</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">arrival_date</span><span class="o">.</span><span class="n">timestamp</span><span class="p">()</span> <span class="o">*</span> <span class="mi">1000</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">index_suffix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"dmarc_forensic_</span><span class="si">{0}</span><span class="s2">*"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"dmarc_failure_</span><span class="si">{0}</span><span class="s2">*,dmarc_forensic_</span><span class="si">{0}</span><span class="s2">*"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"dmarc_forensic*"</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"dmarc_failure*,dmarc_forensic*"</span>
|
||||
<span class="k">if</span> <span class="n">index_prefix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">search_index</span><span class="p">)</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">","</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
|
||||
<span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">part</span><span class="p">)</span> <span class="k">for</span> <span class="n">part</span> <span class="ow">in</span> <span class="n">search_index</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">","</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="n">search_index</span><span class="p">)</span>
|
||||
<span class="n">q</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">arrival_date</span><span class="o">=</span><span class="n">arrival_date_epoch_milliseconds</span><span class="p">)))</span>
|
||||
|
||||
@@ -764,67 +813,65 @@
|
||||
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">existing</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">AlreadySaved</span><span class="p">(</span>
|
||||
<span class="s2">"A forensic sample to </span><span class="si">{0}</span><span class="s2"> from </span><span class="si">{1}</span><span class="s2"> "</span>
|
||||
<span class="s2">"A failure sample to </span><span class="si">{0}</span><span class="s2"> from </span><span class="si">{1}</span><span class="s2"> "</span>
|
||||
<span class="s2">"with a subject of </span><span class="si">{2}</span><span class="s2"> and arrival date of </span><span class="si">{3}</span><span class="s2"> "</span>
|
||||
<span class="s2">"already exists in "</span>
|
||||
<span class="s2">"OpenSearch"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">to_</span><span class="p">,</span> <span class="n">from_</span><span class="p">,</span> <span class="n">subject</span><span class="p">,</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"arrival_date_utc"</span><span class="p">]</span>
|
||||
<span class="p">)</span>
|
||||
<span class="s2">"OpenSearch"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">to_</span><span class="p">,</span> <span class="n">from_</span><span class="p">,</span> <span class="n">subject</span><span class="p">,</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"arrival_date_utc"</span><span class="p">])</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">parsed_sample</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">]</span>
|
||||
<span class="n">sample</span> <span class="o">=</span> <span class="n">_ForensicSampleDoc</span><span class="p">(</span>
|
||||
<span class="n">raw</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"sample"</span><span class="p">],</span>
|
||||
<span class="n">parsed_sample</span> <span class="o">=</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">]</span>
|
||||
<span class="n">sample</span> <span class="o">=</span> <span class="n">_FailureSampleDoc</span><span class="p">(</span>
|
||||
<span class="n">raw</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"sample"</span><span class="p">],</span>
|
||||
<span class="n">headers</span><span class="o">=</span><span class="n">headers</span><span class="p">,</span>
|
||||
<span class="n">headers_only</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"sample_headers_only"</span><span class="p">],</span>
|
||||
<span class="n">headers_only</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"sample_headers_only"</span><span class="p">],</span>
|
||||
<span class="n">date</span><span class="o">=</span><span class="n">sample_date</span><span class="p">,</span>
|
||||
<span class="n">subject</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"subject"</span><span class="p">],</span>
|
||||
<span class="n">subject</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"subject"</span><span class="p">],</span>
|
||||
<span class="n">filename_safe_subject</span><span class="o">=</span><span class="n">parsed_sample</span><span class="p">[</span><span class="s2">"filename_safe_subject"</span><span class="p">],</span>
|
||||
<span class="n">body</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"body"</span><span class="p">],</span>
|
||||
<span class="n">body</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"body"</span><span class="p">],</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"to"</span><span class="p">]:</span>
|
||||
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"to"</span><span class="p">]:</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_to</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">])</span>
|
||||
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"reply_to"</span><span class="p">]:</span>
|
||||
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"reply_to"</span><span class="p">]:</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_reply_to</span><span class="p">(</span>
|
||||
<span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">]</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"cc"</span><span class="p">]:</span>
|
||||
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"cc"</span><span class="p">]:</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_cc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">])</span>
|
||||
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"bcc"</span><span class="p">]:</span>
|
||||
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"bcc"</span><span class="p">]:</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_bcc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">])</span>
|
||||
<span class="k">for</span> <span class="n">attachment</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"attachments"</span><span class="p">]:</span>
|
||||
<span class="k">for</span> <span class="n">attachment</span> <span class="ow">in</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"attachments"</span><span class="p">]:</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_attachment</span><span class="p">(</span>
|
||||
<span class="n">filename</span><span class="o">=</span><span class="n">attachment</span><span class="p">[</span><span class="s2">"filename"</span><span class="p">],</span>
|
||||
<span class="n">content_type</span><span class="o">=</span><span class="n">attachment</span><span class="p">[</span><span class="s2">"mail_content_type"</span><span class="p">],</span>
|
||||
<span class="n">sha256</span><span class="o">=</span><span class="n">attachment</span><span class="p">[</span><span class="s2">"sha256"</span><span class="p">],</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">forensic_doc</span> <span class="o">=</span> <span class="n">_ForensicReportDoc</span><span class="p">(</span>
|
||||
<span class="n">feedback_type</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"feedback_type"</span><span class="p">],</span>
|
||||
<span class="n">user_agent</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"user_agent"</span><span class="p">],</span>
|
||||
<span class="n">version</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"version"</span><span class="p">],</span>
|
||||
<span class="n">original_mail_from</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"original_mail_from"</span><span class="p">],</span>
|
||||
<span class="n">failure_doc</span> <span class="o">=</span> <span class="n">_FailureReportDoc</span><span class="p">(</span>
|
||||
<span class="n">feedback_type</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"feedback_type"</span><span class="p">],</span>
|
||||
<span class="n">user_agent</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"user_agent"</span><span class="p">],</span>
|
||||
<span class="n">version</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"version"</span><span class="p">],</span>
|
||||
<span class="n">original_mail_from</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"original_mail_from"</span><span class="p">],</span>
|
||||
<span class="n">arrival_date</span><span class="o">=</span><span class="n">arrival_date_epoch_milliseconds</span><span class="p">,</span>
|
||||
<span class="n">domain</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"reported_domain"</span><span class="p">],</span>
|
||||
<span class="n">original_envelope_id</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"original_envelope_id"</span><span class="p">],</span>
|
||||
<span class="n">authentication_results</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"authentication_results"</span><span class="p">],</span>
|
||||
<span class="n">delivery_results</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"delivery_result"</span><span class="p">],</span>
|
||||
<span class="n">source_ip_address</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"ip_address"</span><span class="p">],</span>
|
||||
<span class="n">source_country</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"country"</span><span class="p">],</span>
|
||||
<span class="n">source_reverse_dns</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"reverse_dns"</span><span class="p">],</span>
|
||||
<span class="n">source_base_domain</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"base_domain"</span><span class="p">],</span>
|
||||
<span class="n">source_asn</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"asn"</span><span class="p">],</span>
|
||||
<span class="n">source_as_name</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"as_name"</span><span class="p">],</span>
|
||||
<span class="n">source_as_domain</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"as_domain"</span><span class="p">],</span>
|
||||
<span class="n">authentication_mechanisms</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"authentication_mechanisms"</span><span class="p">],</span>
|
||||
<span class="n">auth_failure</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"auth_failure"</span><span class="p">],</span>
|
||||
<span class="n">dkim_domain</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"dkim_domain"</span><span class="p">],</span>
|
||||
<span class="n">original_rcpt_to</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"original_rcpt_to"</span><span class="p">],</span>
|
||||
<span class="n">domain</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"reported_domain"</span><span class="p">],</span>
|
||||
<span class="n">original_envelope_id</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"original_envelope_id"</span><span class="p">],</span>
|
||||
<span class="n">authentication_results</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"authentication_results"</span><span class="p">],</span>
|
||||
<span class="n">delivery_results</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"delivery_result"</span><span class="p">],</span>
|
||||
<span class="n">source_ip_address</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"ip_address"</span><span class="p">],</span>
|
||||
<span class="n">source_country</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"country"</span><span class="p">],</span>
|
||||
<span class="n">source_reverse_dns</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"reverse_dns"</span><span class="p">],</span>
|
||||
<span class="n">source_base_domain</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"base_domain"</span><span class="p">],</span>
|
||||
<span class="n">source_asn</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"asn"</span><span class="p">],</span>
|
||||
<span class="n">source_as_name</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"as_name"</span><span class="p">],</span>
|
||||
<span class="n">source_as_domain</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"as_domain"</span><span class="p">],</span>
|
||||
<span class="n">authentication_mechanisms</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"authentication_mechanisms"</span><span class="p">],</span>
|
||||
<span class="n">auth_failure</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"auth_failure"</span><span class="p">],</span>
|
||||
<span class="n">dkim_domain</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"dkim_domain"</span><span class="p">],</span>
|
||||
<span class="n">original_rcpt_to</span><span class="o">=</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"original_rcpt_to"</span><span class="p">],</span>
|
||||
<span class="n">sample</span><span class="o">=</span><span class="n">sample</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"dmarc_forensic"</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"dmarc_failure"</span>
|
||||
<span class="k">if</span> <span class="n">index_suffix</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">_</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">index_prefix</span><span class="p">:</span>
|
||||
@@ -838,14 +885,14 @@
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span> <span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">create_indexes</span><span class="p">([</span><span class="n">index</span><span class="p">],</span> <span class="n">index_settings</span><span class="p">)</span>
|
||||
<span class="n">forensic_doc</span><span class="o">.</span><span class="n">meta</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">index</span>
|
||||
<span class="n">failure_doc</span><span class="o">.</span><span class="n">meta</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">index</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">forensic_doc</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
|
||||
<span class="n">failure_doc</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span><span class="s2">"OpenSearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
||||
<span class="k">except</span> <span class="ne">KeyError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidForensicReport</span><span class="p">(</span>
|
||||
<span class="s2">"Forensic report missing required field: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidFailureReport</span><span class="p">(</span>
|
||||
<span class="s2">"Failure report missing required field: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="p">)</span></div>
|
||||
|
||||
|
||||
@@ -1003,6 +1050,12 @@
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span><span class="s2">"OpenSearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
|
||||
|
||||
|
||||
<span class="c1"># Backward-compatible aliases</span>
|
||||
<span class="n">_ForensicSampleDoc</span> <span class="o">=</span> <span class="n">_FailureSampleDoc</span>
|
||||
<span class="n">_ForensicReportDoc</span> <span class="o">=</span> <span class="n">_FailureReportDoc</span>
|
||||
<span class="n">save_forensic_report_to_opensearch</span> <span class="o">=</span> <span class="n">save_failure_report_to_opensearch</span>
|
||||
</pre></div>
|
||||
|
||||
</div>
|
||||
|
||||
@@ -5,14 +5,14 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc.splunk — parsedmarc 9.11.2 documentation</title>
|
||||
<title>parsedmarc.splunk — parsedmarc 10.0.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<script src="../../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../../_static/documentation_options.js?v=de4344a5"></script>
|
||||
<script src="../../_static/documentation_options.js?v=335988e4"></script>
|
||||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="../../_static/js/theme.js"></script>
|
||||
@@ -229,30 +229,30 @@
|
||||
<span class="k">raise</span> <span class="n">SplunkError</span><span class="p">(</span><span class="n">response</span><span class="p">[</span><span class="s2">"text"</span><span class="p">])</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="HECClient.save_forensic_reports_to_splunk">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.splunk.HECClient.save_forensic_reports_to_splunk">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">save_forensic_reports_to_splunk</span><span class="p">(</span>
|
||||
<div class="viewcode-block" id="HECClient.save_failure_reports_to_splunk">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.splunk.HECClient.save_failure_reports_to_splunk">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">save_failure_reports_to_splunk</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="p">,</span>
|
||||
<span class="n">forensic_reports</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">list</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]],</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]],</span>
|
||||
<span class="n">failure_reports</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">list</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]],</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]],</span>
|
||||
<span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Saves forensic DMARC reports to Splunk</span>
|
||||
<span class="sd"> Saves failure DMARC reports to Splunk</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> forensic_reports (list): A list of forensic report dictionaries</span>
|
||||
<span class="sd"> failure_reports (list): A list of failure report dictionaries</span>
|
||||
<span class="sd"> to save in Splunk</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Saving forensic reports to Splunk"</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">forensic_reports</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
|
||||
<span class="n">forensic_reports</span> <span class="o">=</span> <span class="p">[</span><span class="n">forensic_reports</span><span class="p">]</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Saving failure reports to Splunk"</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">failure_reports</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
|
||||
<span class="n">failure_reports</span> <span class="o">=</span> <span class="p">[</span><span class="n">failure_reports</span><span class="p">]</span>
|
||||
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">forensic_reports</span><span class="p">)</span> <span class="o"><</span> <span class="mi">1</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">failure_reports</span><span class="p">)</span> <span class="o"><</span> <span class="mi">1</span><span class="p">:</span>
|
||||
<span class="k">return</span>
|
||||
|
||||
<span class="n">json_str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||
<span class="k">for</span> <span class="n">report</span> <span class="ow">in</span> <span class="n">forensic_reports</span><span class="p">:</span>
|
||||
<span class="k">for</span> <span class="n">report</span> <span class="ow">in</span> <span class="n">failure_reports</span><span class="p">:</span>
|
||||
<span class="n">data</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_common_data</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
<span class="n">data</span><span class="p">[</span><span class="s2">"sourcetype"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"dmarc:forensic"</span>
|
||||
<span class="n">data</span><span class="p">[</span><span class="s2">"sourcetype"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"dmarc:failure"</span>
|
||||
<span class="n">timestamp</span> <span class="o">=</span> <span class="n">human_timestamp_to_unix_timestamp</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">"arrival_date_utc"</span><span class="p">])</span>
|
||||
<span class="n">data</span><span class="p">[</span><span class="s2">"time"</span><span class="p">]</span> <span class="o">=</span> <span class="n">timestamp</span>
|
||||
<span class="n">data</span><span class="p">[</span><span class="s2">"event"</span><span class="p">]</span> <span class="o">=</span> <span class="n">report</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
@@ -320,6 +320,10 @@
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">close</span><span class="p">()</span></div>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<span class="c1"># Backward-compatible aliases</span>
|
||||
<span class="n">HECClient</span><span class="o">.</span><span class="n">save_forensic_reports_to_splunk</span> <span class="o">=</span> <span class="n">HECClient</span><span class="o">.</span><span class="n">save_failure_reports_to_splunk</span>
|
||||
</pre></div>
|
||||
|
||||
</div>
|
||||
|
||||
@@ -5,14 +5,14 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc.types — parsedmarc 9.11.2 documentation</title>
|
||||
<title>parsedmarc.types — parsedmarc 10.0.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<script src="../../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../../_static/documentation_options.js?v=de4344a5"></script>
|
||||
<script src="../../_static/documentation_options.js?v=335988e4"></script>
|
||||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="../../_static/js/theme.js"></script>
|
||||
@@ -90,7 +90,7 @@
|
||||
<span class="c1"># For optional keys, use total=False TypedDicts.</span>
|
||||
|
||||
|
||||
<span class="n">ReportType</span> <span class="o">=</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">"aggregate"</span><span class="p">,</span> <span class="s2">"forensic"</span><span class="p">,</span> <span class="s2">"smtp_tls"</span><span class="p">]</span>
|
||||
<span class="n">ReportType</span> <span class="o">=</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">"aggregate"</span><span class="p">,</span> <span class="s2">"failure"</span><span class="p">,</span> <span class="s2">"smtp_tls"</span><span class="p">]</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="AggregateReportMetadata">
|
||||
@@ -104,7 +104,8 @@
|
||||
<span class="n">end_date</span><span class="p">:</span> <span class="nb">str</span>
|
||||
<span class="n">timespan_requires_normalization</span><span class="p">:</span> <span class="nb">bool</span>
|
||||
<span class="n">original_timespan_seconds</span><span class="p">:</span> <span class="nb">int</span>
|
||||
<span class="n">errors</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span></div>
|
||||
<span class="n">errors</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
|
||||
<span class="n">generator</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span></div>
|
||||
|
||||
|
||||
|
||||
@@ -116,8 +117,11 @@
|
||||
<span class="n">aspf</span><span class="p">:</span> <span class="nb">str</span>
|
||||
<span class="n">p</span><span class="p">:</span> <span class="nb">str</span>
|
||||
<span class="n">sp</span><span class="p">:</span> <span class="nb">str</span>
|
||||
<span class="n">pct</span><span class="p">:</span> <span class="nb">str</span>
|
||||
<span class="n">fo</span><span class="p">:</span> <span class="nb">str</span></div>
|
||||
<span class="n">pct</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
|
||||
<span class="n">fo</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
|
||||
<span class="n">np</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
|
||||
<span class="n">testing</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
|
||||
<span class="n">discovery_method</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span></div>
|
||||
|
||||
|
||||
|
||||
@@ -167,7 +171,8 @@
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">AggregateAuthResultDKIM</span><span class="p">(</span><span class="n">TypedDict</span><span class="p">):</span>
|
||||
<span class="n">domain</span><span class="p">:</span> <span class="nb">str</span>
|
||||
<span class="n">result</span><span class="p">:</span> <span class="nb">str</span>
|
||||
<span class="n">selector</span><span class="p">:</span> <span class="nb">str</span></div>
|
||||
<span class="n">selector</span><span class="p">:</span> <span class="nb">str</span>
|
||||
<span class="n">human_result</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span></div>
|
||||
|
||||
|
||||
|
||||
@@ -176,7 +181,8 @@
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">AggregateAuthResultSPF</span><span class="p">(</span><span class="n">TypedDict</span><span class="p">):</span>
|
||||
<span class="n">domain</span><span class="p">:</span> <span class="nb">str</span>
|
||||
<span class="n">result</span><span class="p">:</span> <span class="nb">str</span>
|
||||
<span class="n">scope</span><span class="p">:</span> <span class="nb">str</span></div>
|
||||
<span class="n">scope</span><span class="p">:</span> <span class="nb">str</span>
|
||||
<span class="n">human_result</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span></div>
|
||||
|
||||
|
||||
|
||||
@@ -217,6 +223,7 @@
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.types.AggregateReport">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">AggregateReport</span><span class="p">(</span><span class="n">TypedDict</span><span class="p">):</span>
|
||||
<span class="n">xml_schema</span><span class="p">:</span> <span class="nb">str</span>
|
||||
<span class="n">xml_namespace</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
|
||||
<span class="n">report_metadata</span><span class="p">:</span> <span class="n">AggregateReportMetadata</span>
|
||||
<span class="n">policy_published</span><span class="p">:</span> <span class="n">AggregatePolicyPublished</span>
|
||||
<span class="n">records</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">AggregateRecord</span><span class="p">]</span></div>
|
||||
@@ -246,7 +253,7 @@
|
||||
<span class="s2">"ParsedEmail"</span><span class="p">,</span>
|
||||
<span class="p">{</span>
|
||||
<span class="c1"># This is a lightly-specified version of mailsuite/mailparser JSON.</span>
|
||||
<span class="c1"># It focuses on the fields parsedmarc uses in forensic handling.</span>
|
||||
<span class="c1"># It focuses on the fields parsedmarc uses in failure report handling.</span>
|
||||
<span class="s2">"headers"</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">],</span>
|
||||
<span class="s2">"subject"</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span>
|
||||
<span class="s2">"filename_safe_subject"</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span>
|
||||
@@ -265,9 +272,9 @@
|
||||
<span class="p">)</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="ForensicReport">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.types.ForensicReport">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">ForensicReport</span><span class="p">(</span><span class="n">TypedDict</span><span class="p">):</span>
|
||||
<div class="viewcode-block" id="FailureReport">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.types.FailureReport">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">FailureReport</span><span class="p">(</span><span class="n">TypedDict</span><span class="p">):</span>
|
||||
<span class="n">feedback_type</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
|
||||
<span class="n">user_agent</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
|
||||
<span class="n">version</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
|
||||
@@ -289,6 +296,10 @@
|
||||
|
||||
|
||||
|
||||
<span class="c1"># Backward-compatible alias</span>
|
||||
<span class="n">ForensicReport</span> <span class="o">=</span> <span class="n">FailureReport</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="SMTPTLSFailureDetails">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.types.SMTPTLSFailureDetails">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">SMTPTLSFailureDetails</span><span class="p">(</span><span class="n">TypedDict</span><span class="p">):</span>
|
||||
@@ -349,14 +360,18 @@
|
||||
|
||||
|
||||
|
||||
<div class="viewcode-block" id="ForensicParsedReport">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.types.ForensicParsedReport">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">ForensicParsedReport</span><span class="p">(</span><span class="n">TypedDict</span><span class="p">):</span>
|
||||
<span class="n">report_type</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">"forensic"</span><span class="p">]</span>
|
||||
<span class="n">report</span><span class="p">:</span> <span class="n">ForensicReport</span></div>
|
||||
<div class="viewcode-block" id="FailureParsedReport">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.types.FailureParsedReport">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">FailureParsedReport</span><span class="p">(</span><span class="n">TypedDict</span><span class="p">):</span>
|
||||
<span class="n">report_type</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">"failure"</span><span class="p">]</span>
|
||||
<span class="n">report</span><span class="p">:</span> <span class="n">FailureReport</span></div>
|
||||
|
||||
|
||||
|
||||
<span class="c1"># Backward-compatible alias</span>
|
||||
<span class="n">ForensicParsedReport</span> <span class="o">=</span> <span class="n">FailureParsedReport</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="SMTPTLSParsedReport">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.types.SMTPTLSParsedReport">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">SMTPTLSParsedReport</span><span class="p">(</span><span class="n">TypedDict</span><span class="p">):</span>
|
||||
@@ -365,14 +380,14 @@
|
||||
|
||||
|
||||
|
||||
<span class="n">ParsedReport</span> <span class="o">=</span> <span class="n">Union</span><span class="p">[</span><span class="n">AggregateParsedReport</span><span class="p">,</span> <span class="n">ForensicParsedReport</span><span class="p">,</span> <span class="n">SMTPTLSParsedReport</span><span class="p">]</span>
|
||||
<span class="n">ParsedReport</span> <span class="o">=</span> <span class="n">Union</span><span class="p">[</span><span class="n">AggregateParsedReport</span><span class="p">,</span> <span class="n">FailureParsedReport</span><span class="p">,</span> <span class="n">SMTPTLSParsedReport</span><span class="p">]</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="ParsingResults">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.types.ParsingResults">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">ParsingResults</span><span class="p">(</span><span class="n">TypedDict</span><span class="p">):</span>
|
||||
<span class="n">aggregate_reports</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">AggregateReport</span><span class="p">]</span>
|
||||
<span class="n">forensic_reports</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">ForensicReport</span><span class="p">]</span>
|
||||
<span class="n">failure_reports</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">FailureReport</span><span class="p">]</span>
|
||||
<span class="n">smtp_tls_reports</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">SMTPTLSReport</span><span class="p">]</span></div>
|
||||
|
||||
</pre></div>
|
||||
|
||||
@@ -5,14 +5,14 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc.utils — parsedmarc 9.11.2 documentation</title>
|
||||
<title>parsedmarc.utils — parsedmarc 10.0.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<script src="../../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../../_static/documentation_options.js?v=de4344a5"></script>
|
||||
<script src="../../_static/documentation_options.js?v=335988e4"></script>
|
||||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="../../_static/js/theme.js"></script>
|
||||
@@ -139,7 +139,7 @@
|
||||
|
||||
<span class="n">parenthesis_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">"\s*\(.*\)\s*"</span><span class="p">)</span>
|
||||
|
||||
<span class="n">null_file</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">devnull</span><span class="p">,</span> <span class="s2">"w"</span><span class="p">)</span>
|
||||
<span class="n">null_file</span> <span class="o">=</span> <span class="n">subprocess</span><span class="o">.</span><span class="n">DEVNULL</span>
|
||||
<span class="n">mailparser_logger</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s2">"mailparser"</span><span class="p">)</span>
|
||||
<span class="n">mailparser_logger</span><span class="o">.</span><span class="n">setLevel</span><span class="p">(</span><span class="n">logging</span><span class="o">.</span><span class="n">CRITICAL</span><span class="p">)</span>
|
||||
<span class="n">psl</span> <span class="o">=</span> <span class="n">publicsuffixlist</span><span class="o">.</span><span class="n">PublicSuffixList</span><span class="p">()</span>
|
||||
|
||||
@@ -125,7 +125,7 @@ server.ssl.key: /etc/kibana/kibana.key
|
||||
```
|
||||
|
||||
:::{note}
|
||||
For more security, you can configure Kibana to use a local network connexion
|
||||
For more security, you can configure Kibana to use a local network connection
|
||||
to elasticsearch :
|
||||
```text
|
||||
elasticsearch.hosts: ['https://SERVER_IP:9200']
|
||||
@@ -214,7 +214,7 @@ Kibana index patterns with versions that match the upgraded indexes:
|
||||
|
||||
1. Login in to Kibana, and click on Management
|
||||
2. Under Kibana, click on Saved Objects
|
||||
3. Check the checkboxes for the `dmarc_aggregate` and `dmarc_forensic`
|
||||
3. Check the checkboxes for the `dmarc_aggregate` and `dmarc_failure`
|
||||
index patterns
|
||||
4. Click Delete
|
||||
5. Click Delete on the conformation message
|
||||
|
||||
+23
-18
@@ -29,35 +29,40 @@ and Valimail.
|
||||
|
||||
## Features
|
||||
|
||||
- Parses draft and 1.0 standard aggregate/rua DMARC reports
|
||||
- Parses forensic/failure/ruf DMARC reports
|
||||
- Parses reports from SMTP TLS Reporting
|
||||
- Parses aggregate/rua DMARC reports: the legacy draft and 1.0 schemas
|
||||
(RFC 7489) and the new RFC 9990 schema for the final DMARC standard
|
||||
(RFC 9989)
|
||||
- Parses failure/ruf DMARC reports (RFC 6591 and RFC 9991; formerly called
|
||||
forensic reports)
|
||||
- Parses reports from SMTP TLS Reporting (TLS-RPT, RFC 8460)
|
||||
- Can parse reports from an inbox over IMAP, Microsoft Graph, or Gmail API
|
||||
- Transparently handles gzip or zip compressed reports
|
||||
- Consistent data structures
|
||||
- Simple JSON and/or CSV output
|
||||
- Optionally email the results
|
||||
- Optionally send the results to Elasticsearch, Opensearch, and/or Splunk, for use
|
||||
with premade dashboards
|
||||
- Optionally send reports to Apache Kafka
|
||||
- Optionally send the results to Elasticsearch, OpenSearch, Splunk, or
|
||||
PostgreSQL, for use with premade dashboards
|
||||
- Optionally send the results to Apache Kafka, Amazon S3, Azure Log
|
||||
Analytics (Microsoft Sentinel), a Graylog (GELF) endpoint, a syslog server,
|
||||
or an HTTP webhook
|
||||
|
||||
## Python Compatibility
|
||||
|
||||
This project supports the following Python versions, which are either actively maintained or are the default versions
|
||||
for RHEL or Debian.
|
||||
|
||||
| Version | Supported | Reason |
|
||||
|---------|-----------|------------------------------------------------------------|
|
||||
| < 3.6 | ❌ | End of Life (EOL) |
|
||||
| 3.6 | ❌ | Used in RHEL 8, but not supported by project dependencies |
|
||||
| 3.7 | ❌ | End of Life (EOL) |
|
||||
| 3.8 | ❌ | End of Life (EOL) |
|
||||
| 3.9 | ❌ | Used in Debian 11 and RHEL 9, but not supported by project dependencies |
|
||||
| 3.10 | ✅ | Actively maintained |
|
||||
| 3.11 | ✅ | Actively maintained; supported until June 2028 (Debian 12) |
|
||||
| 3.12 | ✅ | Actively maintained; supported until May 2035 (RHEL 10) |
|
||||
| 3.13 | ✅ | Actively maintained; supported until June 2030 (Debian 13) |
|
||||
| 3.14 | ✅ | Supported (requires `imapclient>=3.1.0`) |
|
||||
| Version | Supported | Reason |
|
||||
| --- | --- | --- |
|
||||
| < 3.6 | ❌ | End of Life (EOL) |
|
||||
| 3.6 | ❌ | Used in RHEL 8, but not supported by project dependencies |
|
||||
| 3.7 | ❌ | End of Life (EOL) |
|
||||
| 3.8 | ❌ | End of Life (EOL) |
|
||||
| 3.9 | ❌ | Used in Debian 11 and RHEL 9, but not supported by project dependencies |
|
||||
| 3.10 | ✅ | Actively maintained |
|
||||
| 3.11 | ✅ | Actively maintained; supported until June 2028 (Debian 12) |
|
||||
| 3.12 | ✅ | Actively maintained; supported until May 2035 (RHEL 10) |
|
||||
| 3.13 | ✅ | Actively maintained; supported until June 2030 (Debian 13) |
|
||||
| 3.14 | ✅ | Supported (requires `imapclient>=3.1.0`) |
|
||||
|
||||
```{toctree}
|
||||
:caption: 'Contents'
|
||||
|
||||
@@ -89,7 +89,7 @@ information on DMARC failure reports (also known as forensic or ruf reports).
|
||||
These reports contain samples of emails that have failed to pass DMARC.
|
||||
|
||||
:::{note}
|
||||
Most recipients do not send forensic/failure/ruf reports at all to avoid
|
||||
Most recipients do not send failure/ruf reports at all to avoid
|
||||
privacy leaks. Some recipients (notably Chinese webmail services) will only
|
||||
supply the headers of sample emails. Very few provide the entire email.
|
||||
:::
|
||||
|
||||
@@ -99,12 +99,12 @@ draft,acme.com,noreply-dmarc-support@acme.com,http://acme.com/dmarc/support,9391
|
||||
|
||||
```
|
||||
|
||||
## Sample forensic report output
|
||||
## Sample failure report output
|
||||
|
||||
Thanks to GitHub user [xennn](https://github.com/xennn) for the anonymized
|
||||
[forensic report email sample](<https://github.com/domainaware/parsedmarc/raw/master/samples/forensic/DMARC%20Failure%20Report%20for%20domain.de%20(mail-from%3Dsharepoint%40domain.de%2C%20ip%3D10.10.10.10).eml>).
|
||||
[failure report email sample](<https://github.com/domainaware/parsedmarc/raw/master/samples/failure/DMARC%20Failure%20Report%20for%20domain.de%20(mail-from%3Dsharepoint%40domain.de%2C%20ip%3D10.10.10.10).eml>).
|
||||
|
||||
### JSON forensic report
|
||||
### JSON failure report
|
||||
|
||||
```json
|
||||
{
|
||||
@@ -198,7 +198,7 @@ Thanks to GitHub user [xennn](https://github.com/xennn) for the anonymized
|
||||
}
|
||||
```
|
||||
|
||||
### CSV forensic report
|
||||
### CSV failure report
|
||||
|
||||
```text
|
||||
feedback_type,user_agent,version,original_envelope_id,original_mail_from,original_rcpt_to,arrival_date,arrival_date_utc,subject,message_id,authentication_results,dkim_domain,source_ip_address,source_country,source_reverse_dns,source_base_domain,source_name,source_type,source_asn,source_as_name,source_as_domain,delivery_result,auth_failure,reported_domain,authentication_mechanisms,sample_headers_only
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
# Splunk
|
||||
|
||||
Starting in version 4.3.0 `parsedmarc` supports sending aggregate and/or
|
||||
forensic DMARC data to a Splunk [HTTP Event collector (HEC)].
|
||||
failure DMARC data to a Splunk [HTTP Event collector (HEC)].
|
||||
|
||||
The project repository contains [XML files] for premade Splunk
|
||||
dashboards for aggregate and forensic DMARC reports.
|
||||
dashboards for aggregate and failure DMARC reports.
|
||||
|
||||
Copy and paste the contents of each file into a separate Splunk
|
||||
dashboard XML editor.
|
||||
|
||||
+142
-45
@@ -4,9 +4,9 @@
|
||||
|
||||
```text
|
||||
usage: parsedmarc [-h] [-c CONFIG_FILE] [--strip-attachment-payloads] [-o OUTPUT]
|
||||
[--aggregate-json-filename AGGREGATE_JSON_FILENAME] [--forensic-json-filename FORENSIC_JSON_FILENAME]
|
||||
[--aggregate-json-filename AGGREGATE_JSON_FILENAME] [--failure-json-filename FAILURE_JSON_FILENAME]
|
||||
[--smtp-tls-json-filename SMTP_TLS_JSON_FILENAME] [--aggregate-csv-filename AGGREGATE_CSV_FILENAME]
|
||||
[--forensic-csv-filename FORENSIC_CSV_FILENAME] [--smtp-tls-csv-filename SMTP_TLS_CSV_FILENAME]
|
||||
[--failure-csv-filename FAILURE_CSV_FILENAME] [--smtp-tls-csv-filename SMTP_TLS_CSV_FILENAME]
|
||||
[-n NAMESERVERS [NAMESERVERS ...]] [-t DNS_TIMEOUT] [--offline] [-s] [-w] [--verbose] [--debug]
|
||||
[--log-file LOG_FILE] [--no-prettify-json] [-v]
|
||||
[file_path ...]
|
||||
@@ -14,26 +14,26 @@ usage: parsedmarc [-h] [-c CONFIG_FILE] [--strip-attachment-payloads] [-o OUTPUT
|
||||
Parses DMARC reports
|
||||
|
||||
positional arguments:
|
||||
file_path one or more paths to aggregate or forensic report files, emails, or mbox files'
|
||||
file_path one or more paths to aggregate or failure report files, emails, or mbox files'
|
||||
|
||||
options:
|
||||
-h, --help show this help message and exit
|
||||
-c CONFIG_FILE, --config-file CONFIG_FILE
|
||||
a path to a configuration file (--silent implied)
|
||||
--strip-attachment-payloads
|
||||
remove attachment payloads from forensic report output
|
||||
remove attachment payloads from failure report output
|
||||
-o OUTPUT, --output OUTPUT
|
||||
write output files to the given directory
|
||||
--aggregate-json-filename AGGREGATE_JSON_FILENAME
|
||||
filename for the aggregate JSON output file
|
||||
--forensic-json-filename FORENSIC_JSON_FILENAME
|
||||
filename for the forensic JSON output file
|
||||
--failure-json-filename FAILURE_JSON_FILENAME
|
||||
filename for the failure JSON output file
|
||||
--smtp-tls-json-filename SMTP_TLS_JSON_FILENAME
|
||||
filename for the SMTP TLS JSON output file
|
||||
--aggregate-csv-filename AGGREGATE_CSV_FILENAME
|
||||
filename for the aggregate CSV output file
|
||||
--forensic-csv-filename FORENSIC_CSV_FILENAME
|
||||
filename for the forensic CSV output file
|
||||
--failure-csv-filename FAILURE_CSV_FILENAME
|
||||
filename for the failure CSV output file
|
||||
--smtp-tls-csv-filename SMTP_TLS_CSV_FILENAME
|
||||
filename for the SMTP TLS CSV output file
|
||||
-n NAMESERVERS [NAMESERVERS ...], --nameservers NAMESERVERS [NAMESERVERS ...]
|
||||
@@ -70,7 +70,7 @@ For example
|
||||
|
||||
[general]
|
||||
save_aggregate = True
|
||||
save_forensic = True
|
||||
save_failure = True
|
||||
|
||||
[imap]
|
||||
host = imap.example.com
|
||||
@@ -109,7 +109,7 @@ mode = tcp
|
||||
|
||||
[webhook]
|
||||
aggregate_url = https://aggregate_url.example.com
|
||||
forensic_url = https://forensic_url.example.com
|
||||
failure_url = https://failure_url.example.com
|
||||
smtp_tls_url = https://smtp_tls_url.example.com
|
||||
timeout = 60
|
||||
```
|
||||
@@ -119,7 +119,7 @@ The full set of configuration options are:
|
||||
- `general`
|
||||
- `save_aggregate` - bool: Save aggregate report data to
|
||||
Elasticsearch, Splunk and/or S3
|
||||
- `save_forensic` - bool: Save forensic report data to
|
||||
- `save_failure` - bool: Save failure report data to
|
||||
Elasticsearch, Splunk and/or S3
|
||||
- `save_smtp_tls` - bool: Save SMTP-STS report data to
|
||||
Elasticsearch, Splunk and/or S3
|
||||
@@ -130,7 +130,7 @@ The full set of configuration options are:
|
||||
- `output` - str: Directory to place JSON and CSV files in. This is required if you set either of the JSON output file options.
|
||||
- `aggregate_json_filename` - str: filename for the aggregate
|
||||
JSON output file
|
||||
- `forensic_json_filename` - str: filename for the forensic
|
||||
- `failure_json_filename` - str: filename for the failure
|
||||
JSON output file
|
||||
- `ip_db_path` - str: An optional custom path to a MMDB file
|
||||
from IPinfo, MaxMind, or DBIP
|
||||
@@ -297,6 +297,12 @@ The full set of configuration options are:
|
||||
creating the index (Default: `1`)
|
||||
- `number_of_replicas` - int: The number of replicas to use when
|
||||
creating the index (Default: `0`)
|
||||
- `serverless` - bool: Set to `True` when targeting an Elastic Cloud
|
||||
Serverless project. Serverless manages sharding and replication itself
|
||||
and rejects the `number_of_shards` / `number_of_replicas` index settings
|
||||
with HTTP 400. With this flag set, parsedmarc strips those keys from the
|
||||
settings sent at index creation; any other settings (e.g.
|
||||
`refresh_interval`) are passed through unchanged (Default: `False`)
|
||||
- `opensearch`
|
||||
- `hosts` - str: A comma separated list of hostnames and ports
|
||||
or URLs (e.g. `127.0.0.1:9200` or
|
||||
@@ -340,7 +346,7 @@ The full set of configuration options are:
|
||||
- `skip_certificate_verification` - bool: Skip certificate
|
||||
verification (not recommended)
|
||||
- `aggregate_topic` - str: The Kafka topic for aggregate reports
|
||||
- `forensic_topic` - str: The Kafka topic for forensic reports
|
||||
- `failure_topic` - str: The Kafka topic for failure reports
|
||||
- `smtp`
|
||||
- `host` - str: The SMTP hostname
|
||||
- `port` - int: The SMTP port (Default: `25`)
|
||||
@@ -361,6 +367,52 @@ The full set of configuration options are:
|
||||
`%` characters must be escaped with another `%` character,
|
||||
so use `%%` wherever a `%` character is used.
|
||||
:::
|
||||
- `postgresql`
|
||||
- `host` - str: The PostgreSQL server hostname or IP address.
|
||||
Required unless `connection_string` is provided.
|
||||
- `port` - int: The PostgreSQL server port (Default: `5432`)
|
||||
- `user` - str: The database user name (Optional)
|
||||
- `password` - str: The database user password (Optional)
|
||||
- `database` - str: The database name (Optional)
|
||||
- `connection_string` - str: A full libpq connection string or URI
|
||||
(e.g. `postgresql://user:pass@host/dbname`). When provided,
|
||||
all individual parameters above are ignored.
|
||||
|
||||
The PostgreSQL backend is an optional extra. Install it with
|
||||
`pip install parsedmarc[postgresql]` (it pulls in `psycopg`); the
|
||||
prebuilt binary wheels are not available for every platform, which is
|
||||
why it is not a mandatory dependency.
|
||||
|
||||
Tables are created automatically on first run using
|
||||
`CREATE TABLE IF NOT EXISTS`, so no manual schema migration is needed
|
||||
for fresh installations.
|
||||
|
||||
**Example configuration:**
|
||||
|
||||
```ini
|
||||
[postgresql]
|
||||
host = localhost
|
||||
port = 5432
|
||||
user = parsedmarc
|
||||
password = secret
|
||||
database = parsedmarc
|
||||
```
|
||||
|
||||
Or using a DSN/URI:
|
||||
|
||||
```ini
|
||||
[postgresql]
|
||||
connection_string = postgresql://parsedmarc:secret@localhost/parsedmarc
|
||||
```
|
||||
|
||||
Saving parsed data to PostgreSQL is controlled by the `[general]`
|
||||
options `save_aggregate`, `save_failure`, and `save_smtp_tls`
|
||||
(`save_forensic` is still accepted as a deprecated alias for
|
||||
`save_failure`). These flags must be set to `True` for the
|
||||
corresponding report types (aggregate DMARC, failure DMARC, and
|
||||
SMTP TLS reports) or no data will be written to PostgreSQL, even if
|
||||
this section is configured.
|
||||
|
||||
- `s3`
|
||||
- `bucket` - str: The S3 bucket name
|
||||
- `path` - str: The path to upload reports to (Default: `/`)
|
||||
@@ -458,7 +510,7 @@ The full set of configuration options are:
|
||||
- `dce` - str: The Data Collection Endpoint (DCE). Example: `https://{DCE-NAME}.{REGION}.ingest.monitor.azure.com`.
|
||||
- `dcr_immutable_id` - str: The immutable ID of the Data Collection Rule (DCR)
|
||||
- `dcr_aggregate_stream` - str: The stream name for aggregate reports in the DCR
|
||||
- `dcr_forensic_stream` - str: The stream name for the forensic reports in the DCR
|
||||
- `dcr_failure_stream` - str: The stream name for the failure reports in the DCR
|
||||
- `dcr_smtp_tls_stream` - str: The stream name for the SMTP TLS reports in the DCR
|
||||
|
||||
:::{note}
|
||||
@@ -470,12 +522,12 @@ The full set of configuration options are:
|
||||
- `mode` - str: The GELF transport type to use. Valid modes: `tcp`, `udp`, `tls`
|
||||
|
||||
- `maildir`
|
||||
- `maildir_path` - str: Full path for mailbox maidir location (Default: `INBOX`)
|
||||
- `maildir_path` - str: Full path for mailbox maildir location (Default: `INBOX`)
|
||||
- `maildir_create` - bool: Create maildir if not present (Default: False)
|
||||
|
||||
- `webhook` - Post the individual reports to a webhook url with the report as the JSON body
|
||||
- `aggregate_url` - str: URL of the webhook which should receive the aggregate reports
|
||||
- `forensic_url` - str: URL of the webhook which should receive the forensic reports
|
||||
- `failure_url` - str: URL of the webhook which should receive the failure reports
|
||||
- `smtp_tls_url` - str: URL of the webhook which should receive the smtp_tls reports
|
||||
- `timeout` - int: Interval in which the webhook call should timeout
|
||||
|
||||
@@ -490,26 +542,26 @@ blocks DNS requests to outside resolvers.
|
||||
:::
|
||||
|
||||
:::{note}
|
||||
`save_aggregate` and `save_forensic` are separate options
|
||||
because you may not want to save forensic reports
|
||||
(also known as failure reports) to your Elasticsearch instance,
|
||||
`save_aggregate` and `save_failure` are separate options
|
||||
because you may not want to save failure reports
|
||||
(formerly known as forensic reports) to your Elasticsearch instance,
|
||||
particularly if you are in a highly-regulated industry that
|
||||
handles sensitive data, such as healthcare or finance. If your
|
||||
legitimate outgoing email fails DMARC, it is possible
|
||||
that email may appear later in a forensic report.
|
||||
that email may appear later in a failure report.
|
||||
|
||||
Forensic reports contain the original headers of an email that
|
||||
Failure reports contain the original headers of an email that
|
||||
failed a DMARC check, and sometimes may also include the
|
||||
full message body, depending on the policy of the reporting
|
||||
organization.
|
||||
|
||||
Most reporting organizations do not send forensic reports of any
|
||||
Most reporting organizations do not send failure reports of any
|
||||
kind for privacy reasons. While aggregate DMARC reports are sent
|
||||
at least daily, it is normal to receive very few forensic reports.
|
||||
at least daily, it is normal to receive very few failure reports.
|
||||
|
||||
An alternative approach is to still collect forensic/failure/ruf
|
||||
An alternative approach is to still collect failure/ruf
|
||||
reports in your DMARC inbox, but run `parsedmarc` with
|
||||
```save_forensic = True``` manually on a separate IMAP folder (using
|
||||
```save_failure = True``` manually on a separate IMAP folder (using
|
||||
the ```reports_folder``` option), after you have manually moved
|
||||
known samples you want to save to that folder
|
||||
(e.g. malicious samples and non-sensitive legitimate samples).
|
||||
@@ -610,31 +662,76 @@ services:
|
||||
PARSEDMARC_MAILBOX_WATCH: "true"
|
||||
PARSEDMARC_ELASTICSEARCH_HOSTS: http://elasticsearch:9200
|
||||
PARSEDMARC_GENERAL_SAVE_AGGREGATE: "true"
|
||||
PARSEDMARC_GENERAL_SAVE_FORENSIC: "true"
|
||||
PARSEDMARC_GENERAL_SAVE_FAILURE: "true"
|
||||
```
|
||||
|
||||
### Docker secrets (`_FILE` suffix)
|
||||
|
||||
Any `PARSEDMARC_{SECTION}_{KEY}` environment variable can also be supplied
|
||||
via a file by appending `_FILE` to its name. The file's contents (with any
|
||||
trailing CR/LF characters stripped) are used as the value. This is the
|
||||
same convention used by the official Postgres, MariaDB, and Redis container
|
||||
images, and is designed to plug straight into Docker / Docker Compose /
|
||||
Kubernetes secrets so credentials never appear in plain `environment:`
|
||||
blocks (where they would be readable via `docker inspect`, container logs,
|
||||
and `/proc/<pid>/environ`).
|
||||
|
||||
The bare `DEBUG` / `PARSEDMARC_DEBUG` aliases and `PARSEDMARC_CONFIG_FILE`
|
||||
do not have a `_FILE` form; only `PARSEDMARC_{SECTION}_{KEY}` vars resolved
|
||||
to a known config section are eligible.
|
||||
|
||||
If both the direct env var and the `_FILE` variant are set, the `_FILE`
|
||||
variant wins. If the file does not exist or is unreadable, parsedmarc
|
||||
exits with a configuration error rather than silently falling back to an
|
||||
empty value.
|
||||
|
||||
```yaml
|
||||
secrets:
|
||||
imap_password:
|
||||
file: ./secrets/imap_password.txt
|
||||
|
||||
services:
|
||||
parsedmarc:
|
||||
image: parsedmarc:latest
|
||||
secrets:
|
||||
- imap_password
|
||||
environment:
|
||||
PARSEDMARC_IMAP_HOST: imap.example.com
|
||||
PARSEDMARC_IMAP_USER: dmarc@example.com
|
||||
PARSEDMARC_IMAP_PASSWORD_FILE: /run/secrets/imap_password
|
||||
```
|
||||
|
||||
Note that a small set of config keys whose own names already end in
|
||||
`_file` (`[general] log_file`, `[msgraph] token_file`,
|
||||
`[gmail_api] credentials_file`, `[gmail_api] token_file`) keep their
|
||||
pre-existing meaning when set via `PARSEDMARC_..._FILE` — that env var is
|
||||
the path itself, not a wrapper around a file containing the path. To pass
|
||||
*those* paths via a Docker secret, double up the suffix
|
||||
(`PARSEDMARC_GMAIL_API_CREDENTIALS_FILE_FILE`); the inner contents are
|
||||
then read and stored as the `credentials_file` value.
|
||||
|
||||
### Section name mapping
|
||||
|
||||
For sections with underscores in the name, the full section name is used:
|
||||
|
||||
| Section | Env var prefix |
|
||||
|------------------|-------------------------------|
|
||||
| `general` | `PARSEDMARC_GENERAL_` |
|
||||
| `mailbox` | `PARSEDMARC_MAILBOX_` |
|
||||
| `imap` | `PARSEDMARC_IMAP_` |
|
||||
| `msgraph` | `PARSEDMARC_MSGRAPH_` |
|
||||
| `elasticsearch` | `PARSEDMARC_ELASTICSEARCH_` |
|
||||
| `opensearch` | `PARSEDMARC_OPENSEARCH_` |
|
||||
| `splunk_hec` | `PARSEDMARC_SPLUNK_HEC_` |
|
||||
| `kafka` | `PARSEDMARC_KAFKA_` |
|
||||
| `smtp` | `PARSEDMARC_SMTP_` |
|
||||
| `s3` | `PARSEDMARC_S3_` |
|
||||
| `syslog` | `PARSEDMARC_SYSLOG_` |
|
||||
| `gmail_api` | `PARSEDMARC_GMAIL_API_` |
|
||||
| `maildir` | `PARSEDMARC_MAILDIR_` |
|
||||
| `log_analytics` | `PARSEDMARC_LOG_ANALYTICS_` |
|
||||
| `gelf` | `PARSEDMARC_GELF_` |
|
||||
| `webhook` | `PARSEDMARC_WEBHOOK_` |
|
||||
| Section | Env var prefix |
|
||||
| --- | --- |
|
||||
| `general` | `PARSEDMARC_GENERAL_` |
|
||||
| `mailbox` | `PARSEDMARC_MAILBOX_` |
|
||||
| `imap` | `PARSEDMARC_IMAP_` |
|
||||
| `msgraph` | `PARSEDMARC_MSGRAPH_` |
|
||||
| `elasticsearch` | `PARSEDMARC_ELASTICSEARCH_` |
|
||||
| `opensearch` | `PARSEDMARC_OPENSEARCH_` |
|
||||
| `splunk_hec` | `PARSEDMARC_SPLUNK_HEC_` |
|
||||
| `kafka` | `PARSEDMARC_KAFKA_` |
|
||||
| `smtp` | `PARSEDMARC_SMTP_` |
|
||||
| `s3` | `PARSEDMARC_S3_` |
|
||||
| `syslog` | `PARSEDMARC_SYSLOG_` |
|
||||
| `gmail_api` | `PARSEDMARC_GMAIL_API_` |
|
||||
| `maildir` | `PARSEDMARC_MAILDIR_` |
|
||||
| `log_analytics` | `PARSEDMARC_LOG_ANALYTICS_` |
|
||||
| `gelf` | `PARSEDMARC_GELF_` |
|
||||
| `webhook` | `PARSEDMARC_WEBHOOK_` |
|
||||
|
||||
## Performance tuning
|
||||
|
||||
@@ -651,7 +748,7 @@ imports more predictable:
|
||||
- Use `mailbox.since` to process reports in smaller time windows such as `1d`,
|
||||
`7d`, or another interval that fits the backlog. This makes it easier to catch
|
||||
up incrementally instead of loading an entire mailbox history in one run.
|
||||
- Set `strip_attachment_payloads = True` when forensic reports contain large
|
||||
- Set `strip_attachment_payloads = True` when failure reports contain large
|
||||
attachments and you do not need to retain the raw payloads in the parsed
|
||||
output.
|
||||
- Prefer running parsedmarc separately from Elasticsearch or OpenSearch, or
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
const DOCUMENTATION_OPTIONS = {
|
||||
VERSION: '9.11.2',
|
||||
VERSION: '10.0.0',
|
||||
LANGUAGE: 'en',
|
||||
COLLAPSE_INDEX: false,
|
||||
BUILDER: 'html',
|
||||
|
||||
+2
-2
@@ -6,14 +6,14 @@
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Contributing to parsedmarc — parsedmarc 9.11.2 documentation</title>
|
||||
<title>Contributing to parsedmarc — parsedmarc 10.0.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=de4344a5"></script>
|
||||
<script src="_static/documentation_options.js?v=335988e4"></script>
|
||||
<script src="_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
|
||||
+2
-2
@@ -6,14 +6,14 @@
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Accessing an inbox using OWA/EWS — parsedmarc 9.11.2 documentation</title>
|
||||
<title>Accessing an inbox using OWA/EWS — parsedmarc 10.0.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=de4344a5"></script>
|
||||
<script src="_static/documentation_options.js?v=335988e4"></script>
|
||||
<script src="_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
|
||||
+2
-2
@@ -6,14 +6,14 @@
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Understanding DMARC — parsedmarc 9.11.2 documentation</title>
|
||||
<title>Understanding DMARC — parsedmarc 10.0.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=de4344a5"></script>
|
||||
<script src="_static/documentation_options.js?v=335988e4"></script>
|
||||
<script src="_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
|
||||
+4
-4
@@ -6,14 +6,14 @@
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Elasticsearch and Kibana — parsedmarc 9.11.2 documentation</title>
|
||||
<title>Elasticsearch and Kibana — parsedmarc 10.0.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=de4344a5"></script>
|
||||
<script src="_static/documentation_options.js?v=335988e4"></script>
|
||||
<script src="_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
@@ -186,7 +186,7 @@ server.ssl.key: /etc/kibana/kibana.key
|
||||
</div>
|
||||
<div class="admonition note">
|
||||
<p class="admonition-title">Note</p>
|
||||
<p>For more security, you can configure Kibana to use a local network connexion
|
||||
<p>For more security, you can configure Kibana to use a local network connection
|
||||
to elasticsearch :</p>
|
||||
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span>elasticsearch.hosts: ['https://SERVER_IP:9200']
|
||||
</pre></div>
|
||||
@@ -251,7 +251,7 @@ Kibana index patterns with versions that match the upgraded indexes:</p>
|
||||
<ol class="arabic simple">
|
||||
<li><p>Login in to Kibana, and click on Management</p></li>
|
||||
<li><p>Under Kibana, click on Saved Objects</p></li>
|
||||
<li><p>Check the checkboxes for the <code class="docutils literal notranslate"><span class="pre">dmarc_aggregate</span></code> and <code class="docutils literal notranslate"><span class="pre">dmarc_forensic</span></code>
|
||||
<li><p>Check the checkboxes for the <code class="docutils literal notranslate"><span class="pre">dmarc_aggregate</span></code> and <code class="docutils literal notranslate"><span class="pre">dmarc_failure</span></code>
|
||||
index patterns</p></li>
|
||||
<li><p>Click Delete</p></li>
|
||||
<li><p>Click Delete on the conformation message</p></li>
|
||||
|
||||
+31
-11
@@ -5,14 +5,14 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Index — parsedmarc 9.11.2 documentation</title>
|
||||
<title>Index — parsedmarc 10.0.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=de4344a5"></script>
|
||||
<script src="_static/documentation_options.js?v=335988e4"></script>
|
||||
<script src="_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
@@ -115,10 +115,10 @@
|
||||
</li>
|
||||
<li><a href="api.html#parsedmarc.types.AggregateParsedReport">AggregateParsedReport (class in parsedmarc.types)</a>
|
||||
</li>
|
||||
</ul></td>
|
||||
<td style="width: 33%; vertical-align: top;"><ul>
|
||||
<li><a href="api.html#parsedmarc.types.AggregatePolicyEvaluated">AggregatePolicyEvaluated (class in parsedmarc.types)</a>
|
||||
</li>
|
||||
</ul></td>
|
||||
<td style="width: 33%; vertical-align: top;"><ul>
|
||||
<li><a href="api.html#parsedmarc.types.AggregatePolicyOverrideReason">AggregatePolicyOverrideReason (class in parsedmarc.types)</a>
|
||||
</li>
|
||||
<li><a href="api.html#parsedmarc.types.AggregatePolicyPublished">AggregatePolicyPublished (class in parsedmarc.types)</a>
|
||||
@@ -130,6 +130,8 @@
|
||||
<li><a href="api.html#parsedmarc.types.AggregateReportMetadata">AggregateReportMetadata (class in parsedmarc.types)</a>
|
||||
</li>
|
||||
<li><a href="api.html#parsedmarc.elastic.AlreadySaved">AlreadySaved</a>, <a href="api.html#parsedmarc.opensearch.AlreadySaved">[1]</a>
|
||||
</li>
|
||||
<li><a href="api.html#parsedmarc.append_json">append_json() (in module parsedmarc)</a>
|
||||
</li>
|
||||
</ul></td>
|
||||
</tr></table>
|
||||
@@ -191,11 +193,15 @@
|
||||
<h2 id="F">F</h2>
|
||||
<table style="width: 100%" class="indextable genindextable"><tr>
|
||||
<td style="width: 33%; vertical-align: top;"><ul>
|
||||
<li><a href="api.html#parsedmarc.types.ForensicParsedReport">ForensicParsedReport (class in parsedmarc.types)</a>
|
||||
<li><a href="api.html#parsedmarc.types.FailureParsedReport">FailureParsedReport (class in parsedmarc.types)</a>
|
||||
</li>
|
||||
<li><a href="api.html#parsedmarc.types.FailureReport">FailureReport (class in parsedmarc.types)</a>
|
||||
</li>
|
||||
</ul></td>
|
||||
<td style="width: 33%; vertical-align: top;"><ul>
|
||||
<li><a href="api.html#parsedmarc.types.ForensicReport">ForensicReport (class in parsedmarc.types)</a>
|
||||
<li><a href="api.html#parsedmarc.types.ForensicParsedReport">ForensicParsedReport (in module parsedmarc.types)</a>
|
||||
</li>
|
||||
<li><a href="api.html#parsedmarc.types.ForensicReport">ForensicReport (in module parsedmarc.types)</a>
|
||||
</li>
|
||||
</ul></td>
|
||||
</tr></table>
|
||||
@@ -249,7 +255,9 @@
|
||||
</li>
|
||||
<li><a href="api.html#parsedmarc.InvalidDMARCReport">InvalidDMARCReport</a>
|
||||
</li>
|
||||
<li><a href="api.html#parsedmarc.InvalidForensicReport">InvalidForensicReport</a>
|
||||
<li><a href="api.html#parsedmarc.InvalidFailureReport">InvalidFailureReport</a>
|
||||
</li>
|
||||
<li><a href="api.html#parsedmarc.InvalidForensicReport">InvalidForensicReport (in module parsedmarc)</a>
|
||||
</li>
|
||||
<li><a href="api.html#parsedmarc.utils.InvalidIPinfoAPIKey">InvalidIPinfoAPIKey</a>
|
||||
</li>
|
||||
@@ -327,6 +335,8 @@
|
||||
<li><a href="api.html#parsedmarc.parse_aggregate_report_xml">parse_aggregate_report_xml() (in module parsedmarc)</a>
|
||||
</li>
|
||||
<li><a href="api.html#parsedmarc.utils.parse_email">parse_email() (in module parsedmarc.utils)</a>
|
||||
</li>
|
||||
<li><a href="api.html#parsedmarc.parse_failure_report">parse_failure_report() (in module parsedmarc)</a>
|
||||
</li>
|
||||
<li><a href="api.html#parsedmarc.parse_forensic_report">parse_forensic_report() (in module parsedmarc)</a>
|
||||
</li>
|
||||
@@ -339,6 +349,10 @@
|
||||
<li><a href="api.html#parsedmarc.parsed_aggregate_reports_to_csv">parsed_aggregate_reports_to_csv() (in module parsedmarc)</a>
|
||||
</li>
|
||||
<li><a href="api.html#parsedmarc.parsed_aggregate_reports_to_csv_rows">parsed_aggregate_reports_to_csv_rows() (in module parsedmarc)</a>
|
||||
</li>
|
||||
<li><a href="api.html#parsedmarc.parsed_failure_reports_to_csv">parsed_failure_reports_to_csv() (in module parsedmarc)</a>
|
||||
</li>
|
||||
<li><a href="api.html#parsedmarc.parsed_failure_reports_to_csv_rows">parsed_failure_reports_to_csv_rows() (in module parsedmarc)</a>
|
||||
</li>
|
||||
<li><a href="api.html#parsedmarc.parsed_forensic_reports_to_csv">parsed_forensic_reports_to_csv() (in module parsedmarc)</a>
|
||||
</li>
|
||||
@@ -346,12 +360,12 @@
|
||||
</li>
|
||||
<li><a href="api.html#parsedmarc.parsed_smtp_tls_reports_to_csv">parsed_smtp_tls_reports_to_csv() (in module parsedmarc)</a>
|
||||
</li>
|
||||
</ul></td>
|
||||
<td style="width: 33%; vertical-align: top;"><ul>
|
||||
<li><a href="api.html#parsedmarc.parsed_smtp_tls_reports_to_csv_rows">parsed_smtp_tls_reports_to_csv_rows() (in module parsedmarc)</a>
|
||||
</li>
|
||||
<li><a href="api.html#parsedmarc.types.ParsedEmail">ParsedEmail (class in parsedmarc.types)</a>
|
||||
</li>
|
||||
</ul></td>
|
||||
<td style="width: 33%; vertical-align: top;"><ul>
|
||||
<li>
|
||||
parsedmarc
|
||||
|
||||
@@ -425,6 +439,12 @@
|
||||
<li><a href="api.html#parsedmarc.opensearch.save_aggregate_report_to_opensearch">save_aggregate_report_to_opensearch() (in module parsedmarc.opensearch)</a>
|
||||
</li>
|
||||
<li><a href="api.html#parsedmarc.splunk.HECClient.save_aggregate_reports_to_splunk">save_aggregate_reports_to_splunk() (parsedmarc.splunk.HECClient method)</a>
|
||||
</li>
|
||||
<li><a href="api.html#parsedmarc.elastic.save_failure_report_to_elasticsearch">save_failure_report_to_elasticsearch() (in module parsedmarc.elastic)</a>
|
||||
</li>
|
||||
<li><a href="api.html#parsedmarc.opensearch.save_failure_report_to_opensearch">save_failure_report_to_opensearch() (in module parsedmarc.opensearch)</a>
|
||||
</li>
|
||||
<li><a href="api.html#parsedmarc.splunk.HECClient.save_failure_reports_to_splunk">save_failure_reports_to_splunk() (parsedmarc.splunk.HECClient method)</a>
|
||||
</li>
|
||||
<li><a href="api.html#parsedmarc.elastic.save_forensic_report_to_elasticsearch">save_forensic_report_to_elasticsearch() (in module parsedmarc.elastic)</a>
|
||||
</li>
|
||||
@@ -435,11 +455,11 @@
|
||||
<li><a href="api.html#parsedmarc.save_output">save_output() (in module parsedmarc)</a>
|
||||
</li>
|
||||
<li><a href="api.html#parsedmarc.elastic.save_smtp_tls_report_to_elasticsearch">save_smtp_tls_report_to_elasticsearch() (in module parsedmarc.elastic)</a>
|
||||
</li>
|
||||
<li><a href="api.html#parsedmarc.opensearch.save_smtp_tls_report_to_opensearch">save_smtp_tls_report_to_opensearch() (in module parsedmarc.opensearch)</a>
|
||||
</li>
|
||||
</ul></td>
|
||||
<td style="width: 33%; vertical-align: top;"><ul>
|
||||
<li><a href="api.html#parsedmarc.opensearch.save_smtp_tls_report_to_opensearch">save_smtp_tls_report_to_opensearch() (in module parsedmarc.opensearch)</a>
|
||||
</li>
|
||||
<li><a href="api.html#parsedmarc.splunk.HECClient.save_smtp_tls_reports_to_splunk">save_smtp_tls_reports_to_splunk() (parsedmarc.splunk.HECClient method)</a>
|
||||
</li>
|
||||
<li><a href="api.html#parsedmarc.elastic.set_hosts">set_hosts() (in module parsedmarc.elastic)</a>
|
||||
|
||||
+14
-9
@@ -6,14 +6,14 @@
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc documentation - Open source DMARC report analyzer and visualizer — parsedmarc 9.11.2 documentation</title>
|
||||
<title>parsedmarc documentation - Open source DMARC report analyzer and visualizer — parsedmarc 10.0.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=de4344a5"></script>
|
||||
<script src="_static/documentation_options.js?v=335988e4"></script>
|
||||
<script src="_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
@@ -101,17 +101,22 @@ and Valimail.</p>
|
||||
<section id="features">
|
||||
<h2>Features<a class="headerlink" href="#features" title="Link to this heading"></a></h2>
|
||||
<ul class="simple">
|
||||
<li><p>Parses draft and 1.0 standard aggregate/rua DMARC reports</p></li>
|
||||
<li><p>Parses forensic/failure/ruf DMARC reports</p></li>
|
||||
<li><p>Parses reports from SMTP TLS Reporting</p></li>
|
||||
<li><p>Parses aggregate/rua DMARC reports: the legacy draft and 1.0 schemas
|
||||
(RFC 7489) and the new RFC 9990 schema for the final DMARC standard
|
||||
(RFC 9989)</p></li>
|
||||
<li><p>Parses failure/ruf DMARC reports (RFC 6591 and RFC 9991; formerly called
|
||||
forensic reports)</p></li>
|
||||
<li><p>Parses reports from SMTP TLS Reporting (TLS-RPT, RFC 8460)</p></li>
|
||||
<li><p>Can parse reports from an inbox over IMAP, Microsoft Graph, or Gmail API</p></li>
|
||||
<li><p>Transparently handles gzip or zip compressed reports</p></li>
|
||||
<li><p>Consistent data structures</p></li>
|
||||
<li><p>Simple JSON and/or CSV output</p></li>
|
||||
<li><p>Optionally email the results</p></li>
|
||||
<li><p>Optionally send the results to Elasticsearch, Opensearch, and/or Splunk, for use
|
||||
with premade dashboards</p></li>
|
||||
<li><p>Optionally send reports to Apache Kafka</p></li>
|
||||
<li><p>Optionally send the results to Elasticsearch, OpenSearch, Splunk, or
|
||||
PostgreSQL, for use with premade dashboards</p></li>
|
||||
<li><p>Optionally send the results to Apache Kafka, Amazon S3, Azure Log
|
||||
Analytics (Microsoft Sentinel), a Graylog (GELF) endpoint, a syslog server,
|
||||
or an HTTP webhook</p></li>
|
||||
</ul>
|
||||
</section>
|
||||
<section id="python-compatibility">
|
||||
@@ -189,7 +194,7 @@ for RHEL or Debian.</p>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="output.html">Sample outputs</a><ul>
|
||||
<li class="toctree-l2"><a class="reference internal" href="output.html#sample-aggregate-report-output">Sample aggregate report output</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="output.html#sample-forensic-report-output">Sample forensic report output</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="output.html#sample-failure-report-output">Sample failure report output</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="elasticsearch.html">Elasticsearch and Kibana</a><ul>
|
||||
|
||||
+2
-2
@@ -6,14 +6,14 @@
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Installation — parsedmarc 9.11.2 documentation</title>
|
||||
<title>Installation — parsedmarc 10.0.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=de4344a5"></script>
|
||||
<script src="_static/documentation_options.js?v=335988e4"></script>
|
||||
<script src="_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
|
||||
+3
-3
@@ -6,14 +6,14 @@
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Using the Kibana dashboards — parsedmarc 9.11.2 documentation</title>
|
||||
<title>Using the Kibana dashboards — parsedmarc 10.0.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=de4344a5"></script>
|
||||
<script src="_static/documentation_options.js?v=335988e4"></script>
|
||||
<script src="_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
@@ -166,7 +166,7 @@ information on DMARC failure reports (also known as forensic or ruf reports).
|
||||
These reports contain samples of emails that have failed to pass DMARC.</p>
|
||||
<div class="admonition note">
|
||||
<p class="admonition-title">Note</p>
|
||||
<p>Most recipients do not send forensic/failure/ruf reports at all to avoid
|
||||
<p>Most recipients do not send failure/ruf reports at all to avoid
|
||||
privacy leaks. Some recipients (notably Chinese webmail services) will only
|
||||
supply the headers of sample emails. Very few provide the entire email.</p>
|
||||
</div>
|
||||
|
||||
+2
-2
@@ -6,14 +6,14 @@
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>What about mailing lists? — parsedmarc 9.11.2 documentation</title>
|
||||
<title>What about mailing lists? — parsedmarc 10.0.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=de4344a5"></script>
|
||||
<script src="_static/documentation_options.js?v=335988e4"></script>
|
||||
<script src="_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
|
||||
BIN
Binary file not shown.
+2
-2
@@ -6,14 +6,14 @@
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>OpenSearch and Grafana — parsedmarc 9.11.2 documentation</title>
|
||||
<title>OpenSearch and Grafana — parsedmarc 10.0.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=de4344a5"></script>
|
||||
<script src="_static/documentation_options.js?v=335988e4"></script>
|
||||
<script src="_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
|
||||
+12
-12
@@ -6,14 +6,14 @@
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Sample outputs — parsedmarc 9.11.2 documentation</title>
|
||||
<title>Sample outputs — parsedmarc 10.0.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=de4344a5"></script>
|
||||
<script src="_static/documentation_options.js?v=335988e4"></script>
|
||||
<script src="_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
@@ -52,9 +52,9 @@
|
||||
<li class="toctree-l3"><a class="reference internal" href="#csv-aggregate-report">CSV aggregate report</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="#sample-forensic-report-output">Sample forensic report output</a><ul>
|
||||
<li class="toctree-l3"><a class="reference internal" href="#json-forensic-report">JSON forensic report</a></li>
|
||||
<li class="toctree-l3"><a class="reference internal" href="#csv-forensic-report">CSV forensic report</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="#sample-failure-report-output">Sample failure report output</a><ul>
|
||||
<li class="toctree-l3"><a class="reference internal" href="#json-failure-report">JSON failure report</a></li>
|
||||
<li class="toctree-l3"><a class="reference internal" href="#csv-failure-report">CSV failure report</a></li>
|
||||
<li class="toctree-l3"><a class="reference internal" href="#json-smtp-tls-report">JSON SMTP TLS report</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
@@ -193,12 +193,12 @@ draft,acme.com,noreply-dmarc-support@acme.com,http://acme.com/dmarc/support,9391
|
||||
</div>
|
||||
</section>
|
||||
</section>
|
||||
<section id="sample-forensic-report-output">
|
||||
<h2>Sample forensic report output<a class="headerlink" href="#sample-forensic-report-output" title="Link to this heading"></a></h2>
|
||||
<section id="sample-failure-report-output">
|
||||
<h2>Sample failure report output<a class="headerlink" href="#sample-failure-report-output" title="Link to this heading"></a></h2>
|
||||
<p>Thanks to GitHub user <a class="reference external" href="https://github.com/xennn">xennn</a> for the anonymized
|
||||
<a class="reference external" href="https://github.com/domainaware/parsedmarc/raw/master/samples/forensic/DMARC%20Failure%20Report%20for%20domain.de%20(mail-from%3Dsharepoint%40domain.de%2C%20ip%3D10.10.10.10).eml">forensic report email sample</a>.</p>
|
||||
<section id="json-forensic-report">
|
||||
<h3>JSON forensic report<a class="headerlink" href="#json-forensic-report" title="Link to this heading"></a></h3>
|
||||
<a class="reference external" href="https://github.com/domainaware/parsedmarc/raw/master/samples/failure/DMARC%20Failure%20Report%20for%20domain.de%20(mail-from%3Dsharepoint%40domain.de%2C%20ip%3D10.10.10.10).eml">failure report email sample</a>.</p>
|
||||
<section id="json-failure-report">
|
||||
<h3>JSON failure report<a class="headerlink" href="#json-failure-report" title="Link to this heading"></a></h3>
|
||||
<div class="highlight-json notranslate"><div class="highlight"><pre><span></span><span class="p">{</span>
|
||||
<span class="w"> </span><span class="nt">"feedback_type"</span><span class="p">:</span><span class="w"> </span><span class="s2">"auth-failure"</span><span class="p">,</span>
|
||||
<span class="w"> </span><span class="nt">"user_agent"</span><span class="p">:</span><span class="w"> </span><span class="s2">"Lua/1.0"</span><span class="p">,</span>
|
||||
@@ -291,8 +291,8 @@ draft,acme.com,noreply-dmarc-support@acme.com,http://acme.com/dmarc/support,9391
|
||||
</pre></div>
|
||||
</div>
|
||||
</section>
|
||||
<section id="csv-forensic-report">
|
||||
<h3>CSV forensic report<a class="headerlink" href="#csv-forensic-report" title="Link to this heading"></a></h3>
|
||||
<section id="csv-failure-report">
|
||||
<h3>CSV failure report<a class="headerlink" href="#csv-failure-report" title="Link to this heading"></a></h3>
|
||||
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span>feedback_type,user_agent,version,original_envelope_id,original_mail_from,original_rcpt_to,arrival_date,arrival_date_utc,subject,message_id,authentication_results,dkim_domain,source_ip_address,source_country,source_reverse_dns,source_base_domain,source_name,source_type,source_asn,source_as_name,source_as_domain,delivery_result,auth_failure,reported_domain,authentication_mechanisms,sample_headers_only
|
||||
auth-failure,Lua/1.0,1.0,,sharepoint@domain.de,peter.pan@domain.de,"Mon, 01 Oct 2018 11:20:27 +0200",2018-10-01 09:20:27,Subject,<38.E7.30937.BD6E1BB5@ mailrelay.de>,"dmarc=fail (p=none, dis=none) header.from=domain.de",,10.10.10.10,,,,policy,dmarc,domain.de,,False
|
||||
</pre></div>
|
||||
|
||||
+2
-2
@@ -5,14 +5,14 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Python Module Index — parsedmarc 9.11.2 documentation</title>
|
||||
<title>Python Module Index — parsedmarc 10.0.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=de4344a5"></script>
|
||||
<script src="_static/documentation_options.js?v=335988e4"></script>
|
||||
<script src="_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
|
||||
+2
-2
@@ -5,7 +5,7 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Search — parsedmarc 9.11.2 documentation</title>
|
||||
<title>Search — parsedmarc 10.0.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=de4344a5"></script>
|
||||
<script src="_static/documentation_options.js?v=335988e4"></script>
|
||||
<script src="_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
|
||||
+1
-1
File diff suppressed because one or more lines are too long
+4
-4
@@ -6,14 +6,14 @@
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Splunk — parsedmarc 9.11.2 documentation</title>
|
||||
<title>Splunk — parsedmarc 10.0.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=de4344a5"></script>
|
||||
<script src="_static/documentation_options.js?v=335988e4"></script>
|
||||
<script src="_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
@@ -84,9 +84,9 @@
|
||||
<section class="tex2jax_ignore mathjax_ignore" id="splunk">
|
||||
<h1>Splunk<a class="headerlink" href="#splunk" title="Link to this heading"></a></h1>
|
||||
<p>Starting in version 4.3.0 <code class="docutils literal notranslate"><span class="pre">parsedmarc</span></code> supports sending aggregate and/or
|
||||
forensic DMARC data to a Splunk <a class="reference external" href="http://docs.splunk.com/Documentation/Splunk/latest/Data/AboutHEC">HTTP Event collector (HEC)</a>.</p>
|
||||
failure DMARC data to a Splunk <a class="reference external" href="http://docs.splunk.com/Documentation/Splunk/latest/Data/AboutHEC">HTTP Event collector (HEC)</a>.</p>
|
||||
<p>The project repository contains <a class="reference external" href="https://github.com/domainaware/parsedmarc/tree/master/splunk">XML files</a> for premade Splunk
|
||||
dashboards for aggregate and forensic DMARC reports.</p>
|
||||
dashboards for aggregate and failure DMARC reports.</p>
|
||||
<p>Copy and paste the contents of each file into a separate Splunk
|
||||
dashboard XML editor.</p>
|
||||
<div class="admonition warning">
|
||||
|
||||
+118
-29
@@ -6,14 +6,14 @@
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Using parsedmarc — parsedmarc 9.11.2 documentation</title>
|
||||
<title>Using parsedmarc — parsedmarc 10.0.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=de4344a5"></script>
|
||||
<script src="_static/documentation_options.js?v=335988e4"></script>
|
||||
<script src="_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
@@ -53,6 +53,7 @@
|
||||
<li class="toctree-l3"><a class="reference internal" href="#specifying-the-config-file-via-environment-variable">Specifying the config file via environment variable</a></li>
|
||||
<li class="toctree-l3"><a class="reference internal" href="#running-without-a-config-file-env-only-mode">Running without a config file (env-only mode)</a></li>
|
||||
<li class="toctree-l3"><a class="reference internal" href="#docker-compose-example">Docker Compose example</a></li>
|
||||
<li class="toctree-l3"><a class="reference internal" href="#docker-secrets-file-suffix">Docker secrets (<code class="docutils literal notranslate"><span class="pre">_FILE</span></code> suffix)</a></li>
|
||||
<li class="toctree-l3"><a class="reference internal" href="#section-name-mapping">Section name mapping</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
@@ -104,9 +105,9 @@
|
||||
<section id="cli-help">
|
||||
<h2>CLI help<a class="headerlink" href="#cli-help" title="Link to this heading"></a></h2>
|
||||
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span>usage: parsedmarc [-h] [-c CONFIG_FILE] [--strip-attachment-payloads] [-o OUTPUT]
|
||||
[--aggregate-json-filename AGGREGATE_JSON_FILENAME] [--forensic-json-filename FORENSIC_JSON_FILENAME]
|
||||
[--aggregate-json-filename AGGREGATE_JSON_FILENAME] [--failure-json-filename FAILURE_JSON_FILENAME]
|
||||
[--smtp-tls-json-filename SMTP_TLS_JSON_FILENAME] [--aggregate-csv-filename AGGREGATE_CSV_FILENAME]
|
||||
[--forensic-csv-filename FORENSIC_CSV_FILENAME] [--smtp-tls-csv-filename SMTP_TLS_CSV_FILENAME]
|
||||
[--failure-csv-filename FAILURE_CSV_FILENAME] [--smtp-tls-csv-filename SMTP_TLS_CSV_FILENAME]
|
||||
[-n NAMESERVERS [NAMESERVERS ...]] [-t DNS_TIMEOUT] [--offline] [-s] [-w] [--verbose] [--debug]
|
||||
[--log-file LOG_FILE] [--no-prettify-json] [-v]
|
||||
[file_path ...]
|
||||
@@ -114,26 +115,26 @@
|
||||
Parses DMARC reports
|
||||
|
||||
positional arguments:
|
||||
file_path one or more paths to aggregate or forensic report files, emails, or mbox files'
|
||||
file_path one or more paths to aggregate or failure report files, emails, or mbox files'
|
||||
|
||||
options:
|
||||
-h, --help show this help message and exit
|
||||
-c CONFIG_FILE, --config-file CONFIG_FILE
|
||||
a path to a configuration file (--silent implied)
|
||||
--strip-attachment-payloads
|
||||
remove attachment payloads from forensic report output
|
||||
remove attachment payloads from failure report output
|
||||
-o OUTPUT, --output OUTPUT
|
||||
write output files to the given directory
|
||||
--aggregate-json-filename AGGREGATE_JSON_FILENAME
|
||||
filename for the aggregate JSON output file
|
||||
--forensic-json-filename FORENSIC_JSON_FILENAME
|
||||
filename for the forensic JSON output file
|
||||
--failure-json-filename FAILURE_JSON_FILENAME
|
||||
filename for the failure JSON output file
|
||||
--smtp-tls-json-filename SMTP_TLS_JSON_FILENAME
|
||||
filename for the SMTP TLS JSON output file
|
||||
--aggregate-csv-filename AGGREGATE_CSV_FILENAME
|
||||
filename for the aggregate CSV output file
|
||||
--forensic-csv-filename FORENSIC_CSV_FILENAME
|
||||
filename for the forensic CSV output file
|
||||
--failure-csv-filename FAILURE_CSV_FILENAME
|
||||
filename for the failure CSV output file
|
||||
--smtp-tls-csv-filename SMTP_TLS_CSV_FILENAME
|
||||
filename for the SMTP TLS CSV output file
|
||||
-n NAMESERVERS [NAMESERVERS ...], --nameservers NAMESERVERS [NAMESERVERS ...]
|
||||
@@ -167,7 +168,7 @@ configuration file, described below.</p>
|
||||
|
||||
<span class="k">[general]</span>
|
||||
<span class="na">save_aggregate</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">True</span>
|
||||
<span class="na">save_forensic</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">True</span>
|
||||
<span class="na">save_failure</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">True</span>
|
||||
|
||||
<span class="k">[imap]</span>
|
||||
<span class="na">host</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">imap.example.com</span>
|
||||
@@ -206,7 +207,7 @@ configuration file, described below.</p>
|
||||
|
||||
<span class="k">[webhook]</span>
|
||||
<span class="na">aggregate_url</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">https://aggregate_url.example.com</span>
|
||||
<span class="na">forensic_url</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">https://forensic_url.example.com</span>
|
||||
<span class="na">failure_url</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">https://failure_url.example.com</span>
|
||||
<span class="na">smtp_tls_url</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">https://smtp_tls_url.example.com</span>
|
||||
<span class="na">timeout</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">60</span>
|
||||
</pre></div>
|
||||
@@ -217,7 +218,7 @@ configuration file, described below.</p>
|
||||
<ul>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">save_aggregate</span></code> - bool: Save aggregate report data to
|
||||
Elasticsearch, Splunk and/or S3</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">save_forensic</span></code> - bool: Save forensic report data to
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">save_failure</span></code> - bool: Save failure report data to
|
||||
Elasticsearch, Splunk and/or S3</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">save_smtp_tls</span></code> - bool: Save SMTP-STS report data to
|
||||
Elasticsearch, Splunk and/or S3</p></li>
|
||||
@@ -228,7 +229,7 @@ payloads from results</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">output</span></code> - str: Directory to place JSON and CSV files in. This is required if you set either of the JSON output file options.</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">aggregate_json_filename</span></code> - str: filename for the aggregate
|
||||
JSON output file</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">forensic_json_filename</span></code> - str: filename for the forensic
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">failure_json_filename</span></code> - str: filename for the failure
|
||||
JSON output file</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">ip_db_path</span></code> - str: An optional custom path to a MMDB file
|
||||
from IPinfo, MaxMind, or DBIP</p></li>
|
||||
@@ -406,6 +407,12 @@ verification (not recommended)</p></li>
|
||||
creating the index (Default: <code class="docutils literal notranslate"><span class="pre">1</span></code>)</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">number_of_replicas</span></code> - int: The number of replicas to use when
|
||||
creating the index (Default: <code class="docutils literal notranslate"><span class="pre">0</span></code>)</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">serverless</span></code> - bool: Set to <code class="docutils literal notranslate"><span class="pre">True</span></code> when targeting an Elastic Cloud
|
||||
Serverless project. Serverless manages sharding and replication itself
|
||||
and rejects the <code class="docutils literal notranslate"><span class="pre">number_of_shards</span></code> / <code class="docutils literal notranslate"><span class="pre">number_of_replicas</span></code> index settings
|
||||
with HTTP 400. With this flag set, parsedmarc strips those keys from the
|
||||
settings sent at index creation; any other settings (e.g.
|
||||
<code class="docutils literal notranslate"><span class="pre">refresh_interval</span></code>) are passed through unchanged (Default: <code class="docutils literal notranslate"><span class="pre">False</span></code>)</p></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">opensearch</span></code></p>
|
||||
@@ -459,7 +466,7 @@ verification (not recommended)</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">skip_certificate_verification</span></code> - bool: Skip certificate
|
||||
verification (not recommended)</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">aggregate_topic</span></code> - str: The Kafka topic for aggregate reports</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">forensic_topic</span></code> - str: The Kafka topic for forensic reports</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">failure_topic</span></code> - str: The Kafka topic for failure reports</p></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">smtp</span></code></p>
|
||||
@@ -486,6 +493,47 @@ so use <code class="docutils literal notranslate"><span class="pre">%%</span></c
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">postgresql</span></code></p>
|
||||
<ul class="simple">
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">host</span></code> - str: The PostgreSQL server hostname or IP address.
|
||||
Required unless <code class="docutils literal notranslate"><span class="pre">connection_string</span></code> is provided.</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">port</span></code> - int: The PostgreSQL server port (Default: <code class="docutils literal notranslate"><span class="pre">5432</span></code>)</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">user</span></code> - str: The database user name (Optional)</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">password</span></code> - str: The database user password (Optional)</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">database</span></code> - str: The database name (Optional)</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">connection_string</span></code> - str: A full libpq connection string or URI
|
||||
(e.g. <code class="docutils literal notranslate"><span class="pre">postgresql://user:pass@host/dbname</span></code>). When provided,
|
||||
all individual parameters above are ignored.</p></li>
|
||||
</ul>
|
||||
<p>The PostgreSQL backend is an optional extra. Install it with
|
||||
<code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">install</span> <span class="pre">parsedmarc[postgresql]</span></code> (it pulls in <code class="docutils literal notranslate"><span class="pre">psycopg</span></code>); the
|
||||
prebuilt binary wheels are not available for every platform, which is
|
||||
why it is not a mandatory dependency.</p>
|
||||
<p>Tables are created automatically on first run using
|
||||
<code class="docutils literal notranslate"><span class="pre">CREATE</span> <span class="pre">TABLE</span> <span class="pre">IF</span> <span class="pre">NOT</span> <span class="pre">EXISTS</span></code>, so no manual schema migration is needed
|
||||
for fresh installations.</p>
|
||||
<p><strong>Example configuration:</strong></p>
|
||||
<div class="highlight-ini notranslate"><div class="highlight"><pre><span></span><span class="k">[postgresql]</span>
|
||||
<span class="na">host</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">localhost</span>
|
||||
<span class="na">port</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">5432</span>
|
||||
<span class="na">user</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">parsedmarc</span>
|
||||
<span class="na">password</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">secret</span>
|
||||
<span class="na">database</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">parsedmarc</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
<p>Or using a DSN/URI:</p>
|
||||
<div class="highlight-ini notranslate"><div class="highlight"><pre><span></span><span class="k">[postgresql]</span>
|
||||
<span class="na">connection_string</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">postgresql://parsedmarc:secret@localhost/parsedmarc</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
<p>Saving parsed data to PostgreSQL is controlled by the <code class="docutils literal notranslate"><span class="pre">[general]</span></code>
|
||||
options <code class="docutils literal notranslate"><span class="pre">save_aggregate</span></code>, <code class="docutils literal notranslate"><span class="pre">save_failure</span></code>, and <code class="docutils literal notranslate"><span class="pre">save_smtp_tls</span></code>
|
||||
(<code class="docutils literal notranslate"><span class="pre">save_forensic</span></code> is still accepted as a deprecated alias for
|
||||
<code class="docutils literal notranslate"><span class="pre">save_failure</span></code>). These flags must be set to <code class="docutils literal notranslate"><span class="pre">True</span></code> for the
|
||||
corresponding report types (aggregate DMARC, failure DMARC, and
|
||||
SMTP TLS reports) or no data will be written to PostgreSQL, even if
|
||||
this section is configured.</p>
|
||||
</li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">s3</span></code></p>
|
||||
<ul class="simple">
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">bucket</span></code> - str: The S3 bucket name</p></li>
|
||||
@@ -586,7 +634,7 @@ When <code class="docutils literal notranslate"><span class="pre">False</span></
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">dce</span></code> - str: The Data Collection Endpoint (DCE). Example: <code class="docutils literal notranslate"><span class="pre">https://{DCE-NAME}.{REGION}.ingest.monitor.azure.com</span></code>.</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">dcr_immutable_id</span></code> - str: The immutable ID of the Data Collection Rule (DCR)</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">dcr_aggregate_stream</span></code> - str: The stream name for aggregate reports in the DCR</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">dcr_forensic_stream</span></code> - str: The stream name for the forensic reports in the DCR</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">dcr_failure_stream</span></code> - str: The stream name for the failure reports in the DCR</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">dcr_smtp_tls_stream</span></code> - str: The stream name for the SMTP TLS reports in the DCR</p></li>
|
||||
</ul>
|
||||
<div class="admonition note">
|
||||
@@ -603,14 +651,14 @@ When <code class="docutils literal notranslate"><span class="pre">False</span></
|
||||
</li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">maildir</span></code></p>
|
||||
<ul class="simple">
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">maildir_path</span></code> - str: Full path for mailbox maidir location (Default: <code class="docutils literal notranslate"><span class="pre">INBOX</span></code>)</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">maildir_path</span></code> - str: Full path for mailbox maildir location (Default: <code class="docutils literal notranslate"><span class="pre">INBOX</span></code>)</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">maildir_create</span></code> - bool: Create maildir if not present (Default: False)</p></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">webhook</span></code> - Post the individual reports to a webhook url with the report as the JSON body</p>
|
||||
<ul class="simple">
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">aggregate_url</span></code> - str: URL of the webhook which should receive the aggregate reports</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">forensic_url</span></code> - str: URL of the webhook which should receive the forensic reports</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">failure_url</span></code> - str: URL of the webhook which should receive the failure reports</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">smtp_tls_url</span></code> - str: URL of the webhook which should receive the smtp_tls reports</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">timeout</span></code> - int: Interval in which the webhook call should timeout</p></li>
|
||||
</ul>
|
||||
@@ -627,23 +675,23 @@ blocks DNS requests to outside resolvers.</p>
|
||||
</div>
|
||||
<div class="admonition note">
|
||||
<p class="admonition-title">Note</p>
|
||||
<p><code class="docutils literal notranslate"><span class="pre">save_aggregate</span></code> and <code class="docutils literal notranslate"><span class="pre">save_forensic</span></code> are separate options
|
||||
because you may not want to save forensic reports
|
||||
(also known as failure reports) to your Elasticsearch instance,
|
||||
<p><code class="docutils literal notranslate"><span class="pre">save_aggregate</span></code> and <code class="docutils literal notranslate"><span class="pre">save_failure</span></code> are separate options
|
||||
because you may not want to save failure reports
|
||||
(formerly known as forensic reports) to your Elasticsearch instance,
|
||||
particularly if you are in a highly-regulated industry that
|
||||
handles sensitive data, such as healthcare or finance. If your
|
||||
legitimate outgoing email fails DMARC, it is possible
|
||||
that email may appear later in a forensic report.</p>
|
||||
<p>Forensic reports contain the original headers of an email that
|
||||
that email may appear later in a failure report.</p>
|
||||
<p>Failure reports contain the original headers of an email that
|
||||
failed a DMARC check, and sometimes may also include the
|
||||
full message body, depending on the policy of the reporting
|
||||
organization.</p>
|
||||
<p>Most reporting organizations do not send forensic reports of any
|
||||
<p>Most reporting organizations do not send failure reports of any
|
||||
kind for privacy reasons. While aggregate DMARC reports are sent
|
||||
at least daily, it is normal to receive very few forensic reports.</p>
|
||||
<p>An alternative approach is to still collect forensic/failure/ruf
|
||||
at least daily, it is normal to receive very few failure reports.</p>
|
||||
<p>An alternative approach is to still collect failure/ruf
|
||||
reports in your DMARC inbox, but run <code class="docutils literal notranslate"><span class="pre">parsedmarc</span></code> with
|
||||
<code class="docutils literal notranslate"><span class="pre">save_forensic</span> <span class="pre">=</span> <span class="pre">True</span></code> manually on a separate IMAP folder (using
|
||||
<code class="docutils literal notranslate"><span class="pre">save_failure</span> <span class="pre">=</span> <span class="pre">True</span></code> manually on a separate IMAP folder (using
|
||||
the <code class="docutils literal notranslate"><span class="pre">reports_folder</span></code> option), after you have manually moved
|
||||
known samples you want to save to that folder
|
||||
(e.g. malicious samples and non-sensitive legitimate samples).</p>
|
||||
@@ -736,10 +784,51 @@ parsedmarc<span class="w"> </span>/path/to/reports/*
|
||||
<span class="w"> </span><span class="nt">PARSEDMARC_MAILBOX_WATCH</span><span class="p">:</span><span class="w"> </span><span class="s">"true"</span>
|
||||
<span class="w"> </span><span class="nt">PARSEDMARC_ELASTICSEARCH_HOSTS</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">http://elasticsearch:9200</span>
|
||||
<span class="w"> </span><span class="nt">PARSEDMARC_GENERAL_SAVE_AGGREGATE</span><span class="p">:</span><span class="w"> </span><span class="s">"true"</span>
|
||||
<span class="w"> </span><span class="nt">PARSEDMARC_GENERAL_SAVE_FORENSIC</span><span class="p">:</span><span class="w"> </span><span class="s">"true"</span>
|
||||
<span class="w"> </span><span class="nt">PARSEDMARC_GENERAL_SAVE_FAILURE</span><span class="p">:</span><span class="w"> </span><span class="s">"true"</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</section>
|
||||
<section id="docker-secrets-file-suffix">
|
||||
<h3>Docker secrets (<code class="docutils literal notranslate"><span class="pre">_FILE</span></code> suffix)<a class="headerlink" href="#docker-secrets-file-suffix" title="Link to this heading"></a></h3>
|
||||
<p>Any <code class="docutils literal notranslate"><span class="pre">PARSEDMARC_{SECTION}_{KEY}</span></code> environment variable can also be supplied
|
||||
via a file by appending <code class="docutils literal notranslate"><span class="pre">_FILE</span></code> to its name. The file’s contents (with any
|
||||
trailing CR/LF characters stripped) are used as the value. This is the
|
||||
same convention used by the official Postgres, MariaDB, and Redis container
|
||||
images, and is designed to plug straight into Docker / Docker Compose /
|
||||
Kubernetes secrets so credentials never appear in plain <code class="docutils literal notranslate"><span class="pre">environment:</span></code>
|
||||
blocks (where they would be readable via <code class="docutils literal notranslate"><span class="pre">docker</span> <span class="pre">inspect</span></code>, container logs,
|
||||
and <code class="docutils literal notranslate"><span class="pre">/proc/<pid>/environ</span></code>).</p>
|
||||
<p>The bare <code class="docutils literal notranslate"><span class="pre">DEBUG</span></code> / <code class="docutils literal notranslate"><span class="pre">PARSEDMARC_DEBUG</span></code> aliases and <code class="docutils literal notranslate"><span class="pre">PARSEDMARC_CONFIG_FILE</span></code>
|
||||
do not have a <code class="docutils literal notranslate"><span class="pre">_FILE</span></code> form; only <code class="docutils literal notranslate"><span class="pre">PARSEDMARC_{SECTION}_{KEY}</span></code> vars resolved
|
||||
to a known config section are eligible.</p>
|
||||
<p>If both the direct env var and the <code class="docutils literal notranslate"><span class="pre">_FILE</span></code> variant are set, the <code class="docutils literal notranslate"><span class="pre">_FILE</span></code>
|
||||
variant wins. If the file does not exist or is unreadable, parsedmarc
|
||||
exits with a configuration error rather than silently falling back to an
|
||||
empty value.</p>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><span class="nt">secrets</span><span class="p">:</span>
|
||||
<span class="w"> </span><span class="nt">imap_password</span><span class="p">:</span>
|
||||
<span class="w"> </span><span class="nt">file</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">./secrets/imap_password.txt</span>
|
||||
|
||||
<span class="nt">services</span><span class="p">:</span>
|
||||
<span class="w"> </span><span class="nt">parsedmarc</span><span class="p">:</span>
|
||||
<span class="w"> </span><span class="nt">image</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">parsedmarc:latest</span>
|
||||
<span class="w"> </span><span class="nt">secrets</span><span class="p">:</span>
|
||||
<span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">imap_password</span>
|
||||
<span class="w"> </span><span class="nt">environment</span><span class="p">:</span>
|
||||
<span class="w"> </span><span class="nt">PARSEDMARC_IMAP_HOST</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">imap.example.com</span>
|
||||
<span class="w"> </span><span class="nt">PARSEDMARC_IMAP_USER</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">dmarc@example.com</span>
|
||||
<span class="w"> </span><span class="nt">PARSEDMARC_IMAP_PASSWORD_FILE</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">/run/secrets/imap_password</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
<p>Note that a small set of config keys whose own names already end in
|
||||
<code class="docutils literal notranslate"><span class="pre">_file</span></code> (<code class="docutils literal notranslate"><span class="pre">[general]</span> <span class="pre">log_file</span></code>, <code class="docutils literal notranslate"><span class="pre">[msgraph]</span> <span class="pre">token_file</span></code>,
|
||||
<code class="docutils literal notranslate"><span class="pre">[gmail_api]</span> <span class="pre">credentials_file</span></code>, <code class="docutils literal notranslate"><span class="pre">[gmail_api]</span> <span class="pre">token_file</span></code>) keep their
|
||||
pre-existing meaning when set via <code class="docutils literal notranslate"><span class="pre">PARSEDMARC_..._FILE</span></code> — that env var is
|
||||
the path itself, not a wrapper around a file containing the path. To pass
|
||||
<em>those</em> paths via a Docker secret, double up the suffix
|
||||
(<code class="docutils literal notranslate"><span class="pre">PARSEDMARC_GMAIL_API_CREDENTIALS_FILE_FILE</span></code>); the inner contents are
|
||||
then read and stored as the <code class="docutils literal notranslate"><span class="pre">credentials_file</span></code> value.</p>
|
||||
</section>
|
||||
<section id="section-name-mapping">
|
||||
<h3>Section name mapping<a class="headerlink" href="#section-name-mapping" title="Link to this heading"></a></h3>
|
||||
<p>For sections with underscores in the name, the full section name is used:</p>
|
||||
@@ -817,7 +906,7 @@ a safer starting point for large backfills than aggressive parallelism.</p></li>
|
||||
<li><p>Use <code class="docutils literal notranslate"><span class="pre">mailbox.since</span></code> to process reports in smaller time windows such as <code class="docutils literal notranslate"><span class="pre">1d</span></code>,
|
||||
<code class="docutils literal notranslate"><span class="pre">7d</span></code>, or another interval that fits the backlog. This makes it easier to catch
|
||||
up incrementally instead of loading an entire mailbox history in one run.</p></li>
|
||||
<li><p>Set <code class="docutils literal notranslate"><span class="pre">strip_attachment_payloads</span> <span class="pre">=</span> <span class="pre">True</span></code> when forensic reports contain large
|
||||
<li><p>Set <code class="docutils literal notranslate"><span class="pre">strip_attachment_payloads</span> <span class="pre">=</span> <span class="pre">True</span></code> when failure reports contain large
|
||||
attachments and you do not need to retain the raw payloads in the parsed
|
||||
output.</p></li>
|
||||
<li><p>Prefer running parsedmarc separately from Elasticsearch or OpenSearch, or
|
||||
|
||||
Reference in New Issue
Block a user