mirror of
https://github.com/domainaware/parsedmarc.git
synced 2026-08-02 21:52:18 +00:00
Update docs
This commit is contained in:
Sean Whalen
+2
-2
@@ -5,14 +5,14 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Overview: module code — parsedmarc 10.2.4 documentation</title>
|
||||
<title>Overview: module code — parsedmarc 10.3.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="../_static/css/theme.css?v=9edc463e" />
|
||||
|
||||
|
||||
<script src="../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../_static/documentation_options.js?v=2c8206f1"></script>
|
||||
<script src="../_static/documentation_options.js?v=a126560f"></script>
|
||||
<script src="../_static/doctools.js?v=fd6eb6e6"></script>
|
||||
<script src="../_static/sphinx_highlight.js?v=6ffebe34"></script>
|
||||
<script src="../_static/js/theme.js"></script>
|
||||
|
||||
+77
-124
@@ -5,14 +5,14 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc — parsedmarc 10.2.4 documentation</title>
|
||||
<title>parsedmarc — parsedmarc 10.3.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="../_static/css/theme.css?v=9edc463e" />
|
||||
|
||||
|
||||
<script src="../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../_static/documentation_options.js?v=2c8206f1"></script>
|
||||
<script src="../_static/documentation_options.js?v=a126560f"></script>
|
||||
<script src="../_static/doctools.js?v=fd6eb6e6"></script>
|
||||
<script src="../_static/sphinx_highlight.js?v=6ffebe34"></script>
|
||||
<script src="../_static/js/theme.js"></script>
|
||||
@@ -115,6 +115,7 @@
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">xmltodict</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">expiringdict</span><span class="w"> </span><span class="kn">import</span> <span class="n">ExpiringDict</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">mailsuite.smtp</span><span class="w"> </span><span class="kn">import</span> <span class="n">send_email</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">tqdm</span><span class="w"> </span><span class="kn">import</span> <span class="n">tqdm</span>
|
||||
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.constants</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
|
||||
<span class="n">DEFAULT_DNS_MAX_RETRIES</span><span class="p">,</span>
|
||||
@@ -147,7 +148,7 @@
|
||||
<span class="n">timestamp_to_human</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"parsedmarc v</span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">__version__</span><span class="p">))</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">"parsedmarc v</span><span class="si">{</span><span class="n">__version__</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
|
||||
<span class="n">feedback_report_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">"^([\w\-]+): (.+)$"</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">MULTILINE</span><span class="p">)</span>
|
||||
<span class="n">xml_header_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">"^<\?xml .*?>"</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">MULTILINE</span><span class="p">)</span>
|
||||
@@ -254,7 +255,7 @@
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">frames</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="s2">""</span>
|
||||
<span class="n">last</span> <span class="o">=</span> <span class="n">frames</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
|
||||
<span class="k">return</span> <span class="s2">" (raised at </span><span class="si">{0}</span><span class="s2">:</span><span class="si">{1}</span><span class="s2">)"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">last</span><span class="o">.</span><span class="n">filename</span><span class="p">,</span> <span class="n">last</span><span class="o">.</span><span class="n">lineno</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="sa">f</span><span class="s2">" (raised at </span><span class="si">{</span><span class="n">last</span><span class="o">.</span><span class="n">filename</span><span class="si">}</span><span class="s2">:</span><span class="si">{</span><span class="n">last</span><span class="o">.</span><span class="n">lineno</span><span class="si">}</span><span class="s2">)"</span>
|
||||
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">_text</span><span class="p">(</span><span class="n">value</span><span class="p">:</span> <span class="n">Any</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">:</span>
|
||||
@@ -930,7 +931,7 @@
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">xmltodict</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">xml</span><span class="p">)[</span><span class="s2">"feedback"</span><span class="p">]</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">errors</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">"Invalid XML: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
||||
<span class="n">errors</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Invalid XML: </span><span class="si">{</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">tree</span> <span class="o">=</span> <span class="n">etree</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span>
|
||||
<span class="n">BytesIO</span><span class="p">(</span><span class="n">xml</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s2">"utf-8"</span><span class="p">)),</span>
|
||||
@@ -978,13 +979,10 @@
|
||||
<span class="k">if</span> <span class="n">new_org_name</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">org_name</span> <span class="o">=</span> <span class="n">new_org_name</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">org_name</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
||||
<span class="s2">"Could not parse org_name from XML.</span><span class="se">\r\n</span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">report</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Could not parse org_name from XML.</span><span class="se">\r\n</span><span class="si">{</span><span class="n">report</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="k">raise</span> <span class="ne">KeyError</span><span class="p">(</span>
|
||||
<span class="s2">"Organization name is missing. </span><span class="se">\</span>
|
||||
<span class="s2"> This field is a requirement for </span><span class="se">\</span>
|
||||
<span class="s2"> saving the report"</span>
|
||||
<span class="s2">"Organization name is missing. This field is a requirement "</span>
|
||||
<span class="s2">"for saving the report"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"org_name"</span><span class="p">]</span> <span class="o">=</span> <span class="n">org_name</span>
|
||||
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"org_email"</span><span class="p">]</span> <span class="o">=</span> <span class="n">report_metadata</span><span class="p">[</span><span class="s2">"email"</span><span class="p">]</span>
|
||||
@@ -1083,14 +1081,14 @@
|
||||
<span class="k">if</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"np"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">np_</span> <span class="o">=</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"np"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="n">np_</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">(</span><span class="s2">"none"</span><span class="p">,</span> <span class="s2">"quarantine"</span><span class="p">,</span> <span class="s2">"reject"</span><span class="p">):</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"Invalid np value: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">np_</span><span class="p">))</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Invalid np value: </span><span class="si">{</span><span class="n">np_</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="n">new_policy_published</span><span class="p">[</span><span class="s2">"np"</span><span class="p">]</span> <span class="o">=</span> <span class="n">np_</span>
|
||||
<span class="n">testing</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="k">if</span> <span class="s2">"testing"</span> <span class="ow">in</span> <span class="n">policy_published</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"testing"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">testing</span> <span class="o">=</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"testing"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="n">testing</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">(</span><span class="s2">"n"</span><span class="p">,</span> <span class="s2">"y"</span><span class="p">):</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"Invalid testing value: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">testing</span><span class="p">))</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Invalid testing value: </span><span class="si">{</span><span class="n">testing</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="n">new_policy_published</span><span class="p">[</span><span class="s2">"testing"</span><span class="p">]</span> <span class="o">=</span> <span class="n">testing</span>
|
||||
<span class="n">discovery_method</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="k">if</span> <span class="s2">"discovery_method"</span> <span class="ow">in</span> <span class="n">policy_published</span><span class="p">:</span>
|
||||
@@ -1098,7 +1096,7 @@
|
||||
<span class="n">discovery_method</span> <span class="o">=</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"discovery_method"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="n">discovery_method</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">(</span><span class="s2">"psl"</span><span class="p">,</span> <span class="s2">"treewalk"</span><span class="p">):</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
||||
<span class="s2">"Invalid discovery_method value: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">discovery_method</span><span class="p">)</span>
|
||||
<span class="sa">f</span><span class="s2">"Invalid discovery_method value: </span><span class="si">{</span><span class="n">discovery_method</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">new_policy_published</span><span class="p">[</span><span class="s2">"discovery_method"</span><span class="p">]</span> <span class="o">=</span> <span class="n">discovery_method</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">]</span> <span class="o">=</span> <span class="n">new_policy_published</span>
|
||||
@@ -1108,7 +1106,7 @@
|
||||
<span class="k">if</span> <span class="n">keep_alive</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">i</span> <span class="o">></span> <span class="mi">0</span> <span class="ow">and</span> <span class="n">i</span> <span class="o">%</span> <span class="mi">20</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Sending keepalive cmd"</span><span class="p">)</span>
|
||||
<span class="n">keep_alive</span><span class="p">()</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Processed </span><span class="si">{0}</span><span class="s2">/</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">i</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">"record"</span><span class="p">])))</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Processed </span><span class="si">{}</span><span class="s2">/</span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">i</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">"record"</span><span class="p">])))</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">report_record</span> <span class="o">=</span> <span class="n">_parse_report_record</span><span class="p">(</span>
|
||||
<span class="n">report</span><span class="p">[</span><span class="s2">"record"</span><span class="p">][</span><span class="n">i</span><span class="p">],</span>
|
||||
@@ -1130,7 +1128,7 @@
|
||||
<span class="n">normalize</span><span class="o">=</span><span class="n">normalize_timespan</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"Could not parse record: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="p">))</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Could not parse record: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">report_record</span> <span class="o">=</span> <span class="n">_parse_report_record</span><span class="p">(</span>
|
||||
@@ -1158,20 +1156,16 @@
|
||||
<span class="k">return</span> <span class="n">cast</span><span class="p">(</span><span class="n">AggregateReport</span><span class="p">,</span> <span class="n">new_report</span><span class="p">)</span>
|
||||
|
||||
<span class="k">except</span> <span class="n">expat</span><span class="o">.</span><span class="n">ExpatError</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidAggregateReport</span><span class="p">(</span>
|
||||
<span class="s2">"Invalid XML: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">error</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidAggregateReport</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Invalid XML: </span><span class="si">{</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">error</span>
|
||||
|
||||
<span class="k">except</span> <span class="ne">KeyError</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidAggregateReport</span><span class="p">(</span>
|
||||
<span class="s2">"Missing field: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">error</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidAggregateReport</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Missing field: </span><span class="si">{</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">error</span>
|
||||
<span class="k">except</span> <span class="ne">AttributeError</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidAggregateReport</span><span class="p">(</span><span class="s2">"Report missing required section"</span><span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">error</span>
|
||||
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidAggregateReport</span><span class="p">(</span>
|
||||
<span class="s2">"Unexpected error: </span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">(),</span> <span class="n">_exc_origin</span><span class="p">(</span><span class="n">error</span><span class="p">))</span>
|
||||
<span class="sa">f</span><span class="s2">"Unexpected error: </span><span class="si">{</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}{</span><span class="n">_exc_origin</span><span class="p">(</span><span class="n">error</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">error</span></div>
|
||||
|
||||
|
||||
@@ -1252,7 +1246,7 @@
|
||||
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">ParserError</span><span class="p">(</span>
|
||||
<span class="s2">"Invalid archive file: </span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">(),</span> <span class="n">_exc_origin</span><span class="p">(</span><span class="n">error</span><span class="p">))</span>
|
||||
<span class="sa">f</span><span class="s2">"Invalid archive file: </span><span class="si">{</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}{</span><span class="n">_exc_origin</span><span class="p">(</span><span class="n">error</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">error</span>
|
||||
<span class="k">finally</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">file_object</span><span class="p">:</span>
|
||||
@@ -1731,13 +1725,11 @@
|
||||
<span class="k">return</span> <span class="n">cast</span><span class="p">(</span><span class="n">FailureReport</span><span class="p">,</span> <span class="n">parsed_report</span><span class="p">)</span>
|
||||
|
||||
<span class="k">except</span> <span class="ne">KeyError</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidFailureReport</span><span class="p">(</span>
|
||||
<span class="s2">"Missing value: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">error</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidFailureReport</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Missing value: </span><span class="si">{</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">error</span>
|
||||
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidFailureReport</span><span class="p">(</span>
|
||||
<span class="s2">"Unexpected error: </span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">(),</span> <span class="n">_exc_origin</span><span class="p">(</span><span class="n">error</span><span class="p">))</span>
|
||||
<span class="sa">f</span><span class="s2">"Unexpected error: </span><span class="si">{</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}{</span><span class="n">_exc_origin</span><span class="p">(</span><span class="n">error</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">error</span></div>
|
||||
|
||||
|
||||
@@ -1922,7 +1914,7 @@
|
||||
<span class="n">sample</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="n">is_feedback_report</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
<span class="k">if</span> <span class="s2">"From"</span> <span class="ow">in</span> <span class="n">msg_headers</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Parsing mail from </span><span class="si">{0}</span><span class="s2"> on </span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">msg_headers</span><span class="p">[</span><span class="s2">"From"</span><span class="p">],</span> <span class="n">date</span><span class="p">))</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Parsing mail from </span><span class="si">{}</span><span class="s2"> on </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">msg_headers</span><span class="p">[</span><span class="s2">"From"</span><span class="p">],</span> <span class="n">date</span><span class="p">))</span>
|
||||
<span class="k">if</span> <span class="s2">"Subject"</span> <span class="ow">in</span> <span class="n">msg_headers</span><span class="p">:</span>
|
||||
<span class="n">subject</span> <span class="o">=</span> <span class="n">msg_headers</span><span class="p">[</span><span class="s2">"Subject"</span><span class="p">]</span>
|
||||
<span class="k">for</span> <span class="n">part</span> <span class="ow">in</span> <span class="n">msg</span><span class="o">.</span><span class="n">walk</span><span class="p">():</span>
|
||||
@@ -1972,9 +1964,7 @@
|
||||
<span class="n">fields</span><span class="p">[</span><span class="s2">"received-date"</span><span class="p">],</span> <span class="n">fields</span><span class="p">[</span><span class="s2">"sender-ip-address"</span><span class="p">]</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">error</span> <span class="o">=</span> <span class="s1">'Unable to parse message with subject "</span><span class="si">{0}</span><span class="s1">": </span><span class="si">{1}{2}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">subject</span><span class="p">,</span> <span class="n">e</span><span class="p">,</span> <span class="n">_exc_origin</span><span class="p">(</span><span class="n">e</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">error</span> <span class="o">=</span> <span class="sa">f</span><span class="s1">'Unable to parse message with subject "</span><span class="si">{</span><span class="n">subject</span><span class="si">}</span><span class="s1">": </span><span class="si">{</span><span class="n">e</span><span class="si">}{</span><span class="n">_exc_origin</span><span class="p">(</span><span class="n">e</span><span class="p">)</span><span class="si">}</span><span class="s1">'</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidDMARCReport</span><span class="p">(</span><span class="n">error</span><span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">e</span>
|
||||
|
||||
<span class="n">sample</span> <span class="o">=</span> <span class="n">parts</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">lstrip</span><span class="p">()</span>
|
||||
@@ -2016,15 +2006,12 @@
|
||||
|
||||
<span class="k">except</span> <span class="n">InvalidDMARCReport</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">error</span> <span class="o">=</span> <span class="p">(</span>
|
||||
<span class="s1">'Message with subject "</span><span class="si">{0}</span><span class="s1">" is not a valid '</span>
|
||||
<span class="s2">"DMARC report: </span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">subject</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
|
||||
<span class="sa">f</span><span class="s1">'Message with subject "</span><span class="si">{</span><span class="n">subject</span><span class="si">}</span><span class="s1">" is not a valid DMARC report: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s1">'</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">raise</span> <span class="n">ParserError</span><span class="p">(</span><span class="n">error</span><span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">e</span>
|
||||
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">error</span> <span class="o">=</span> <span class="s1">'Unable to parse message with subject "</span><span class="si">{0}</span><span class="s1">": </span><span class="si">{1}{2}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">subject</span><span class="p">,</span> <span class="n">e</span><span class="p">,</span> <span class="n">_exc_origin</span><span class="p">(</span><span class="n">e</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">error</span> <span class="o">=</span> <span class="sa">f</span><span class="s1">'Unable to parse message with subject "</span><span class="si">{</span><span class="n">subject</span><span class="si">}</span><span class="s1">": </span><span class="si">{</span><span class="n">e</span><span class="si">}{</span><span class="n">_exc_origin</span><span class="p">(</span><span class="n">e</span><span class="p">)</span><span class="si">}</span><span class="s1">'</span>
|
||||
<span class="k">raise</span> <span class="n">ParserError</span><span class="p">(</span><span class="n">error</span><span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">e</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">feedback_report</span> <span class="ow">and</span> <span class="n">sample</span><span class="p">:</span>
|
||||
@@ -2045,9 +2032,9 @@
|
||||
<span class="p">)</span>
|
||||
<span class="k">except</span> <span class="n">InvalidFailureReport</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">error</span> <span class="o">=</span> <span class="p">(</span>
|
||||
<span class="s1">'Message with subject "</span><span class="si">{0}</span><span class="s1">" '</span>
|
||||
<span class="sa">f</span><span class="s1">'Message with subject "</span><span class="si">{</span><span class="n">subject</span><span class="si">}</span><span class="s1">" '</span>
|
||||
<span class="s2">"is not a valid "</span>
|
||||
<span class="s2">"failure DMARC report: </span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">subject</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
|
||||
<span class="sa">f</span><span class="s2">"failure DMARC report: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidFailureReport</span><span class="p">(</span><span class="n">error</span><span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">e</span>
|
||||
|
||||
@@ -2055,7 +2042,7 @@
|
||||
<span class="k">return</span> <span class="n">result</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">result</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">error</span> <span class="o">=</span> <span class="s1">'Message with subject "</span><span class="si">{0}</span><span class="s1">" is not a valid report'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">subject</span><span class="p">)</span>
|
||||
<span class="n">error</span> <span class="o">=</span> <span class="sa">f</span><span class="s1">'Message with subject "</span><span class="si">{</span><span class="n">subject</span><span class="si">}</span><span class="s1">" is not a valid report'</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidDMARCReport</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">result</span></div>
|
||||
@@ -2099,11 +2086,11 @@
|
||||
<span class="n">sniff</span> <span class="o">=</span> <span class="n">sniff</span><span class="o">.</span><span class="n">lstrip</span><span class="p">()</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">sniff</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">"<"</span><span class="p">):</span>
|
||||
<span class="k">return</span> <span class="s2">"Invalid aggregate report: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">aggregate_error</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="sa">f</span><span class="s2">"Invalid aggregate report: </span><span class="si">{</span><span class="n">aggregate_error</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="k">if</span> <span class="n">sniff</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">"{"</span><span class="p">):</span>
|
||||
<span class="k">return</span> <span class="s2">"Invalid SMTP TLS report: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">smtp_tls_error</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="sa">f</span><span class="s2">"Invalid SMTP TLS report: </span><span class="si">{</span><span class="n">smtp_tls_error</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="k">if</span> <span class="n">_looks_like_email</span><span class="p">(</span><span class="n">sniff</span><span class="p">):</span>
|
||||
<span class="k">return</span> <span class="s2">"Invalid report email: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">email_error</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="sa">f</span><span class="s2">"Invalid report email: </span><span class="si">{</span><span class="n">email_error</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="k">return</span> <span class="p">(</span>
|
||||
<span class="s2">"Not a recognized report format (not a DMARC aggregate XML report, "</span>
|
||||
<span class="s2">"an SMTP TLS JSON report, or a DMARC report email)"</span>
|
||||
@@ -2153,7 +2140,7 @@
|
||||
<span class="n">file_object</span><span class="p">:</span> <span class="n">BinaryIO</span>
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">input_</span><span class="p">,</span> <span class="p">(</span><span class="nb">str</span><span class="p">,</span> <span class="n">os</span><span class="o">.</span><span class="n">PathLike</span><span class="p">)):</span>
|
||||
<span class="n">file_path</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">fspath</span><span class="p">(</span><span class="n">input_</span><span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Parsing </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">file_path</span><span class="p">))</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Parsing </span><span class="si">{</span><span class="n">file_path</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="n">file_object</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="n">file_path</span><span class="p">,</span> <span class="s2">"rb"</span><span class="p">)</span>
|
||||
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">input_</span><span class="p">,</span> <span class="p">(</span><span class="nb">bytes</span><span class="p">,</span> <span class="nb">bytearray</span><span class="p">,</span> <span class="nb">memoryview</span><span class="p">)):</span>
|
||||
<span class="n">file_object</span> <span class="o">=</span> <span class="n">BytesIO</span><span class="p">(</span><span class="nb">bytes</span><span class="p">(</span><span class="n">input_</span><span class="p">))</span>
|
||||
@@ -2265,10 +2252,10 @@
|
||||
<span class="n">mbox</span> <span class="o">=</span> <span class="n">mailbox</span><span class="o">.</span><span class="n">mbox</span><span class="p">(</span><span class="n">input_</span><span class="p">)</span>
|
||||
<span class="n">message_keys</span> <span class="o">=</span> <span class="n">mbox</span><span class="o">.</span><span class="n">keys</span><span class="p">()</span>
|
||||
<span class="n">total_messages</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">message_keys</span><span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Found </span><span class="si">{0}</span><span class="s2"> messages in </span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">total_messages</span><span class="p">,</span> <span class="n">input_</span><span class="p">))</span>
|
||||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">message_keys</span><span class="p">)):</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Found </span><span class="si">{</span><span class="n">total_messages</span><span class="si">}</span><span class="s2"> messages in </span><span class="si">{</span><span class="n">input_</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">tqdm</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="n">total_messages</span><span class="p">),</span> <span class="n">disable</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
<span class="n">message_key</span> <span class="o">=</span> <span class="n">message_keys</span><span class="p">[</span><span class="n">i</span><span class="p">]</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Processing message </span><span class="si">{0}</span><span class="s2"> of </span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">i</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="n">total_messages</span><span class="p">))</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Processing message </span><span class="si">{</span><span class="n">i</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="mi">1</span><span class="si">}</span><span class="s2"> of </span><span class="si">{</span><span class="n">total_messages</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="n">msg_content</span> <span class="o">=</span> <span class="n">mbox</span><span class="o">.</span><span class="n">get_string</span><span class="p">(</span><span class="n">message_key</span><span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">sa</span> <span class="o">=</span> <span class="n">strip_attachment_payloads</span>
|
||||
@@ -2304,7 +2291,7 @@
|
||||
<span class="k">except</span> <span class="n">InvalidDMARCReport</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="k">except</span> <span class="n">mailbox</span><span class="o">.</span><span class="n">NoSuchMailboxError</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidDMARCReport</span><span class="p">(</span><span class="s2">"Mailbox </span><span class="si">{0}</span><span class="s2"> does not exist"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">input_</span><span class="p">))</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidDMARCReport</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Mailbox </span><span class="si">{</span><span class="n">input_</span><span class="si">}</span><span class="s2"> does not exist"</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="p">{</span>
|
||||
<span class="s2">"aggregate_reports"</span><span class="p">:</span> <span class="n">aggregate_reports</span><span class="p">,</span>
|
||||
<span class="s2">"failure_reports"</span><span class="p">:</span> <span class="n">failure_reports</span><span class="p">,</span>
|
||||
@@ -2331,8 +2318,8 @@
|
||||
<span class="sd"> (warn, don't crash). Uses the folder-management API added in mailsuite</span>
|
||||
<span class="sd"> 2.1.0 (``folder_exists`` / ``rename_folder`` / ``merge_folders``).</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">old_folder</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">/Forensic"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">archive_folder</span><span class="p">)</span>
|
||||
<span class="n">new_folder</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">/Failure"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">archive_folder</span><span class="p">)</span>
|
||||
<span class="n">old_folder</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">archive_folder</span><span class="si">}</span><span class="s2">/Forensic"</span>
|
||||
<span class="n">new_folder</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">archive_folder</span><span class="si">}</span><span class="s2">/Failure"</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">connection</span><span class="o">.</span><span class="n">folder_exists</span><span class="p">(</span><span class="n">old_folder</span><span class="p">):</span>
|
||||
<span class="k">return</span>
|
||||
@@ -2341,23 +2328,13 @@
|
||||
<span class="c1"># created Failure folder): move the legacy folder's messages into</span>
|
||||
<span class="c1"># the new one and drop the now-empty legacy folder.</span>
|
||||
<span class="n">connection</span><span class="o">.</span><span class="n">merge_folders</span><span class="p">(</span><span class="n">old_folder</span><span class="p">,</span> <span class="n">new_folder</span><span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
|
||||
<span class="s2">"Merged legacy archive folder </span><span class="si">{0}</span><span class="s2"> into </span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">old_folder</span><span class="p">,</span> <span class="n">new_folder</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Merged legacy archive folder </span><span class="si">{</span><span class="n">old_folder</span><span class="si">}</span><span class="s2"> into </span><span class="si">{</span><span class="n">new_folder</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">connection</span><span class="o">.</span><span class="n">rename_folder</span><span class="p">(</span><span class="n">old_folder</span><span class="p">,</span> <span class="n">new_folder</span><span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
|
||||
<span class="s2">"Renamed legacy archive folder </span><span class="si">{0}</span><span class="s2"> to </span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">old_folder</span><span class="p">,</span> <span class="n">new_folder</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Renamed legacy archive folder </span><span class="si">{</span><span class="n">old_folder</span><span class="si">}</span><span class="s2"> to </span><span class="si">{</span><span class="n">new_folder</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
||||
<span class="s2">"Could not migrate legacy archive folder </span><span class="si">{0}</span><span class="s2"> to </span><span class="si">{1}</span><span class="s2">: </span><span class="si">{2}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">old_folder</span><span class="p">,</span> <span class="n">new_folder</span><span class="p">,</span> <span class="n">error</span>
|
||||
<span class="p">)</span>
|
||||
<span class="sa">f</span><span class="s2">"Could not migrate legacy archive folder </span><span class="si">{</span><span class="n">old_folder</span><span class="si">}</span><span class="s2"> to </span><span class="si">{</span><span class="n">new_folder</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="n">error</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
|
||||
@@ -2432,10 +2409,10 @@
|
||||
<span class="n">aggregate_report_msg_uids</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="n">failure_report_msg_uids</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="n">smtp_tls_msg_uids</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="n">aggregate_reports_folder</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">/Aggregate"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">archive_folder</span><span class="p">)</span>
|
||||
<span class="n">failure_reports_folder</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">/Failure"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">archive_folder</span><span class="p">)</span>
|
||||
<span class="n">smtp_tls_reports_folder</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">/SMTP-TLS"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">archive_folder</span><span class="p">)</span>
|
||||
<span class="n">invalid_reports_folder</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">/Invalid"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">archive_folder</span><span class="p">)</span>
|
||||
<span class="n">aggregate_reports_folder</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">archive_folder</span><span class="si">}</span><span class="s2">/Aggregate"</span>
|
||||
<span class="n">failure_reports_folder</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">archive_folder</span><span class="si">}</span><span class="s2">/Failure"</span>
|
||||
<span class="n">smtp_tls_reports_folder</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">archive_folder</span><span class="si">}</span><span class="s2">/SMTP-TLS"</span>
|
||||
<span class="n">invalid_reports_folder</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">archive_folder</span><span class="si">}</span><span class="s2">/Invalid"</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">results</span><span class="p">:</span>
|
||||
<span class="n">aggregate_reports</span> <span class="o">=</span> <span class="n">results</span><span class="p">[</span><span class="s2">"aggregate_reports"</span><span class="p">]</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
@@ -2464,17 +2441,17 @@
|
||||
<span class="n">_since</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">s</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span> <span class="o">*</span> <span class="mi">60</span> <span class="o">*</span> <span class="mi">24</span> <span class="o">*</span> <span class="mi">7</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
||||
<span class="s2">"Incorrect format for 'since' option. </span><span class="se">\</span>
|
||||
<span class="s2"> Provided value:</span><span class="si">{0}</span><span class="s2">, Expected values:(5m|3h|2d|1w). </span><span class="se">\</span>
|
||||
<span class="s2"> Ignoring option, fetching messages for last 24hrs"</span>
|
||||
<span class="s2">"SMTP does not support a time or timezone in since."</span>
|
||||
<span class="s2">"See https://www.rfc-editor.org/rfc/rfc3501#page-52"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">since</span><span class="p">)</span>
|
||||
<span class="sa">f</span><span class="s2">"Incorrect format for 'since' option. Provided value: </span><span class="si">{</span><span class="n">since</span><span class="si">}</span><span class="s2">, "</span>
|
||||
<span class="s2">"expected values: (5m|3h|2d|1w). Ignoring option, fetching "</span>
|
||||
<span class="s2">"messages for last 24hrs. SMTP does not support a time or "</span>
|
||||
<span class="s2">"timezone in since. See "</span>
|
||||
<span class="s2">"https://www.rfc-editor.org/rfc/rfc3501#page-52"</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">connection</span><span class="p">,</span> <span class="n">IMAPConnection</span><span class="p">):</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
||||
<span class="s2">"Only days and weeks values in 'since' option are </span><span class="se">\</span>
|
||||
<span class="s2"> considered for IMAP connections. Examples: 2d or 1w"</span>
|
||||
<span class="s2">"Only days and weeks values in 'since' option are considered "</span>
|
||||
<span class="s2">"for IMAP connections. Examples: 2d or 1w"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">since</span> <span class="o">=</span> <span class="p">(</span><span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">(</span><span class="n">timezone</span><span class="o">.</span><span class="n">utc</span><span class="p">)</span> <span class="o">-</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">minutes</span><span class="o">=</span><span class="n">_since</span><span class="p">))</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span>
|
||||
<span class="s2">"</span><span class="si">%d</span><span class="s2">-%b-%Y"</span>
|
||||
@@ -2495,22 +2472,18 @@
|
||||
<span class="n">reports_folder</span><span class="p">,</span> <span class="n">batch_size</span><span class="o">=</span><span class="n">batch_size</span><span class="p">,</span> <span class="n">since</span><span class="o">=</span><span class="n">since</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">total_messages</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">messages</span><span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Found </span><span class="si">{0}</span><span class="s2"> messages in </span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">messages</span><span class="p">),</span> <span class="n">reports_folder</span><span class="p">))</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Found </span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="n">messages</span><span class="p">)</span><span class="si">}</span><span class="s2"> messages in </span><span class="si">{</span><span class="n">reports_folder</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">batch_size</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">since</span><span class="p">:</span>
|
||||
<span class="n">message_limit</span> <span class="o">=</span> <span class="nb">min</span><span class="p">(</span><span class="n">total_messages</span><span class="p">,</span> <span class="n">batch_size</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">message_limit</span> <span class="o">=</span> <span class="n">total_messages</span>
|
||||
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Processing </span><span class="si">{0}</span><span class="s2"> messages"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">message_limit</span><span class="p">))</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Processing </span><span class="si">{</span><span class="n">message_limit</span><span class="si">}</span><span class="s2"> messages"</span><span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">message_limit</span><span class="p">):</span>
|
||||
<span class="n">msg_uid</span> <span class="o">=</span> <span class="n">messages</span><span class="p">[</span><span class="n">i</span><span class="p">]</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
||||
<span class="s2">"Processing message </span><span class="si">{0}</span><span class="s2"> of </span><span class="si">{1}</span><span class="s2">: UID </span><span class="si">{2}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">i</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="n">message_limit</span><span class="p">,</span> <span class="n">msg_uid</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Processing message </span><span class="si">{</span><span class="n">i</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="mi">1</span><span class="si">}</span><span class="s2"> of </span><span class="si">{</span><span class="n">message_limit</span><span class="si">}</span><span class="s2">: UID </span><span class="si">{</span><span class="n">msg_uid</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="n">message_id</span><span class="p">:</span> <span class="nb">int</span> <span class="o">|</span> <span class="nb">str</span>
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">connection</span><span class="p">,</span> <span class="n">IMAPConnection</span><span class="p">):</span>
|
||||
<span class="n">message_id</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">msg_uid</span><span class="p">)</span>
|
||||
@@ -2562,16 +2535,14 @@
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">test</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">delete</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Deleting message UID </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">msg_uid</span><span class="p">))</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Deleting message UID </span><span class="si">{</span><span class="n">msg_uid</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">connection</span><span class="p">,</span> <span class="n">IMAPConnection</span><span class="p">):</span>
|
||||
<span class="n">connection</span><span class="o">.</span><span class="n">delete_message</span><span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="n">message_id</span><span class="p">))</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">connection</span><span class="o">.</span><span class="n">delete_message</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">message_id</span><span class="p">))</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
||||
<span class="s2">"Moving message UID </span><span class="si">{0}</span><span class="s2"> to </span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">msg_uid</span><span class="p">,</span> <span class="n">invalid_reports_folder</span>
|
||||
<span class="p">)</span>
|
||||
<span class="sa">f</span><span class="s2">"Moving message UID </span><span class="si">{</span><span class="n">msg_uid</span><span class="si">}</span><span class="s2"> to </span><span class="si">{</span><span class="n">invalid_reports_folder</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">connection</span><span class="p">,</span> <span class="n">IMAPConnection</span><span class="p">):</span>
|
||||
<span class="n">connection</span><span class="o">.</span><span class="n">move_message</span><span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="n">message_id</span><span class="p">),</span> <span class="n">invalid_reports_folder</span><span class="p">)</span>
|
||||
@@ -2588,81 +2559,63 @@
|
||||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">number_of_processed_msgs</span><span class="p">):</span>
|
||||
<span class="n">msg_uid</span> <span class="o">=</span> <span class="n">processed_messages</span><span class="p">[</span><span class="n">i</span><span class="p">]</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
||||
<span class="s2">"Deleting message </span><span class="si">{0}</span><span class="s2"> of </span><span class="si">{1}</span><span class="s2">: UID </span><span class="si">{2}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">i</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="n">number_of_processed_msgs</span><span class="p">,</span> <span class="n">msg_uid</span>
|
||||
<span class="p">)</span>
|
||||
<span class="sa">f</span><span class="s2">"Deleting message </span><span class="si">{</span><span class="n">i</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="mi">1</span><span class="si">}</span><span class="s2"> of </span><span class="si">{</span><span class="n">number_of_processed_msgs</span><span class="si">}</span><span class="s2">: UID </span><span class="si">{</span><span class="n">msg_uid</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">connection</span><span class="o">.</span><span class="n">delete_message</span><span class="p">(</span><span class="n">msg_uid</span><span class="p">)</span>
|
||||
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">message</span> <span class="o">=</span> <span class="s2">"Error deleting message UID"</span>
|
||||
<span class="n">e</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2"> </span><span class="si">{1}</span><span class="s2">: </span><span class="si">{2}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">message</span><span class="p">,</span> <span class="n">msg_uid</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Mailbox error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="p">))</span>
|
||||
<span class="n">e</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">message</span><span class="si">}</span><span class="s2"> </span><span class="si">{</span><span class="n">msg_uid</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Mailbox error: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">aggregate_report_msg_uids</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="n">log_message</span> <span class="o">=</span> <span class="s2">"Moving aggregate report messages from"</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
||||
<span class="s2">"</span><span class="si">{0}</span><span class="s2"> </span><span class="si">{1}</span><span class="s2"> to </span><span class="si">{2}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">log_message</span><span class="p">,</span> <span class="n">reports_folder</span><span class="p">,</span> <span class="n">aggregate_reports_folder</span>
|
||||
<span class="p">)</span>
|
||||
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">log_message</span><span class="si">}</span><span class="s2"> </span><span class="si">{</span><span class="n">reports_folder</span><span class="si">}</span><span class="s2"> to </span><span class="si">{</span><span class="n">aggregate_reports_folder</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">number_of_agg_report_msgs</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">aggregate_report_msg_uids</span><span class="p">)</span>
|
||||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">number_of_agg_report_msgs</span><span class="p">):</span>
|
||||
<span class="n">msg_uid</span> <span class="o">=</span> <span class="n">aggregate_report_msg_uids</span><span class="p">[</span><span class="n">i</span><span class="p">]</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
||||
<span class="s2">"Moving message </span><span class="si">{0}</span><span class="s2"> of </span><span class="si">{1}</span><span class="s2">: UID </span><span class="si">{2}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">i</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="n">number_of_agg_report_msgs</span><span class="p">,</span> <span class="n">msg_uid</span>
|
||||
<span class="p">)</span>
|
||||
<span class="sa">f</span><span class="s2">"Moving message </span><span class="si">{</span><span class="n">i</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="mi">1</span><span class="si">}</span><span class="s2"> of </span><span class="si">{</span><span class="n">number_of_agg_report_msgs</span><span class="si">}</span><span class="s2">: UID </span><span class="si">{</span><span class="n">msg_uid</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">connection</span><span class="o">.</span><span class="n">move_message</span><span class="p">(</span><span class="n">msg_uid</span><span class="p">,</span> <span class="n">aggregate_reports_folder</span><span class="p">)</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">message</span> <span class="o">=</span> <span class="s2">"Error moving message UID"</span>
|
||||
<span class="n">e</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2"> </span><span class="si">{1}</span><span class="s2">: </span><span class="si">{2}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">message</span><span class="p">,</span> <span class="n">msg_uid</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Mailbox error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="p">))</span>
|
||||
<span class="n">e</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">message</span><span class="si">}</span><span class="s2"> </span><span class="si">{</span><span class="n">msg_uid</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Mailbox error: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">failure_report_msg_uids</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="n">message</span> <span class="o">=</span> <span class="s2">"Moving failure report messages from"</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
||||
<span class="s2">"</span><span class="si">{0}</span><span class="s2"> </span><span class="si">{1}</span><span class="s2"> to </span><span class="si">{2}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">message</span><span class="p">,</span> <span class="n">reports_folder</span><span class="p">,</span> <span class="n">failure_reports_folder</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">message</span><span class="si">}</span><span class="s2"> </span><span class="si">{</span><span class="n">reports_folder</span><span class="si">}</span><span class="s2"> to </span><span class="si">{</span><span class="n">failure_reports_folder</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="n">number_of_failure_msgs</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">failure_report_msg_uids</span><span class="p">)</span>
|
||||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">number_of_failure_msgs</span><span class="p">):</span>
|
||||
<span class="n">msg_uid</span> <span class="o">=</span> <span class="n">failure_report_msg_uids</span><span class="p">[</span><span class="n">i</span><span class="p">]</span>
|
||||
<span class="n">message</span> <span class="o">=</span> <span class="s2">"Moving message"</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
||||
<span class="s2">"</span><span class="si">{0}</span><span class="s2"> </span><span class="si">{1}</span><span class="s2"> of </span><span class="si">{2}</span><span class="s2">: UID </span><span class="si">{3}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">message</span><span class="p">,</span> <span class="n">i</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="n">number_of_failure_msgs</span><span class="p">,</span> <span class="n">msg_uid</span>
|
||||
<span class="p">)</span>
|
||||
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">message</span><span class="si">}</span><span class="s2"> </span><span class="si">{</span><span class="n">i</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="mi">1</span><span class="si">}</span><span class="s2"> of </span><span class="si">{</span><span class="n">number_of_failure_msgs</span><span class="si">}</span><span class="s2">: UID </span><span class="si">{</span><span class="n">msg_uid</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">connection</span><span class="o">.</span><span class="n">move_message</span><span class="p">(</span><span class="n">msg_uid</span><span class="p">,</span> <span class="n">failure_reports_folder</span><span class="p">)</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">e</span> <span class="o">=</span> <span class="s2">"Error moving message UID </span><span class="si">{0}</span><span class="s2">: </span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">msg_uid</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Mailbox error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="p">))</span>
|
||||
<span class="n">e</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"Error moving message UID </span><span class="si">{</span><span class="n">msg_uid</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Mailbox error: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">smtp_tls_msg_uids</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="n">message</span> <span class="o">=</span> <span class="s2">"Moving SMTP TLS report messages from"</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
||||
<span class="s2">"</span><span class="si">{0}</span><span class="s2"> </span><span class="si">{1}</span><span class="s2"> to </span><span class="si">{2}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">message</span><span class="p">,</span> <span class="n">reports_folder</span><span class="p">,</span> <span class="n">smtp_tls_reports_folder</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">message</span><span class="si">}</span><span class="s2"> </span><span class="si">{</span><span class="n">reports_folder</span><span class="si">}</span><span class="s2"> to </span><span class="si">{</span><span class="n">smtp_tls_reports_folder</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="n">number_of_smtp_tls_uids</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">smtp_tls_msg_uids</span><span class="p">)</span>
|
||||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">number_of_smtp_tls_uids</span><span class="p">):</span>
|
||||
<span class="n">msg_uid</span> <span class="o">=</span> <span class="n">smtp_tls_msg_uids</span><span class="p">[</span><span class="n">i</span><span class="p">]</span>
|
||||
<span class="n">message</span> <span class="o">=</span> <span class="s2">"Moving message"</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
||||
<span class="s2">"</span><span class="si">{0}</span><span class="s2"> </span><span class="si">{1}</span><span class="s2"> of </span><span class="si">{2}</span><span class="s2">: UID </span><span class="si">{3}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">message</span><span class="p">,</span> <span class="n">i</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="n">number_of_smtp_tls_uids</span><span class="p">,</span> <span class="n">msg_uid</span>
|
||||
<span class="p">)</span>
|
||||
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">message</span><span class="si">}</span><span class="s2"> </span><span class="si">{</span><span class="n">i</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="mi">1</span><span class="si">}</span><span class="s2"> of </span><span class="si">{</span><span class="n">number_of_smtp_tls_uids</span><span class="si">}</span><span class="s2">: UID </span><span class="si">{</span><span class="n">msg_uid</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">connection</span><span class="o">.</span><span class="n">move_message</span><span class="p">(</span><span class="n">msg_uid</span><span class="p">,</span> <span class="n">smtp_tls_reports_folder</span><span class="p">)</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">e</span> <span class="o">=</span> <span class="s2">"Error moving message UID </span><span class="si">{0}</span><span class="s2">: </span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">msg_uid</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Mailbox error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="p">))</span>
|
||||
<span class="n">e</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"Error moving message UID </span><span class="si">{</span><span class="n">msg_uid</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Mailbox error: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="n">results</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s2">"aggregate_reports"</span><span class="p">:</span> <span class="n">aggregate_reports</span><span class="p">,</span>
|
||||
<span class="s2">"failure_reports"</span><span class="p">:</span> <span class="n">failure_reports</span><span class="p">,</span>
|
||||
@@ -2888,7 +2841,7 @@
|
||||
|
||||
<span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">output_directory</span><span class="p">):</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isdir</span><span class="p">(</span><span class="n">output_directory</span><span class="p">):</span>
|
||||
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"</span><span class="si">{0}</span><span class="s2"> is not a directory"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">output_directory</span><span class="p">))</span>
|
||||
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">output_directory</span><span class="si">}</span><span class="s2"> is not a directory"</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">os</span><span class="o">.</span><span class="n">makedirs</span><span class="p">(</span><span class="n">output_directory</span><span class="p">)</span>
|
||||
|
||||
@@ -2935,11 +2888,11 @@
|
||||
|
||||
<span class="k">while</span> <span class="n">filename</span> <span class="ow">in</span> <span class="n">sample_filenames</span><span class="p">:</span>
|
||||
<span class="n">message_count</span> <span class="o">+=</span> <span class="mi">1</span>
|
||||
<span class="n">filename</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2"> (</span><span class="si">{1}</span><span class="s2">)"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">subject</span><span class="p">,</span> <span class="n">message_count</span><span class="p">)</span>
|
||||
<span class="n">filename</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">subject</span><span class="si">}</span><span class="s2"> (</span><span class="si">{</span><span class="n">message_count</span><span class="si">}</span><span class="s2">)"</span>
|
||||
|
||||
<span class="n">sample_filenames</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">filename</span><span class="p">)</span>
|
||||
|
||||
<span class="n">filename</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">.eml"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">filename</span><span class="p">)</span>
|
||||
<span class="n">filename</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">filename</span><span class="si">}</span><span class="s2">.eml"</span>
|
||||
<span class="n">path</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">samples_directory</span><span class="p">,</span> <span class="n">filename</span><span class="p">)</span>
|
||||
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="s2">"w"</span><span class="p">,</span> <span class="n">newline</span><span class="o">=</span><span class="s2">"</span><span class="se">\n</span><span class="s2">"</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s2">"utf-8"</span><span class="p">)</span> <span class="k">as</span> <span class="n">sample_file</span><span class="p">:</span>
|
||||
<span class="n">sample_file</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">sample</span><span class="p">)</span></div>
|
||||
@@ -3013,12 +2966,12 @@
|
||||
<span class="n">attachment_filename</span> <span class="o">+=</span> <span class="s2">".zip"</span>
|
||||
<span class="n">filename</span> <span class="o">=</span> <span class="n">attachment_filename</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">filename</span> <span class="o">=</span> <span class="s2">"DMARC-</span><span class="si">{0}</span><span class="s2">.zip"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">date_string</span><span class="p">)</span>
|
||||
<span class="n">filename</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"DMARC-</span><span class="si">{</span><span class="n">date_string</span><span class="si">}</span><span class="s2">.zip"</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">subject</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">subject</span> <span class="o">=</span> <span class="s2">"DMARC results for </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">date_string</span><span class="p">)</span>
|
||||
<span class="n">subject</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"DMARC results for </span><span class="si">{</span><span class="n">date_string</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="k">if</span> <span class="n">message</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">message</span> <span class="o">=</span> <span class="s2">"DMARC results for </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">date_string</span><span class="p">)</span>
|
||||
<span class="n">message</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"DMARC results for </span><span class="si">{</span><span class="n">date_string</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="n">zip_bytes</span> <span class="o">=</span> <span class="n">get_report_zip</span><span class="p">(</span><span class="n">results</span><span class="p">)</span>
|
||||
<span class="n">attachments</span> <span class="o">=</span> <span class="p">[(</span><span class="n">filename</span><span class="p">,</span> <span class="n">zip_bytes</span><span class="p">)]</span>
|
||||
|
||||
|
||||
@@ -5,14 +5,14 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc.elastic — parsedmarc 10.2.4 documentation</title>
|
||||
<title>parsedmarc.elastic — parsedmarc 10.3.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=9edc463e" />
|
||||
|
||||
|
||||
<script src="../../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../../_static/documentation_options.js?v=2c8206f1"></script>
|
||||
<script src="../../_static/documentation_options.js?v=a126560f"></script>
|
||||
<script src="../../_static/doctools.js?v=fd6eb6e6"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=6ffebe34"></script>
|
||||
<script src="../../_static/js/theme.js"></script>
|
||||
@@ -126,6 +126,181 @@
|
||||
<span class="c1"># settings (e.g. ``refresh_interval``) are accepted and pass through.</span>
|
||||
<span class="n">_SERVERLESS_REJECTED_SETTINGS</span> <span class="o">=</span> <span class="nb">frozenset</span><span class="p">({</span><span class="s2">"number_of_shards"</span><span class="p">,</span> <span class="s2">"number_of_replicas"</span><span class="p">})</span>
|
||||
|
||||
<span class="c1"># Guard query for the dkim_results_combined/spf_results_combined backfill</span>
|
||||
<span class="c1"># (see ``migrate_indexes``). Matches only documents that have at least one</span>
|
||||
<span class="c1"># DKIM or SPF auth result and are missing the corresponding combined field.</span>
|
||||
<span class="c1"># Empty arrays are invisible to ``exists``, so documents with zero</span>
|
||||
<span class="c1"># DKIM/SPF results are correctly skipped (verified against real data;</span>
|
||||
<span class="c1"># this also makes the query idempotent — a backfilled document no longer</span>
|
||||
<span class="c1"># matches). Each result is matched on an OR of its ``domain``/``result``</span>
|
||||
<span class="c1"># subfields as defense in depth: the parsers we audited never store a</span>
|
||||
<span class="c1"># result without both, but an empty string indexes no text tokens and is</span>
|
||||
<span class="c1"># invisible to ``exists``, and the storage shape of every historical</span>
|
||||
<span class="c1"># parsedmarc version can't be audited — matching either subfield costs</span>
|
||||
<span class="c1"># nothing and cannot skip a document that has something to backfill.</span>
|
||||
<span class="n">_COMBINED_BACKFILL_QUERY</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s2">"bool"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"minimum_should_match"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
|
||||
<span class="s2">"should"</span><span class="p">:</span> <span class="p">[</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s2">"bool"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"must"</span><span class="p">:</span> <span class="p">[</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s2">"bool"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"minimum_should_match"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
|
||||
<span class="s2">"should"</span><span class="p">:</span> <span class="p">[</span>
|
||||
<span class="p">{</span><span class="s2">"exists"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"field"</span><span class="p">:</span> <span class="s2">"dkim_results.domain"</span><span class="p">}},</span>
|
||||
<span class="p">{</span><span class="s2">"exists"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"field"</span><span class="p">:</span> <span class="s2">"dkim_results.result"</span><span class="p">}},</span>
|
||||
<span class="p">],</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">],</span>
|
||||
<span class="s2">"must_not"</span><span class="p">:</span> <span class="p">[{</span><span class="s2">"exists"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"field"</span><span class="p">:</span> <span class="s2">"dkim_results_combined"</span><span class="p">}}],</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">},</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s2">"bool"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"must"</span><span class="p">:</span> <span class="p">[</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s2">"bool"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"minimum_should_match"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
|
||||
<span class="s2">"should"</span><span class="p">:</span> <span class="p">[</span>
|
||||
<span class="p">{</span><span class="s2">"exists"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"field"</span><span class="p">:</span> <span class="s2">"spf_results.domain"</span><span class="p">}},</span>
|
||||
<span class="p">{</span><span class="s2">"exists"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"field"</span><span class="p">:</span> <span class="s2">"spf_results.result"</span><span class="p">}},</span>
|
||||
<span class="p">],</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">],</span>
|
||||
<span class="s2">"must_not"</span><span class="p">:</span> <span class="p">[{</span><span class="s2">"exists"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"field"</span><span class="p">:</span> <span class="s2">"spf_results_combined"</span><span class="p">}}],</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">},</span>
|
||||
<span class="p">],</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">}</span>
|
||||
|
||||
<span class="c1"># Painless script that (re)derives dkim_results_combined/spf_results_combined</span>
|
||||
<span class="c1"># from dkim_results/spf_results, matching the format written by</span>
|
||||
<span class="c1"># save_aggregate_report_to_elasticsearch(): "{selector} / {domain} / {result}"</span>
|
||||
<span class="c1"># per DKIM result and "{scope} / {domain} / {result}" per SPF result.</span>
|
||||
<span class="n">_COMBINED_BACKFILL_SCRIPT</span> <span class="o">=</span> <span class="p">(</span>
|
||||
<span class="s2">"List dk = new ArrayList(); "</span>
|
||||
<span class="s2">"def dr = ctx._source.dkim_results; "</span>
|
||||
<span class="s2">"if (dr != null) { "</span>
|
||||
<span class="s2">"if (!(dr instanceof List)) { dr = [dr]; } "</span>
|
||||
<span class="s2">"for (e in dr) { "</span>
|
||||
<span class="s2">"if (e == null) { continue; } "</span>
|
||||
<span class="s1">'def sel = e.selector != null ? e.selector : "none"; '</span>
|
||||
<span class="s1">'def dom = e.domain != null ? e.domain : "none"; '</span>
|
||||
<span class="s1">'def res = e.result != null ? e.result : "none"; '</span>
|
||||
<span class="s1">'dk.add(sel + " / " + dom + " / " + res); '</span>
|
||||
<span class="s2">"} } "</span>
|
||||
<span class="s2">"ctx._source.dkim_results_combined = dk; "</span>
|
||||
<span class="s2">"List sp = new ArrayList(); "</span>
|
||||
<span class="s2">"def sr = ctx._source.spf_results; "</span>
|
||||
<span class="s2">"if (sr != null) { "</span>
|
||||
<span class="s2">"if (!(sr instanceof List)) { sr = [sr]; } "</span>
|
||||
<span class="s2">"for (e in sr) { "</span>
|
||||
<span class="s2">"if (e == null) { continue; } "</span>
|
||||
<span class="s1">'def sc = e.scope != null ? e.scope : "mfrom"; '</span>
|
||||
<span class="s1">'def dom = e.domain != null ? e.domain : "none"; '</span>
|
||||
<span class="s1">'def res = e.result != null ? e.result : (e.results != null ? e.results : "none"); '</span>
|
||||
<span class="s1">'sp.add(sc + " / " + dom + " / " + res); '</span>
|
||||
<span class="s2">"} } "</span>
|
||||
<span class="s2">"ctx._source.spf_results_combined = sp;"</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="c1"># Guard query for the policies_combined/failure_details_combined backfill</span>
|
||||
<span class="c1"># (see ``migrate_indexes``). Matches only SMTP TLS documents that have at</span>
|
||||
<span class="c1"># least one policy or failure detail and are missing the corresponding</span>
|
||||
<span class="c1"># combined field. Empty arrays are invisible to ``exists``, so documents</span>
|
||||
<span class="c1"># with zero policies/failure details are correctly skipped (this also</span>
|
||||
<span class="c1"># makes the query idempotent — a backfilled document no longer matches).</span>
|
||||
<span class="c1"># Each result is matched on an OR of its relevant subfields as defense in</span>
|
||||
<span class="c1"># depth: the parsers we audited never store a policy/failure detail</span>
|
||||
<span class="c1"># without these fields, but an empty string indexes no text tokens and is</span>
|
||||
<span class="c1"># invisible to ``exists``, and the storage shape of every historical</span>
|
||||
<span class="c1"># parsedmarc version can't be audited — matching either subfield costs</span>
|
||||
<span class="c1"># nothing and cannot skip a document that has something to backfill.</span>
|
||||
<span class="n">_SMTP_TLS_COMBINED_BACKFILL_QUERY</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s2">"bool"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"minimum_should_match"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
|
||||
<span class="s2">"should"</span><span class="p">:</span> <span class="p">[</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s2">"bool"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"must"</span><span class="p">:</span> <span class="p">[</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s2">"bool"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"minimum_should_match"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
|
||||
<span class="s2">"should"</span><span class="p">:</span> <span class="p">[</span>
|
||||
<span class="p">{</span><span class="s2">"exists"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"field"</span><span class="p">:</span> <span class="s2">"policies.policy_domain"</span><span class="p">}},</span>
|
||||
<span class="p">{</span><span class="s2">"exists"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"field"</span><span class="p">:</span> <span class="s2">"policies.policy_type"</span><span class="p">}},</span>
|
||||
<span class="p">],</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">],</span>
|
||||
<span class="s2">"must_not"</span><span class="p">:</span> <span class="p">[{</span><span class="s2">"exists"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"field"</span><span class="p">:</span> <span class="s2">"policies_combined"</span><span class="p">}}],</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">},</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s2">"bool"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"must"</span><span class="p">:</span> <span class="p">[</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s2">"bool"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"minimum_should_match"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
|
||||
<span class="s2">"should"</span><span class="p">:</span> <span class="p">[</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s2">"exists"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"field"</span><span class="p">:</span> <span class="s2">"policies.failure_details.result_type"</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">},</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s2">"exists"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"field"</span><span class="p">:</span> <span class="s2">"policies.failure_details.sending_mta_ip"</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">},</span>
|
||||
<span class="p">],</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">],</span>
|
||||
<span class="s2">"must_not"</span><span class="p">:</span> <span class="p">[{</span><span class="s2">"exists"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"field"</span><span class="p">:</span> <span class="s2">"failure_details_combined"</span><span class="p">}}],</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">},</span>
|
||||
<span class="p">],</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">}</span>
|
||||
|
||||
<span class="c1"># Painless script that (re)derives policies_combined/failure_details_combined</span>
|
||||
<span class="c1"># from policies/policies.failure_details, matching the format written by</span>
|
||||
<span class="c1"># save_smtp_tls_report_to_elasticsearch(): "{policy_domain} / {policy_type}"</span>
|
||||
<span class="c1"># per policy and "{policy_domain} / {policy_type} / {result_type} /</span>
|
||||
<span class="c1"># {sending_mta_ip} / {receiving_ip} / {receiving_mx_hostname}" per failure</span>
|
||||
<span class="c1"># detail.</span>
|
||||
<span class="n">_SMTP_TLS_COMBINED_BACKFILL_SCRIPT</span> <span class="o">=</span> <span class="p">(</span>
|
||||
<span class="s2">"List pols = new ArrayList(); "</span>
|
||||
<span class="s2">"List dets = new ArrayList(); "</span>
|
||||
<span class="s2">"def ps = ctx._source.policies; "</span>
|
||||
<span class="s2">"if (ps != null) { "</span>
|
||||
<span class="s2">"if (!(ps instanceof List)) { ps = [ps]; } "</span>
|
||||
<span class="s2">"for (p in ps) { "</span>
|
||||
<span class="s2">"if (p == null) { continue; } "</span>
|
||||
<span class="s1">'def dom = p.policy_domain != null ? p.policy_domain : "none"; '</span>
|
||||
<span class="s1">'def typ = p.policy_type != null ? p.policy_type : "none"; '</span>
|
||||
<span class="s1">'pols.add(dom + " / " + typ); '</span>
|
||||
<span class="s2">"def fds = p.failure_details; "</span>
|
||||
<span class="s2">"if (fds != null) { "</span>
|
||||
<span class="s2">"if (!(fds instanceof List)) { fds = [fds]; } "</span>
|
||||
<span class="s2">"for (f in fds) { "</span>
|
||||
<span class="s2">"if (f == null) { continue; } "</span>
|
||||
<span class="s1">'def rt = f.result_type != null ? f.result_type : "none"; '</span>
|
||||
<span class="s1">'def smi = f.sending_mta_ip != null ? f.sending_mta_ip : "none"; '</span>
|
||||
<span class="s1">'def ri = f.receiving_ip != null ? f.receiving_ip : "none"; '</span>
|
||||
<span class="s1">'def rmh = f.receiving_mx_hostname != null ? f.receiving_mx_hostname : "none"; '</span>
|
||||
<span class="s1">'dets.add(dom + " / " + typ + " / " + rt + " / " + smi + " / " + ri + " / " + rmh); '</span>
|
||||
<span class="s2">"} } } } "</span>
|
||||
<span class="s2">"ctx._source.policies_combined = pols; "</span>
|
||||
<span class="s2">"ctx._source.failure_details_combined = dets;"</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_PolicyOverride</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="c1"># The elasticsearch.dsl 8.x type stubs use dataclass_transform and only</span>
|
||||
@@ -229,6 +404,11 @@
|
||||
<span class="n">header_from</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">envelope_from</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">envelope_to</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="c1"># Nested(...) on the two auth-result fields below is only the DSL's</span>
|
||||
<span class="c1"># in-memory document shape; it is never installed as a mapping.</span>
|
||||
<span class="c1"># create_indexes() deliberately skips Index.document() registration so</span>
|
||||
<span class="c1"># these fields stay dynamic-mapped as plain `object` in the cluster</span>
|
||||
<span class="c1"># (see the comment there and issue #169).</span>
|
||||
<span class="n">dkim_results</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_DKIMResult</span><span class="p">)</span>
|
||||
<span class="n">spf_results</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SPFResult</span><span class="p">)</span>
|
||||
<span class="c1"># One "{selector} / {domain} / {result}" (DKIM) or "{scope} / {domain} /</span>
|
||||
@@ -266,9 +446,7 @@
|
||||
<span class="n">human_result</span><span class="o">=</span><span class="n">human_result</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">dkim_results_combined</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="s2">"</span><span class="si">{0}</span><span class="s2"> / </span><span class="si">{1}</span><span class="s2"> / </span><span class="si">{2}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">selector</span><span class="p">,</span> <span class="n">domain</span><span class="p">,</span> <span class="n">result</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">dkim_results_combined</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">selector</span><span class="si">}</span><span class="s2"> / </span><span class="si">{</span><span class="n">domain</span><span class="si">}</span><span class="s2"> / </span><span class="si">{</span><span class="n">result</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_spf_result</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="p">,</span>
|
||||
@@ -285,9 +463,7 @@
|
||||
<span class="n">human_result</span><span class="o">=</span><span class="n">human_result</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">spf_results_combined</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="s2">"</span><span class="si">{0}</span><span class="s2"> / </span><span class="si">{1}</span><span class="s2"> / </span><span class="si">{2}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">scope</span><span class="p">,</span> <span class="n">domain</span><span class="p">,</span> <span class="n">result</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">spf_results_combined</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">scope</span><span class="si">}</span><span class="s2"> / </span><span class="si">{</span><span class="n">domain</span><span class="si">}</span><span class="s2"> / </span><span class="si">{</span><span class="n">result</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">save</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span> <span class="c1"># pyright: ignore[reportIncompatibleMethodOverride]</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">passed_dmarc</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
@@ -404,6 +580,7 @@
|
||||
<span class="n">result_type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">sending_mta_ip</span> <span class="o">=</span> <span class="n">Ip</span><span class="p">()</span>
|
||||
<span class="n">receiving_mx_helo</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">receiving_mx_hostname</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">receiving_ip</span> <span class="o">=</span> <span class="n">Ip</span><span class="p">()</span>
|
||||
<span class="n">failed_session_count</span> <span class="o">=</span> <span class="n">Integer</span><span class="p">()</span>
|
||||
<span class="n">additional_information_uri</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
@@ -445,7 +622,7 @@
|
||||
<span class="n">receiving_mx_helo</span><span class="o">=</span><span class="n">receiving_mx_helo</span><span class="p">,</span>
|
||||
<span class="n">receiving_ip</span><span class="o">=</span><span class="n">receiving_ip</span><span class="p">,</span>
|
||||
<span class="n">failed_session_count</span><span class="o">=</span><span class="n">failed_session_count</span><span class="p">,</span>
|
||||
<span class="n">additional_information</span><span class="o">=</span><span class="n">additional_information_uri</span><span class="p">,</span>
|
||||
<span class="n">additional_information_uri</span><span class="o">=</span><span class="n">additional_information_uri</span><span class="p">,</span>
|
||||
<span class="n">failure_reason_code</span><span class="o">=</span><span class="n">failure_reason_code</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">failure_details</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_details</span><span class="p">)</span>
|
||||
@@ -468,6 +645,19 @@
|
||||
<span class="n">contact_info</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">report_id</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">policies</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SMTPTLSPolicyDoc</span><span class="p">)</span>
|
||||
<span class="c1"># One "{policy_domain} / {policy_type}" string per policy. Kibana/</span>
|
||||
<span class="c1"># Grafana tables cannot terms-aggregate the subfields of an object</span>
|
||||
<span class="c1"># array without producing a cross-product of values (issue #169), so</span>
|
||||
<span class="c1"># dashboards aggregate these composed keywords instead. Declared to</span>
|
||||
<span class="c1"># match what dynamic mapping produces for a string array (text +</span>
|
||||
<span class="c1"># .keyword).</span>
|
||||
<span class="n">policies_combined</span> <span class="o">=</span> <span class="n">Text</span><span class="p">(</span><span class="n">multi</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">fields</span><span class="o">=</span><span class="p">{</span><span class="s2">"keyword"</span><span class="p">:</span> <span class="n">Keyword</span><span class="p">(</span><span class="n">ignore_above</span><span class="o">=</span><span class="mi">256</span><span class="p">)})</span>
|
||||
<span class="c1"># One "{policy_domain} / {policy_type} / {result_type} /</span>
|
||||
<span class="c1"># {sending_mta_ip} / {receiving_ip} / {receiving_mx_hostname}" string</span>
|
||||
<span class="c1"># per failure detail, across all policies.</span>
|
||||
<span class="n">failure_details_combined</span> <span class="o">=</span> <span class="n">Text</span><span class="p">(</span>
|
||||
<span class="n">multi</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">fields</span><span class="o">=</span><span class="p">{</span><span class="s2">"keyword"</span><span class="p">:</span> <span class="n">Keyword</span><span class="p">(</span><span class="n">ignore_above</span><span class="o">=</span><span class="mi">256</span><span class="p">)}</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="AlreadySaved">
|
||||
@@ -516,9 +706,7 @@
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">hosts</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
|
||||
<span class="n">hosts</span> <span class="o">=</span> <span class="p">[</span><span class="n">hosts</span><span class="p">]</span>
|
||||
<span class="n">scheme</span> <span class="o">=</span> <span class="s2">"https://"</span> <span class="k">if</span> <span class="n">use_ssl</span> <span class="k">else</span> <span class="s2">"http://"</span>
|
||||
<span class="n">normalized_hosts</span> <span class="o">=</span> <span class="p">[</span>
|
||||
<span class="n">host</span> <span class="k">if</span> <span class="s2">"://"</span> <span class="ow">in</span> <span class="n">host</span> <span class="k">else</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">scheme</span><span class="p">,</span> <span class="n">host</span><span class="p">)</span> <span class="k">for</span> <span class="n">host</span> <span class="ow">in</span> <span class="n">hosts</span>
|
||||
<span class="p">]</span>
|
||||
<span class="n">normalized_hosts</span> <span class="o">=</span> <span class="p">[</span><span class="n">host</span> <span class="k">if</span> <span class="s2">"://"</span> <span class="ow">in</span> <span class="n">host</span> <span class="k">else</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">scheme</span><span class="si">}{</span><span class="n">host</span><span class="si">}</span><span class="s2">"</span> <span class="k">for</span> <span class="n">host</span> <span class="ow">in</span> <span class="n">hosts</span><span class="p">]</span>
|
||||
<span class="n">conn_params</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"hosts"</span><span class="p">:</span> <span class="n">normalized_hosts</span><span class="p">,</span> <span class="s2">"request_timeout"</span><span class="p">:</span> <span class="n">timeout</span><span class="p">}</span>
|
||||
<span class="k">if</span> <span class="n">use_ssl</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">ssl_cert_path</span><span class="p">:</span>
|
||||
@@ -561,18 +749,25 @@
|
||||
<span class="k">for</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">names</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="n">Index</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="c1"># Deliberately no Index.document() registration: Kibana/OpenSearch</span>
|
||||
<span class="c1"># Dashboards/Grafana cannot terms-aggregate fields inside a</span>
|
||||
<span class="c1"># `nested` mapping, so the dynamic `object` mapping produced by a</span>
|
||||
<span class="c1"># bare create is load-bearing for the shipped dashboards (issue</span>
|
||||
<span class="c1"># #169; see the *_combined fields on _AggregateReportDoc).</span>
|
||||
<span class="c1"># Deliberately no Index.document() registration: the shipped</span>
|
||||
<span class="c1"># dashboards cannot rebuild their detail tables on a `nested`</span>
|
||||
<span class="c1"># mapping — Kibana/OSD visual editors do not support nested</span>
|
||||
<span class="c1"># fields, Vega can run nested aggregations but does not render</span>
|
||||
<span class="c1"># tables, and Grafana's nested bucket aggregation (9.4+) lacks</span>
|
||||
<span class="c1"># reverse_nested for parent-level metrics like message_count —</span>
|
||||
<span class="c1"># so the dynamic `object` mapping produced by a bare create is</span>
|
||||
<span class="c1"># load-bearing for the shipped dashboards. _AggregateReportDoc</span>
|
||||
<span class="c1"># still declares dkim_results/spf_results with Nested(...), but</span>
|
||||
<span class="c1"># that is only the DSL's in-memory shape for building documents</span>
|
||||
<span class="c1"># — it is never installed as a mapping. See issue #169 and the</span>
|
||||
<span class="c1"># *_combined fields on _AggregateReportDoc.</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">index</span><span class="o">.</span><span class="n">exists</span><span class="p">():</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Creating Elasticsearch index: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">name</span><span class="p">))</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Creating Elasticsearch index: </span><span class="si">{</span><span class="n">name</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">effective_settings</span><span class="p">:</span>
|
||||
<span class="n">index</span><span class="o">.</span><span class="n">settings</span><span class="p">(</span><span class="o">**</span><span class="n">effective_settings</span><span class="p">)</span>
|
||||
<span class="n">index</span><span class="o">.</span><span class="n">create</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="s2">"Elasticsearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Elasticsearch error: </span><span class="si">{</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
|
||||
@@ -581,25 +776,118 @@
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">migrate_indexes</span><span class="p">(</span>
|
||||
<span class="n">aggregate_indexes</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">failure_indexes</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">smtp_tls_indexes</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Updates index mappings</span>
|
||||
<span class="sd"> Backfills the ``dkim_results_combined``/``spf_results_combined`` fields</span>
|
||||
<span class="sd"> (added for issue #169) on aggregate report documents, and the</span>
|
||||
<span class="sd"> ``policies_combined``/``failure_details_combined`` fields on SMTP TLS</span>
|
||||
<span class="sd"> report documents, that were saved before those fields existed.</span>
|
||||
|
||||
<span class="sd"> This is a no-op kept for API compatibility (``cli.py`` calls it on</span>
|
||||
<span class="sd"> startup). The only migration this function ever performed was</span>
|
||||
<span class="sd"> re-typing ``published_policy.fo`` from ``long`` to ``text``, which</span>
|
||||
<span class="sd"> applied exclusively to indices still carrying the legacy</span>
|
||||
<span class="sd"> Elasticsearch 6-era ``"doc"`` mapping type. The 8.x client can only</span>
|
||||
<span class="sd"> reach servers (Elasticsearch 8.x/9.x) whose indices were created on</span>
|
||||
<span class="sd"> Elasticsearch 7.x or later and are therefore typeless, so that</span>
|
||||
<span class="sd"> migration path is unreachable and has been removed.</span>
|
||||
<span class="sd"> For each name in ``aggregate_indexes``/``smtp_tls_indexes``, this</span>
|
||||
<span class="sd"> submits an ``update_by_query`` against the ``f"{name}*"`` index pattern</span>
|
||||
<span class="sd"> (the real indexes are date-suffixed) as a non-blocking background task</span>
|
||||
<span class="sd"> (``wait_for_completion=False``), so it never delays parsedmarc startup.</span>
|
||||
<span class="sd"> Submission is guarded by a cheap ``count`` query that only matches</span>
|
||||
<span class="sd"> documents with DKIM/SPF results (or policies/failure details) but no</span>
|
||||
<span class="sd"> combined field, so once an index is fully backfilled, later calls are a</span>
|
||||
<span class="sd"> fast no-op. Any error talking to the cluster (e.g. no indexes yet on a</span>
|
||||
<span class="sd"> fresh install, or a transient connection issue) is caught and logged as</span>
|
||||
<span class="sd"> a warning rather than raised; the backfill is simply retried on the</span>
|
||||
<span class="sd"> next startup, and the manual ``_update_by_query`` commands documented in</span>
|
||||
<span class="sd"> ``docs/source/elasticsearch.md`` remain available in the meantime.</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> aggregate_indexes (list): A list of aggregate index names</span>
|
||||
<span class="sd"> (accepted for API compatibility; unused)</span>
|
||||
<span class="sd"> failure_indexes (list): A list of failure index names</span>
|
||||
<span class="sd"> (accepted for API compatibility; unused)</span>
|
||||
<span class="sd"> """</span></div>
|
||||
<span class="sd"> smtp_tls_indexes (list): A list of SMTP TLS index names</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">aggregate_indexes</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">smtp_tls_indexes</span><span class="p">:</span>
|
||||
<span class="k">return</span>
|
||||
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">client</span> <span class="o">=</span> <span class="n">connections</span><span class="o">.</span><span class="n">get_connection</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
||||
<span class="s2">"Skipping the dkim_results_combined/spf_results_combined/"</span>
|
||||
<span class="s2">"policies_combined/failure_details_combined backfill: could "</span>
|
||||
<span class="sa">f</span><span class="s2">"not get an Elasticsearch connection: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">. This will be "</span>
|
||||
<span class="s2">"retried at the next startup."</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">return</span>
|
||||
<span class="k">for</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">aggregate_indexes</span> <span class="ow">or</span> <span class="p">[]:</span>
|
||||
<span class="n">pattern</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">name</span><span class="si">}</span><span class="s2">*"</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">count_response</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">count</span><span class="p">(</span>
|
||||
<span class="n">index</span><span class="o">=</span><span class="n">pattern</span><span class="p">,</span>
|
||||
<span class="n">query</span><span class="o">=</span><span class="n">_COMBINED_BACKFILL_QUERY</span><span class="p">,</span>
|
||||
<span class="n">ignore_unavailable</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="n">allow_no_indices</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">count</span> <span class="o">=</span> <span class="n">count_response</span><span class="p">[</span><span class="s2">"count"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">count</span><span class="p">:</span>
|
||||
<span class="k">continue</span>
|
||||
<span class="n">update_response</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">update_by_query</span><span class="p">(</span>
|
||||
<span class="n">index</span><span class="o">=</span><span class="n">pattern</span><span class="p">,</span>
|
||||
<span class="n">query</span><span class="o">=</span><span class="n">_COMBINED_BACKFILL_QUERY</span><span class="p">,</span>
|
||||
<span class="n">script</span><span class="o">=</span><span class="p">{</span><span class="s2">"source"</span><span class="p">:</span> <span class="n">_COMBINED_BACKFILL_SCRIPT</span><span class="p">,</span> <span class="s2">"lang"</span><span class="p">:</span> <span class="s2">"painless"</span><span class="p">},</span>
|
||||
<span class="n">conflicts</span><span class="o">=</span><span class="s2">"proceed"</span><span class="p">,</span>
|
||||
<span class="n">wait_for_completion</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">ignore_unavailable</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="n">allow_no_indices</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">task_id</span> <span class="o">=</span> <span class="n">update_response</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"task"</span><span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
|
||||
<span class="s2">"Backfilling dkim_results_combined/spf_results_combined on "</span>
|
||||
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">count</span><span class="si">}</span><span class="s2"> existing documents in </span><span class="si">{</span><span class="n">pattern</span><span class="si">}</span><span class="s2"> (task </span><span class="si">{</span><span class="n">task_id</span><span class="si">}</span><span class="s2">)"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
||||
<span class="s2">"Failed to check/submit the dkim_results_combined/"</span>
|
||||
<span class="sa">f</span><span class="s2">"spf_results_combined backfill for </span><span class="si">{</span><span class="n">pattern</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">. This "</span>
|
||||
<span class="s2">"will be retried at the next startup; the manual "</span>
|
||||
<span class="s2">"_update_by_query command in the documentation remains "</span>
|
||||
<span class="s2">"available in the meantime."</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">smtp_tls_indexes</span> <span class="ow">or</span> <span class="p">[]:</span>
|
||||
<span class="n">pattern</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">name</span><span class="si">}</span><span class="s2">*"</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">count_response</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">count</span><span class="p">(</span>
|
||||
<span class="n">index</span><span class="o">=</span><span class="n">pattern</span><span class="p">,</span>
|
||||
<span class="n">query</span><span class="o">=</span><span class="n">_SMTP_TLS_COMBINED_BACKFILL_QUERY</span><span class="p">,</span>
|
||||
<span class="n">ignore_unavailable</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="n">allow_no_indices</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">count</span> <span class="o">=</span> <span class="n">count_response</span><span class="p">[</span><span class="s2">"count"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">count</span><span class="p">:</span>
|
||||
<span class="k">continue</span>
|
||||
<span class="n">update_response</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">update_by_query</span><span class="p">(</span>
|
||||
<span class="n">index</span><span class="o">=</span><span class="n">pattern</span><span class="p">,</span>
|
||||
<span class="n">query</span><span class="o">=</span><span class="n">_SMTP_TLS_COMBINED_BACKFILL_QUERY</span><span class="p">,</span>
|
||||
<span class="n">script</span><span class="o">=</span><span class="p">{</span>
|
||||
<span class="s2">"source"</span><span class="p">:</span> <span class="n">_SMTP_TLS_COMBINED_BACKFILL_SCRIPT</span><span class="p">,</span>
|
||||
<span class="s2">"lang"</span><span class="p">:</span> <span class="s2">"painless"</span><span class="p">,</span>
|
||||
<span class="p">},</span>
|
||||
<span class="n">conflicts</span><span class="o">=</span><span class="s2">"proceed"</span><span class="p">,</span>
|
||||
<span class="n">wait_for_completion</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">ignore_unavailable</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="n">allow_no_indices</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">task_id</span> <span class="o">=</span> <span class="n">update_response</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"task"</span><span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
|
||||
<span class="s2">"Backfilling policies_combined/failure_details_combined on "</span>
|
||||
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">count</span><span class="si">}</span><span class="s2"> existing documents in </span><span class="si">{</span><span class="n">pattern</span><span class="si">}</span><span class="s2"> (task </span><span class="si">{</span><span class="n">task_id</span><span class="si">}</span><span class="s2">)"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
||||
<span class="s2">"Failed to check/submit the policies_combined/"</span>
|
||||
<span class="sa">f</span><span class="s2">"failure_details_combined backfill for </span><span class="si">{</span><span class="n">pattern</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">. "</span>
|
||||
<span class="s2">"This will be retried at the next startup; the manual "</span>
|
||||
<span class="s2">"_update_by_query command in the documentation remains "</span>
|
||||
<span class="s2">"available in the meantime."</span>
|
||||
<span class="p">)</span></div>
|
||||
|
||||
|
||||
|
||||
@@ -648,11 +936,11 @@
|
||||
<span class="n">end_date_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="nb">range</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">date_end</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">lte</span><span class="o">=</span><span class="n">end_date</span><span class="p">))))</span> <span class="c1"># pyright: ignore[reportArgumentType]</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">index_suffix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"dmarc_aggregate_</span><span class="si">{0}</span><span class="s2">*"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"dmarc_aggregate_</span><span class="si">{</span><span class="n">index_suffix</span><span class="si">}</span><span class="s2">*"</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"dmarc_aggregate*"</span>
|
||||
<span class="k">if</span> <span class="n">index_prefix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">search_index</span><span class="p">)</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">index_prefix</span><span class="si">}{</span><span class="n">search_index</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="n">search_index</span><span class="p">)</span>
|
||||
<span class="n">query</span> <span class="o">=</span> <span class="n">org_name_query</span> <span class="o">&</span> <span class="n">report_id_query</span> <span class="o">&</span> <span class="n">domain_query</span>
|
||||
<span class="n">query</span> <span class="o">=</span> <span class="n">query</span> <span class="o">&</span> <span class="n">begin_date_query</span> <span class="o">&</span> <span class="n">end_date_query</span>
|
||||
@@ -667,18 +955,15 @@
|
||||
<span class="n">existing</span> <span class="o">=</span> <span class="n">search</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error_</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span>
|
||||
<span class="s2">"Elasticsearch's search for existing report </span><span class="se">\</span>
|
||||
<span class="s2"> error: </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error_</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="sa">f</span><span class="s2">"Elasticsearch's search for existing report error: </span><span class="si">{</span><span class="n">error_</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">existing</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">AlreadySaved</span><span class="p">(</span>
|
||||
<span class="s2">"An aggregate report ID </span><span class="si">{0}</span><span class="s2"> from </span><span class="si">{1}</span><span class="s2"> about </span><span class="si">{2}</span><span class="s2"> "</span>
|
||||
<span class="s2">"with a date range of </span><span class="si">{3}</span><span class="s2"> UTC to </span><span class="si">{4}</span><span class="s2"> UTC already "</span>
|
||||
<span class="sa">f</span><span class="s2">"An aggregate report ID </span><span class="si">{</span><span class="n">report_id</span><span class="si">}</span><span class="s2"> from </span><span class="si">{</span><span class="n">org_name</span><span class="si">}</span><span class="s2"> about </span><span class="si">{</span><span class="n">domain</span><span class="si">}</span><span class="s2"> "</span>
|
||||
<span class="sa">f</span><span class="s2">"with a date range of </span><span class="si">{</span><span class="n">begin_date_human</span><span class="si">}</span><span class="s2"> UTC to </span><span class="si">{</span><span class="n">end_date_human</span><span class="si">}</span><span class="s2"> UTC already "</span>
|
||||
<span class="s2">"exists in "</span>
|
||||
<span class="s2">"Elasticsearch"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">report_id</span><span class="p">,</span> <span class="n">org_name</span><span class="p">,</span> <span class="n">domain</span><span class="p">,</span> <span class="n">begin_date_human</span><span class="p">,</span> <span class="n">end_date_human</span>
|
||||
<span class="p">)</span>
|
||||
<span class="s2">"Elasticsearch"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">published_policy</span> <span class="o">=</span> <span class="n">_PublishedPolicy</span><span class="p">(</span>
|
||||
<span class="n">domain</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"domain"</span><span class="p">],</span>
|
||||
@@ -771,11 +1056,11 @@
|
||||
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"dmarc_aggregate"</span>
|
||||
<span class="k">if</span> <span class="n">index_suffix</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">_</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">index</span><span class="si">}</span><span class="s2">_</span><span class="si">{</span><span class="n">index_suffix</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="k">if</span> <span class="n">index_prefix</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">index</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">index_prefix</span><span class="si">}{</span><span class="n">index</span><span class="si">}</span><span class="s2">"</span>
|
||||
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">-</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_date</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">index</span><span class="si">}</span><span class="s2">-</span><span class="si">{</span><span class="n">index_date</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span> <span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span>
|
||||
<span class="p">)</span>
|
||||
@@ -785,7 +1070,7 @@
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">agg_doc</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="s2">"Elasticsearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Elasticsearch error: </span><span class="si">{</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
|
||||
@@ -836,12 +1121,12 @@
|
||||
<span class="n">arrival_date_epoch_milliseconds</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">arrival_date</span><span class="o">.</span><span class="n">timestamp</span><span class="p">()</span> <span class="o">*</span> <span class="mi">1000</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">index_suffix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"dmarc_failure_</span><span class="si">{0}</span><span class="s2">*,dmarc_forensic_</span><span class="si">{0}</span><span class="s2">*"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"dmarc_failure_</span><span class="si">{</span><span class="n">index_suffix</span><span class="si">}</span><span class="s2">*,dmarc_forensic_</span><span class="si">{</span><span class="n">index_suffix</span><span class="si">}</span><span class="s2">*"</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"dmarc_failure*,dmarc_forensic*"</span>
|
||||
<span class="k">if</span> <span class="n">index_prefix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">","</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
|
||||
<span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">part</span><span class="p">)</span> <span class="k">for</span> <span class="n">part</span> <span class="ow">in</span> <span class="n">search_index</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">","</span><span class="p">)</span>
|
||||
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">index_prefix</span><span class="si">}{</span><span class="n">part</span><span class="si">}</span><span class="s2">"</span> <span class="k">for</span> <span class="n">part</span> <span class="ow">in</span> <span class="n">search_index</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">","</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="n">search_index</span><span class="p">)</span>
|
||||
<span class="n">q</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">arrival_date</span><span class="o">=</span><span class="n">arrival_date_epoch_milliseconds</span><span class="p">)))</span> <span class="c1"># pyright: ignore[reportArgumentType]</span>
|
||||
@@ -893,8 +1178,8 @@
|
||||
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">existing</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">AlreadySaved</span><span class="p">(</span>
|
||||
<span class="s2">"A failure sample to </span><span class="si">{0}</span><span class="s2"> from </span><span class="si">{1}</span><span class="s2"> "</span>
|
||||
<span class="s2">"with a subject of </span><span class="si">{2}</span><span class="s2"> and arrival date of </span><span class="si">{3}</span><span class="s2"> "</span>
|
||||
<span class="s2">"A failure sample to </span><span class="si">{}</span><span class="s2"> from </span><span class="si">{}</span><span class="s2"> "</span>
|
||||
<span class="s2">"with a subject of </span><span class="si">{}</span><span class="s2"> and arrival date of </span><span class="si">{}</span><span class="s2"> "</span>
|
||||
<span class="s2">"already exists in "</span>
|
||||
<span class="s2">"Elasticsearch"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">to_</span><span class="p">,</span> <span class="n">from_</span><span class="p">,</span> <span class="n">subject</span><span class="p">,</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"arrival_date_utc"</span><span class="p">]</span>
|
||||
@@ -955,14 +1240,14 @@
|
||||
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"dmarc_failure"</span>
|
||||
<span class="k">if</span> <span class="n">index_suffix</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">_</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">index</span><span class="si">}</span><span class="s2">_</span><span class="si">{</span><span class="n">index_suffix</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="k">if</span> <span class="n">index_prefix</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">index</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">index_prefix</span><span class="si">}{</span><span class="n">index</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="k">if</span> <span class="n">monthly_indexes</span><span class="p">:</span>
|
||||
<span class="n">index_date</span> <span class="o">=</span> <span class="n">arrival_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m"</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">index_date</span> <span class="o">=</span> <span class="n">arrival_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">-</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_date</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">index</span><span class="si">}</span><span class="s2">-</span><span class="si">{</span><span class="n">index_date</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span> <span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span>
|
||||
<span class="p">)</span>
|
||||
@@ -971,10 +1256,10 @@
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">failure_doc</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="s2">"Elasticsearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Elasticsearch error: </span><span class="si">{</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="k">except</span> <span class="ne">KeyError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidFailureReport</span><span class="p">(</span>
|
||||
<span class="s2">"Failure report missing required field: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="sa">f</span><span class="s2">"Failure report missing required field: </span><span class="si">{</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="p">)</span></div>
|
||||
|
||||
|
||||
@@ -1024,11 +1309,11 @@
|
||||
<span class="n">end_date_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">date_end</span><span class="o">=</span><span class="n">end_date</span><span class="p">)))</span> <span class="c1"># pyright: ignore[reportArgumentType]</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">index_suffix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"smtp_tls_</span><span class="si">{0}</span><span class="s2">*"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"smtp_tls_</span><span class="si">{</span><span class="n">index_suffix</span><span class="si">}</span><span class="s2">*"</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"smtp_tls*"</span>
|
||||
<span class="k">if</span> <span class="n">index_prefix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">search_index</span><span class="p">)</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">index_prefix</span><span class="si">}{</span><span class="n">search_index</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="n">search_index</span><span class="p">)</span>
|
||||
<span class="n">query</span> <span class="o">=</span> <span class="n">org_name_query</span> <span class="o">&</span> <span class="n">report_id_query</span>
|
||||
<span class="n">query</span> <span class="o">=</span> <span class="n">query</span> <span class="o">&</span> <span class="n">begin_date_query</span> <span class="o">&</span> <span class="n">end_date_query</span>
|
||||
@@ -1038,8 +1323,7 @@
|
||||
<span class="n">existing</span> <span class="o">=</span> <span class="n">search</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error_</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span>
|
||||
<span class="s2">"Elasticsearch's search for existing report </span><span class="se">\</span>
|
||||
<span class="s2"> error: </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error_</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="sa">f</span><span class="s2">"Elasticsearch's search for existing report error: </span><span class="si">{</span><span class="n">error_</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">existing</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||
@@ -1053,10 +1337,10 @@
|
||||
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"smtp_tls"</span>
|
||||
<span class="k">if</span> <span class="n">index_suffix</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">_</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">index</span><span class="si">}</span><span class="s2">_</span><span class="si">{</span><span class="n">index_suffix</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="k">if</span> <span class="n">index_prefix</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">index</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">-</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_date</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">index_prefix</span><span class="si">}{</span><span class="n">index</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">index</span><span class="si">}</span><span class="s2">-</span><span class="si">{</span><span class="n">index_date</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span> <span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span>
|
||||
<span class="p">)</span>
|
||||
@@ -1077,6 +1361,16 @@
|
||||
<span class="n">policy_strings</span> <span class="o">=</span> <span class="n">policy</span><span class="p">[</span><span class="s2">"policy_strings"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="s2">"mx_host_patterns"</span> <span class="ow">in</span> <span class="n">policy</span><span class="p">:</span>
|
||||
<span class="n">mx_host_patterns</span> <span class="o">=</span> <span class="n">policy</span><span class="p">[</span><span class="s2">"mx_host_patterns"</span><span class="p">]</span>
|
||||
<span class="c1"># policies_combined/failure_details_combined: see the field</span>
|
||||
<span class="c1"># declarations on _SMTPTLSReportDoc and issue #169. policies and</span>
|
||||
<span class="c1"># their failure_details are object arrays with the same</span>
|
||||
<span class="c1"># cross-product problem as dkim_results/spf_results, so dashboards</span>
|
||||
<span class="c1"># aggregate these composed strings instead of the raw subfields.</span>
|
||||
<span class="n">policy_domain_combined</span> <span class="o">=</span> <span class="n">policy</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"policy_domain"</span><span class="p">)</span> <span class="ow">or</span> <span class="s2">"none"</span>
|
||||
<span class="n">policy_type_combined</span> <span class="o">=</span> <span class="n">policy</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"policy_type"</span><span class="p">)</span> <span class="ow">or</span> <span class="s2">"none"</span>
|
||||
<span class="n">smtp_tls_doc</span><span class="o">.</span><span class="n">policies_combined</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">policy_domain_combined</span><span class="si">}</span><span class="s2"> / </span><span class="si">{</span><span class="n">policy_type_combined</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">policy_doc</span> <span class="o">=</span> <span class="n">_SMTPTLSPolicyDoc</span><span class="p">(</span>
|
||||
<span class="n">policy_domain</span><span class="o">=</span><span class="n">policy</span><span class="p">[</span><span class="s2">"policy_domain"</span><span class="p">],</span>
|
||||
<span class="n">policy_type</span><span class="o">=</span><span class="n">policy</span><span class="p">[</span><span class="s2">"policy_type"</span><span class="p">],</span>
|
||||
@@ -1097,7 +1391,12 @@
|
||||
|
||||
<span class="k">if</span> <span class="s2">"receiving_mx_hostname"</span> <span class="ow">in</span> <span class="n">failure_detail</span><span class="p">:</span>
|
||||
<span class="n">receiving_mx_hostname</span> <span class="o">=</span> <span class="n">failure_detail</span><span class="p">[</span><span class="s2">"receiving_mx_hostname"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="s2">"additional_information_uri"</span> <span class="ow">in</span> <span class="n">failure_detail</span><span class="p">:</span>
|
||||
<span class="c1"># The parser's key is additional_info_uri (see</span>
|
||||
<span class="c1"># SMTPTLSFailureDetailsOptional in types.py); accept the</span>
|
||||
<span class="c1"># long-form key too for dicts built by other callers.</span>
|
||||
<span class="k">if</span> <span class="s2">"additional_info_uri"</span> <span class="ow">in</span> <span class="n">failure_detail</span><span class="p">:</span>
|
||||
<span class="n">additional_information_uri</span> <span class="o">=</span> <span class="n">failure_detail</span><span class="p">[</span><span class="s2">"additional_info_uri"</span><span class="p">]</span>
|
||||
<span class="k">elif</span> <span class="s2">"additional_information_uri"</span> <span class="ow">in</span> <span class="n">failure_detail</span><span class="p">:</span>
|
||||
<span class="n">additional_information_uri</span> <span class="o">=</span> <span class="n">failure_detail</span><span class="p">[</span>
|
||||
<span class="s2">"additional_information_uri"</span>
|
||||
<span class="p">]</span>
|
||||
@@ -1122,6 +1421,16 @@
|
||||
<span class="n">additional_information_uri</span><span class="o">=</span><span class="n">additional_information_uri</span><span class="p">,</span>
|
||||
<span class="n">failure_reason_code</span><span class="o">=</span><span class="n">failure_reason_code</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">smtp_tls_doc</span><span class="o">.</span><span class="n">failure_details_combined</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="s2">"</span><span class="si">{}</span><span class="s2"> / </span><span class="si">{}</span><span class="s2"> / </span><span class="si">{}</span><span class="s2"> / </span><span class="si">{}</span><span class="s2"> / </span><span class="si">{}</span><span class="s2"> / </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">policy_domain_combined</span><span class="p">,</span>
|
||||
<span class="n">policy_type_combined</span><span class="p">,</span>
|
||||
<span class="n">failure_detail</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"result_type"</span><span class="p">)</span> <span class="ow">or</span> <span class="s2">"none"</span><span class="p">,</span>
|
||||
<span class="n">sending_mta_ip</span> <span class="ow">or</span> <span class="s2">"none"</span><span class="p">,</span>
|
||||
<span class="n">receiving_ip</span> <span class="ow">or</span> <span class="s2">"none"</span><span class="p">,</span>
|
||||
<span class="n">receiving_mx_hostname</span> <span class="ow">or</span> <span class="s2">"none"</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">smtp_tls_doc</span><span class="o">.</span><span class="n">policies</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">policy_doc</span><span class="p">)</span>
|
||||
|
||||
<span class="n">create_indexes</span><span class="p">([</span><span class="n">index</span><span class="p">],</span> <span class="n">index_settings</span><span class="p">)</span>
|
||||
@@ -1130,7 +1439,7 @@
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">smtp_tls_doc</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="s2">"Elasticsearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Elasticsearch error: </span><span class="si">{</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -5,14 +5,14 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc.opensearch — parsedmarc 10.2.4 documentation</title>
|
||||
<title>parsedmarc.opensearch — parsedmarc 10.3.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=9edc463e" />
|
||||
|
||||
|
||||
<script src="../../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../../_static/documentation_options.js?v=2c8206f1"></script>
|
||||
<script src="../../_static/documentation_options.js?v=a126560f"></script>
|
||||
<script src="../../_static/doctools.js?v=fd6eb6e6"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=6ffebe34"></script>
|
||||
<script src="../../_static/js/theme.js"></script>
|
||||
@@ -119,6 +119,182 @@
|
||||
|
||||
|
||||
|
||||
<span class="c1"># Guard query for the dkim_results_combined/spf_results_combined backfill</span>
|
||||
<span class="c1"># (see ``migrate_indexes``). Matches only documents that have at least one</span>
|
||||
<span class="c1"># DKIM or SPF auth result and are missing the corresponding combined field.</span>
|
||||
<span class="c1"># Empty arrays are invisible to ``exists``, so documents with zero</span>
|
||||
<span class="c1"># DKIM/SPF results are correctly skipped (verified against real data;</span>
|
||||
<span class="c1"># this also makes the query idempotent — a backfilled document no longer</span>
|
||||
<span class="c1"># matches). Each result is matched on an OR of its ``domain``/``result``</span>
|
||||
<span class="c1"># subfields as defense in depth: the parsers we audited never store a</span>
|
||||
<span class="c1"># result without both, but an empty string indexes no text tokens and is</span>
|
||||
<span class="c1"># invisible to ``exists``, and the storage shape of every historical</span>
|
||||
<span class="c1"># parsedmarc version can't be audited — matching either subfield costs</span>
|
||||
<span class="c1"># nothing and cannot skip a document that has something to backfill.</span>
|
||||
<span class="n">_COMBINED_BACKFILL_QUERY</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s2">"bool"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"minimum_should_match"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
|
||||
<span class="s2">"should"</span><span class="p">:</span> <span class="p">[</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s2">"bool"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"must"</span><span class="p">:</span> <span class="p">[</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s2">"bool"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"minimum_should_match"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
|
||||
<span class="s2">"should"</span><span class="p">:</span> <span class="p">[</span>
|
||||
<span class="p">{</span><span class="s2">"exists"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"field"</span><span class="p">:</span> <span class="s2">"dkim_results.domain"</span><span class="p">}},</span>
|
||||
<span class="p">{</span><span class="s2">"exists"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"field"</span><span class="p">:</span> <span class="s2">"dkim_results.result"</span><span class="p">}},</span>
|
||||
<span class="p">],</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">],</span>
|
||||
<span class="s2">"must_not"</span><span class="p">:</span> <span class="p">[{</span><span class="s2">"exists"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"field"</span><span class="p">:</span> <span class="s2">"dkim_results_combined"</span><span class="p">}}],</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">},</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s2">"bool"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"must"</span><span class="p">:</span> <span class="p">[</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s2">"bool"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"minimum_should_match"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
|
||||
<span class="s2">"should"</span><span class="p">:</span> <span class="p">[</span>
|
||||
<span class="p">{</span><span class="s2">"exists"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"field"</span><span class="p">:</span> <span class="s2">"spf_results.domain"</span><span class="p">}},</span>
|
||||
<span class="p">{</span><span class="s2">"exists"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"field"</span><span class="p">:</span> <span class="s2">"spf_results.result"</span><span class="p">}},</span>
|
||||
<span class="p">],</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">],</span>
|
||||
<span class="s2">"must_not"</span><span class="p">:</span> <span class="p">[{</span><span class="s2">"exists"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"field"</span><span class="p">:</span> <span class="s2">"spf_results_combined"</span><span class="p">}}],</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">},</span>
|
||||
<span class="p">],</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">}</span>
|
||||
|
||||
<span class="c1"># Painless script that (re)derives dkim_results_combined/spf_results_combined</span>
|
||||
<span class="c1"># from dkim_results/spf_results, matching the format written by</span>
|
||||
<span class="c1"># save_aggregate_report_to_opensearch(): "{selector} / {domain} / {result}"</span>
|
||||
<span class="c1"># per DKIM result and "{scope} / {domain} / {result}" per SPF result.</span>
|
||||
<span class="n">_COMBINED_BACKFILL_SCRIPT</span> <span class="o">=</span> <span class="p">(</span>
|
||||
<span class="s2">"List dk = new ArrayList(); "</span>
|
||||
<span class="s2">"def dr = ctx._source.dkim_results; "</span>
|
||||
<span class="s2">"if (dr != null) { "</span>
|
||||
<span class="s2">"if (!(dr instanceof List)) { dr = [dr]; } "</span>
|
||||
<span class="s2">"for (e in dr) { "</span>
|
||||
<span class="s2">"if (e == null) { continue; } "</span>
|
||||
<span class="s1">'def sel = e.selector != null ? e.selector : "none"; '</span>
|
||||
<span class="s1">'def dom = e.domain != null ? e.domain : "none"; '</span>
|
||||
<span class="s1">'def res = e.result != null ? e.result : "none"; '</span>
|
||||
<span class="s1">'dk.add(sel + " / " + dom + " / " + res); '</span>
|
||||
<span class="s2">"} } "</span>
|
||||
<span class="s2">"ctx._source.dkim_results_combined = dk; "</span>
|
||||
<span class="s2">"List sp = new ArrayList(); "</span>
|
||||
<span class="s2">"def sr = ctx._source.spf_results; "</span>
|
||||
<span class="s2">"if (sr != null) { "</span>
|
||||
<span class="s2">"if (!(sr instanceof List)) { sr = [sr]; } "</span>
|
||||
<span class="s2">"for (e in sr) { "</span>
|
||||
<span class="s2">"if (e == null) { continue; } "</span>
|
||||
<span class="s1">'def sc = e.scope != null ? e.scope : "mfrom"; '</span>
|
||||
<span class="s1">'def dom = e.domain != null ? e.domain : "none"; '</span>
|
||||
<span class="s1">'def res = e.result != null ? e.result : (e.results != null ? e.results : "none"); '</span>
|
||||
<span class="s1">'sp.add(sc + " / " + dom + " / " + res); '</span>
|
||||
<span class="s2">"} } "</span>
|
||||
<span class="s2">"ctx._source.spf_results_combined = sp;"</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="c1"># Guard query for the policies_combined/failure_details_combined backfill</span>
|
||||
<span class="c1"># (see ``migrate_indexes``). Matches only SMTP TLS documents that have at</span>
|
||||
<span class="c1"># least one policy or failure detail and are missing the corresponding</span>
|
||||
<span class="c1"># combined field. Empty arrays are invisible to ``exists``, so documents</span>
|
||||
<span class="c1"># with zero policies/failure details are correctly skipped (this also</span>
|
||||
<span class="c1"># makes the query idempotent — a backfilled document no longer matches).</span>
|
||||
<span class="c1"># Each result is matched on an OR of its relevant subfields as defense in</span>
|
||||
<span class="c1"># depth: the parsers we audited never store a policy/failure detail</span>
|
||||
<span class="c1"># without these fields, but an empty string indexes no text tokens and is</span>
|
||||
<span class="c1"># invisible to ``exists``, and the storage shape of every historical</span>
|
||||
<span class="c1"># parsedmarc version can't be audited — matching either subfield costs</span>
|
||||
<span class="c1"># nothing and cannot skip a document that has something to backfill.</span>
|
||||
<span class="n">_SMTP_TLS_COMBINED_BACKFILL_QUERY</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s2">"bool"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"minimum_should_match"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
|
||||
<span class="s2">"should"</span><span class="p">:</span> <span class="p">[</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s2">"bool"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"must"</span><span class="p">:</span> <span class="p">[</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s2">"bool"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"minimum_should_match"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
|
||||
<span class="s2">"should"</span><span class="p">:</span> <span class="p">[</span>
|
||||
<span class="p">{</span><span class="s2">"exists"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"field"</span><span class="p">:</span> <span class="s2">"policies.policy_domain"</span><span class="p">}},</span>
|
||||
<span class="p">{</span><span class="s2">"exists"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"field"</span><span class="p">:</span> <span class="s2">"policies.policy_type"</span><span class="p">}},</span>
|
||||
<span class="p">],</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">],</span>
|
||||
<span class="s2">"must_not"</span><span class="p">:</span> <span class="p">[{</span><span class="s2">"exists"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"field"</span><span class="p">:</span> <span class="s2">"policies_combined"</span><span class="p">}}],</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">},</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s2">"bool"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"must"</span><span class="p">:</span> <span class="p">[</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s2">"bool"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"minimum_should_match"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
|
||||
<span class="s2">"should"</span><span class="p">:</span> <span class="p">[</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s2">"exists"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"field"</span><span class="p">:</span> <span class="s2">"policies.failure_details.result_type"</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">},</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s2">"exists"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"field"</span><span class="p">:</span> <span class="s2">"policies.failure_details.sending_mta_ip"</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">},</span>
|
||||
<span class="p">],</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">],</span>
|
||||
<span class="s2">"must_not"</span><span class="p">:</span> <span class="p">[{</span><span class="s2">"exists"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"field"</span><span class="p">:</span> <span class="s2">"failure_details_combined"</span><span class="p">}}],</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">},</span>
|
||||
<span class="p">],</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">}</span>
|
||||
|
||||
<span class="c1"># Painless script that (re)derives policies_combined/failure_details_combined</span>
|
||||
<span class="c1"># from policies/policies.failure_details, matching the format written by</span>
|
||||
<span class="c1"># save_smtp_tls_report_to_opensearch(): "{policy_domain} / {policy_type}"</span>
|
||||
<span class="c1"># per policy and "{policy_domain} / {policy_type} / {result_type} /</span>
|
||||
<span class="c1"># {sending_mta_ip} / {receiving_ip} / {receiving_mx_hostname}" per failure</span>
|
||||
<span class="c1"># detail.</span>
|
||||
<span class="n">_SMTP_TLS_COMBINED_BACKFILL_SCRIPT</span> <span class="o">=</span> <span class="p">(</span>
|
||||
<span class="s2">"List pols = new ArrayList(); "</span>
|
||||
<span class="s2">"List dets = new ArrayList(); "</span>
|
||||
<span class="s2">"def ps = ctx._source.policies; "</span>
|
||||
<span class="s2">"if (ps != null) { "</span>
|
||||
<span class="s2">"if (!(ps instanceof List)) { ps = [ps]; } "</span>
|
||||
<span class="s2">"for (p in ps) { "</span>
|
||||
<span class="s2">"if (p == null) { continue; } "</span>
|
||||
<span class="s1">'def dom = p.policy_domain != null ? p.policy_domain : "none"; '</span>
|
||||
<span class="s1">'def typ = p.policy_type != null ? p.policy_type : "none"; '</span>
|
||||
<span class="s1">'pols.add(dom + " / " + typ); '</span>
|
||||
<span class="s2">"def fds = p.failure_details; "</span>
|
||||
<span class="s2">"if (fds != null) { "</span>
|
||||
<span class="s2">"if (!(fds instanceof List)) { fds = [fds]; } "</span>
|
||||
<span class="s2">"for (f in fds) { "</span>
|
||||
<span class="s2">"if (f == null) { continue; } "</span>
|
||||
<span class="s1">'def rt = f.result_type != null ? f.result_type : "none"; '</span>
|
||||
<span class="s1">'def smi = f.sending_mta_ip != null ? f.sending_mta_ip : "none"; '</span>
|
||||
<span class="s1">'def ri = f.receiving_ip != null ? f.receiving_ip : "none"; '</span>
|
||||
<span class="s1">'def rmh = f.receiving_mx_hostname != null ? f.receiving_mx_hostname : "none"; '</span>
|
||||
<span class="s1">'dets.add(dom + " / " + typ + " / " + rt + " / " + smi + " / " + ri + " / " + rmh); '</span>
|
||||
<span class="s2">"} } } } "</span>
|
||||
<span class="s2">"ctx._source.policies_combined = pols; "</span>
|
||||
<span class="s2">"ctx._source.failure_details_combined = dets;"</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_PolicyOverride</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="nb">type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">comment</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
@@ -186,6 +362,11 @@
|
||||
<span class="n">header_from</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">envelope_from</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">envelope_to</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="c1"># Nested(...) on the two auth-result fields below is only the DSL's</span>
|
||||
<span class="c1"># in-memory document shape; it is never installed as a mapping.</span>
|
||||
<span class="c1"># create_indexes() deliberately skips Index.document() registration so</span>
|
||||
<span class="c1"># these fields stay dynamic-mapped as plain `object` in the cluster</span>
|
||||
<span class="c1"># (see the comment there and issue #169).</span>
|
||||
<span class="n">dkim_results</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_DKIMResult</span><span class="p">)</span>
|
||||
<span class="n">spf_results</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SPFResult</span><span class="p">)</span>
|
||||
<span class="c1"># One "{selector} / {domain} / {result}" (DKIM) or "{scope} / {domain} /</span>
|
||||
@@ -223,9 +404,7 @@
|
||||
<span class="n">human_result</span><span class="o">=</span><span class="n">human_result</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">dkim_results_combined</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="s2">"</span><span class="si">{0}</span><span class="s2"> / </span><span class="si">{1}</span><span class="s2"> / </span><span class="si">{2}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">selector</span><span class="p">,</span> <span class="n">domain</span><span class="p">,</span> <span class="n">result</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">dkim_results_combined</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">selector</span><span class="si">}</span><span class="s2"> / </span><span class="si">{</span><span class="n">domain</span><span class="si">}</span><span class="s2"> / </span><span class="si">{</span><span class="n">result</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_spf_result</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="p">,</span>
|
||||
@@ -242,9 +421,7 @@
|
||||
<span class="n">human_result</span><span class="o">=</span><span class="n">human_result</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">spf_results_combined</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="s2">"</span><span class="si">{0}</span><span class="s2"> / </span><span class="si">{1}</span><span class="s2"> / </span><span class="si">{2}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">scope</span><span class="p">,</span> <span class="n">domain</span><span class="p">,</span> <span class="n">result</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">spf_results_combined</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">scope</span><span class="si">}</span><span class="s2"> / </span><span class="si">{</span><span class="n">domain</span><span class="si">}</span><span class="s2"> / </span><span class="si">{</span><span class="n">result</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">save</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span> <span class="c1"># pyright: ignore[reportIncompatibleMethodOverride]</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">passed_dmarc</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
@@ -331,6 +508,7 @@
|
||||
<span class="n">result_type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">sending_mta_ip</span> <span class="o">=</span> <span class="n">Ip</span><span class="p">()</span>
|
||||
<span class="n">receiving_mx_helo</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">receiving_mx_hostname</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">receiving_ip</span> <span class="o">=</span> <span class="n">Ip</span><span class="p">()</span>
|
||||
<span class="n">failed_session_count</span> <span class="o">=</span> <span class="n">Integer</span><span class="p">()</span>
|
||||
<span class="n">additional_information_uri</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
@@ -366,7 +544,7 @@
|
||||
<span class="n">receiving_mx_helo</span><span class="o">=</span><span class="n">receiving_mx_helo</span><span class="p">,</span>
|
||||
<span class="n">receiving_ip</span><span class="o">=</span><span class="n">receiving_ip</span><span class="p">,</span>
|
||||
<span class="n">failed_session_count</span><span class="o">=</span><span class="n">failed_session_count</span><span class="p">,</span>
|
||||
<span class="n">additional_information</span><span class="o">=</span><span class="n">additional_information_uri</span><span class="p">,</span>
|
||||
<span class="n">additional_information_uri</span><span class="o">=</span><span class="n">additional_information_uri</span><span class="p">,</span>
|
||||
<span class="n">failure_reason_code</span><span class="o">=</span><span class="n">failure_reason_code</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">failure_details</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_details</span><span class="p">)</span>
|
||||
@@ -383,6 +561,19 @@
|
||||
<span class="n">contact_info</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">report_id</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">policies</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SMTPTLSPolicyDoc</span><span class="p">)</span>
|
||||
<span class="c1"># One "{policy_domain} / {policy_type}" string per policy. Kibana/</span>
|
||||
<span class="c1"># Grafana tables cannot terms-aggregate the subfields of an object</span>
|
||||
<span class="c1"># array without producing a cross-product of values (issue #169), so</span>
|
||||
<span class="c1"># dashboards aggregate these composed keywords instead. Declared to</span>
|
||||
<span class="c1"># match what dynamic mapping produces for a string array (text +</span>
|
||||
<span class="c1"># .keyword).</span>
|
||||
<span class="n">policies_combined</span> <span class="o">=</span> <span class="n">Text</span><span class="p">(</span><span class="n">multi</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">fields</span><span class="o">=</span><span class="p">{</span><span class="s2">"keyword"</span><span class="p">:</span> <span class="n">Keyword</span><span class="p">(</span><span class="n">ignore_above</span><span class="o">=</span><span class="mi">256</span><span class="p">)})</span>
|
||||
<span class="c1"># One "{policy_domain} / {policy_type} / {result_type} /</span>
|
||||
<span class="c1"># {sending_mta_ip} / {receiving_ip} / {receiving_mx_hostname}" string</span>
|
||||
<span class="c1"># per failure detail, across all policies.</span>
|
||||
<span class="n">failure_details_combined</span> <span class="o">=</span> <span class="n">Text</span><span class="p">(</span>
|
||||
<span class="n">multi</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">fields</span><span class="o">=</span><span class="p">{</span><span class="s2">"keyword"</span><span class="p">:</span> <span class="n">Keyword</span><span class="p">(</span><span class="n">ignore_above</span><span class="o">=</span><span class="mi">256</span><span class="p">)}</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="AlreadySaved">
|
||||
@@ -478,20 +669,27 @@
|
||||
<span class="k">for</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">names</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="n">Index</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="c1"># Deliberately no Index.document() registration: Kibana/OpenSearch</span>
|
||||
<span class="c1"># Dashboards/Grafana cannot terms-aggregate fields inside a</span>
|
||||
<span class="c1"># `nested` mapping, so the dynamic `object` mapping produced by a</span>
|
||||
<span class="c1"># bare create is load-bearing for the shipped dashboards (issue</span>
|
||||
<span class="c1"># #169; see the *_combined fields on _AggregateReportDoc).</span>
|
||||
<span class="c1"># Deliberately no Index.document() registration: the shipped</span>
|
||||
<span class="c1"># dashboards cannot rebuild their detail tables on a `nested`</span>
|
||||
<span class="c1"># mapping — Kibana/OSD visual editors do not support nested</span>
|
||||
<span class="c1"># fields, Vega can run nested aggregations but does not render</span>
|
||||
<span class="c1"># tables, and Grafana's nested bucket aggregation (9.4+) lacks</span>
|
||||
<span class="c1"># reverse_nested for parent-level metrics like message_count —</span>
|
||||
<span class="c1"># so the dynamic `object` mapping produced by a bare create is</span>
|
||||
<span class="c1"># load-bearing for the shipped dashboards. _AggregateReportDoc</span>
|
||||
<span class="c1"># still declares dkim_results/spf_results with Nested(...), but</span>
|
||||
<span class="c1"># that is only the DSL's in-memory shape for building documents</span>
|
||||
<span class="c1"># — it is never installed as a mapping. See issue #169 and the</span>
|
||||
<span class="c1"># *_combined fields on _AggregateReportDoc.</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">index</span><span class="o">.</span><span class="n">exists</span><span class="p">():</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Creating OpenSearch index: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">name</span><span class="p">))</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Creating OpenSearch index: </span><span class="si">{</span><span class="n">name</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">settings</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">index</span><span class="o">.</span><span class="n">settings</span><span class="p">(</span><span class="n">number_of_shards</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">number_of_replicas</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">index</span><span class="o">.</span><span class="n">settings</span><span class="p">(</span><span class="o">**</span><span class="n">settings</span><span class="p">)</span>
|
||||
<span class="n">index</span><span class="o">.</span><span class="n">create</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span><span class="s2">"OpenSearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"OpenSearch error: </span><span class="si">{</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
|
||||
@@ -500,47 +698,175 @@
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">migrate_indexes</span><span class="p">(</span>
|
||||
<span class="n">aggregate_indexes</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">failure_indexes</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">smtp_tls_indexes</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Updates index mappings</span>
|
||||
<span class="sd"> Runs index migrations and backfills.</span>
|
||||
|
||||
<span class="sd"> First, the legacy ``published_policy.fo`` migration: indexes where that</span>
|
||||
<span class="sd"> field was mapped as ``long`` (data indexed by very old parsedmarc</span>
|
||||
<span class="sd"> releases under the Elasticsearch 6-era ``doc`` mapping type) are rebuilt</span>
|
||||
<span class="sd"> as a ``-v2`` index with the text/keyword shape.</span>
|
||||
|
||||
<span class="sd"> Second, the ``dkim_results_combined``/``spf_results_combined`` backfill</span>
|
||||
<span class="sd"> (added for issue #169) for aggregate report documents that were saved</span>
|
||||
<span class="sd"> before those fields existed. For each name in ``aggregate_indexes``,</span>
|
||||
<span class="sd"> this submits an ``update_by_query`` against the ``f"{name}*"`` index</span>
|
||||
<span class="sd"> pattern (the real indexes are date-suffixed) as a non-blocking</span>
|
||||
<span class="sd"> background task (``wait_for_completion=False``), so it never delays</span>
|
||||
<span class="sd"> parsedmarc startup. Submission is guarded by a cheap ``count`` query</span>
|
||||
<span class="sd"> that only matches documents with DKIM/SPF results but no combined</span>
|
||||
<span class="sd"> field, so once an index is fully backfilled, later calls are a fast</span>
|
||||
<span class="sd"> no-op. Any error talking to the cluster (e.g. no indexes yet on a</span>
|
||||
<span class="sd"> fresh install, or a transient connection issue) is caught and logged</span>
|
||||
<span class="sd"> as a warning rather than raised; the backfill is simply retried on the</span>
|
||||
<span class="sd"> next startup, and the manual ``_update_by_query`` command documented</span>
|
||||
<span class="sd"> in ``docs/source/elasticsearch.md`` remains available in the meantime.</span>
|
||||
|
||||
<span class="sd"> Third, the same treatment for the ``policies_combined``/</span>
|
||||
<span class="sd"> ``failure_details_combined`` fields (same issue #169) on SMTP TLS</span>
|
||||
<span class="sd"> report documents, for each name in ``smtp_tls_indexes``.</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> aggregate_indexes (list): A list of aggregate index names</span>
|
||||
<span class="sd"> failure_indexes (list): A list of failure index names</span>
|
||||
<span class="sd"> (accepted for API compatibility; no migrations are</span>
|
||||
<span class="sd"> currently needed for failure indexes)</span>
|
||||
<span class="sd"> smtp_tls_indexes (list): A list of SMTP TLS index names</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">version</span> <span class="o">=</span> <span class="mi">2</span>
|
||||
<span class="k">if</span> <span class="n">aggregate_indexes</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">aggregate_indexes</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="k">for</span> <span class="n">aggregate_index_name</span> <span class="ow">in</span> <span class="n">aggregate_indexes</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">Index</span><span class="p">(</span><span class="n">aggregate_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">exists</span><span class="p">():</span>
|
||||
<span class="k">continue</span>
|
||||
<span class="n">aggregate_index</span> <span class="o">=</span> <span class="n">Index</span><span class="p">(</span><span class="n">aggregate_index_name</span><span class="p">)</span>
|
||||
<span class="n">doc</span> <span class="o">=</span> <span class="s2">"doc"</span>
|
||||
<span class="n">fo_field</span> <span class="o">=</span> <span class="s2">"published_policy.fo"</span>
|
||||
<span class="n">fo</span> <span class="o">=</span> <span class="s2">"fo"</span>
|
||||
<span class="n">fo_mapping</span> <span class="o">=</span> <span class="n">aggregate_index</span><span class="o">.</span><span class="n">get_field_mapping</span><span class="p">(</span><span class="n">fields</span><span class="o">=</span><span class="p">[</span><span class="n">fo_field</span><span class="p">])</span>
|
||||
<span class="n">fo_mapping</span> <span class="o">=</span> <span class="n">fo_mapping</span><span class="p">[</span><span class="nb">list</span><span class="p">(</span><span class="n">fo_mapping</span><span class="o">.</span><span class="n">keys</span><span class="p">())[</span><span class="mi">0</span><span class="p">]][</span><span class="s2">"mappings"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="n">doc</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">fo_mapping</span><span class="p">:</span>
|
||||
<span class="k">continue</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">aggregate_indexes</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">smtp_tls_indexes</span><span class="p">:</span>
|
||||
<span class="k">return</span>
|
||||
|
||||
<span class="n">fo_mapping</span> <span class="o">=</span> <span class="n">fo_mapping</span><span class="p">[</span><span class="n">doc</span><span class="p">][</span><span class="n">fo_field</span><span class="p">][</span><span class="s2">"mapping"</span><span class="p">][</span><span class="n">fo</span><span class="p">]</span>
|
||||
<span class="n">fo_type</span> <span class="o">=</span> <span class="n">fo_mapping</span><span class="p">[</span><span class="s2">"type"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="n">fo_type</span> <span class="o">==</span> <span class="s2">"long"</span><span class="p">:</span>
|
||||
<span class="n">new_index_name</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">-v</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">aggregate_index_name</span><span class="p">,</span> <span class="n">version</span><span class="p">)</span>
|
||||
<span class="n">body</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s2">"properties"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"published_policy.fo"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"type"</span><span class="p">:</span> <span class="s2">"text"</span><span class="p">,</span>
|
||||
<span class="s2">"fields"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"keyword"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"type"</span><span class="p">:</span> <span class="s2">"keyword"</span><span class="p">,</span> <span class="s2">"ignore_above"</span><span class="p">:</span> <span class="mi">256</span><span class="p">}},</span>
|
||||
<span class="n">version</span> <span class="o">=</span> <span class="mi">2</span>
|
||||
<span class="k">for</span> <span class="n">aggregate_index_name</span> <span class="ow">in</span> <span class="n">aggregate_indexes</span> <span class="ow">or</span> <span class="p">[]:</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">Index</span><span class="p">(</span><span class="n">aggregate_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">exists</span><span class="p">():</span>
|
||||
<span class="k">continue</span>
|
||||
<span class="n">aggregate_index</span> <span class="o">=</span> <span class="n">Index</span><span class="p">(</span><span class="n">aggregate_index_name</span><span class="p">)</span>
|
||||
<span class="n">doc</span> <span class="o">=</span> <span class="s2">"doc"</span>
|
||||
<span class="n">fo_field</span> <span class="o">=</span> <span class="s2">"published_policy.fo"</span>
|
||||
<span class="n">fo</span> <span class="o">=</span> <span class="s2">"fo"</span>
|
||||
<span class="n">fo_mapping</span> <span class="o">=</span> <span class="n">aggregate_index</span><span class="o">.</span><span class="n">get_field_mapping</span><span class="p">(</span><span class="n">fields</span><span class="o">=</span><span class="p">[</span><span class="n">fo_field</span><span class="p">])</span>
|
||||
<span class="n">fo_mapping</span> <span class="o">=</span> <span class="n">fo_mapping</span><span class="p">[</span><span class="nb">list</span><span class="p">(</span><span class="n">fo_mapping</span><span class="o">.</span><span class="n">keys</span><span class="p">())[</span><span class="mi">0</span><span class="p">]][</span><span class="s2">"mappings"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="n">doc</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">fo_mapping</span><span class="p">:</span>
|
||||
<span class="k">continue</span>
|
||||
|
||||
<span class="n">fo_mapping</span> <span class="o">=</span> <span class="n">fo_mapping</span><span class="p">[</span><span class="n">doc</span><span class="p">][</span><span class="n">fo_field</span><span class="p">][</span><span class="s2">"mapping"</span><span class="p">][</span><span class="n">fo</span><span class="p">]</span>
|
||||
<span class="n">fo_type</span> <span class="o">=</span> <span class="n">fo_mapping</span><span class="p">[</span><span class="s2">"type"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="n">fo_type</span> <span class="o">==</span> <span class="s2">"long"</span><span class="p">:</span>
|
||||
<span class="n">new_index_name</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">aggregate_index_name</span><span class="si">}</span><span class="s2">-v</span><span class="si">{</span><span class="n">version</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="n">body</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s2">"properties"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"published_policy.fo"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"type"</span><span class="p">:</span> <span class="s2">"text"</span><span class="p">,</span>
|
||||
<span class="s2">"fields"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"keyword"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"type"</span><span class="p">:</span> <span class="s2">"keyword"</span><span class="p">,</span> <span class="s2">"ignore_above"</span><span class="p">:</span> <span class="mi">256</span><span class="p">}</span>
|
||||
<span class="p">},</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">}</span>
|
||||
<span class="n">Index</span><span class="p">(</span><span class="n">new_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">create</span><span class="p">()</span>
|
||||
<span class="n">Index</span><span class="p">(</span><span class="n">new_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">put_mapping</span><span class="p">(</span><span class="n">doc_type</span><span class="o">=</span><span class="n">doc</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="n">body</span><span class="p">)</span>
|
||||
<span class="n">reindex</span><span class="p">(</span><span class="n">connections</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(),</span> <span class="n">aggregate_index_name</span><span class="p">,</span> <span class="n">new_index_name</span><span class="p">)</span>
|
||||
<span class="n">Index</span><span class="p">(</span><span class="n">aggregate_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">delete</span><span class="p">()</span></div>
|
||||
<span class="n">Index</span><span class="p">(</span><span class="n">new_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">create</span><span class="p">()</span>
|
||||
<span class="n">Index</span><span class="p">(</span><span class="n">new_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">put_mapping</span><span class="p">(</span><span class="n">doc_type</span><span class="o">=</span><span class="n">doc</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="n">body</span><span class="p">)</span>
|
||||
<span class="n">reindex</span><span class="p">(</span>
|
||||
<span class="n">connections</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(),</span> <span class="n">aggregate_index_name</span><span class="p">,</span> <span class="n">new_index_name</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">Index</span><span class="p">(</span><span class="n">aggregate_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">delete</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
||||
<span class="s2">"Failed the legacy published_policy.fo migration for "</span>
|
||||
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">aggregate_index_name</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">. This will be retried at the "</span>
|
||||
<span class="s2">"next startup."</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">client</span> <span class="o">=</span> <span class="n">connections</span><span class="o">.</span><span class="n">get_connection</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
||||
<span class="s2">"Skipping the dkim_results_combined/spf_results_combined/"</span>
|
||||
<span class="s2">"policies_combined/failure_details_combined backfill: could "</span>
|
||||
<span class="sa">f</span><span class="s2">"not get an OpenSearch connection: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">. This will be retried "</span>
|
||||
<span class="s2">"at the next startup."</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">return</span>
|
||||
<span class="k">for</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">aggregate_indexes</span> <span class="ow">or</span> <span class="p">[]:</span>
|
||||
<span class="n">pattern</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">name</span><span class="si">}</span><span class="s2">*"</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">count_response</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">count</span><span class="p">(</span>
|
||||
<span class="n">index</span><span class="o">=</span><span class="n">pattern</span><span class="p">,</span>
|
||||
<span class="n">body</span><span class="o">=</span><span class="p">{</span><span class="s2">"query"</span><span class="p">:</span> <span class="n">_COMBINED_BACKFILL_QUERY</span><span class="p">},</span>
|
||||
<span class="n">ignore_unavailable</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="n">allow_no_indices</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">count</span> <span class="o">=</span> <span class="n">count_response</span><span class="p">[</span><span class="s2">"count"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">count</span><span class="p">:</span>
|
||||
<span class="k">continue</span>
|
||||
<span class="n">update_response</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">update_by_query</span><span class="p">(</span>
|
||||
<span class="n">index</span><span class="o">=</span><span class="n">pattern</span><span class="p">,</span>
|
||||
<span class="n">body</span><span class="o">=</span><span class="p">{</span>
|
||||
<span class="s2">"query"</span><span class="p">:</span> <span class="n">_COMBINED_BACKFILL_QUERY</span><span class="p">,</span>
|
||||
<span class="s2">"script"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"source"</span><span class="p">:</span> <span class="n">_COMBINED_BACKFILL_SCRIPT</span><span class="p">,</span>
|
||||
<span class="s2">"lang"</span><span class="p">:</span> <span class="s2">"painless"</span><span class="p">,</span>
|
||||
<span class="p">},</span>
|
||||
<span class="p">},</span>
|
||||
<span class="n">conflicts</span><span class="o">=</span><span class="s2">"proceed"</span><span class="p">,</span>
|
||||
<span class="n">wait_for_completion</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">ignore_unavailable</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="n">allow_no_indices</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">task_id</span> <span class="o">=</span> <span class="n">update_response</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"task"</span><span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
|
||||
<span class="s2">"Backfilling dkim_results_combined/spf_results_combined on "</span>
|
||||
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">count</span><span class="si">}</span><span class="s2"> existing documents in </span><span class="si">{</span><span class="n">pattern</span><span class="si">}</span><span class="s2"> (task </span><span class="si">{</span><span class="n">task_id</span><span class="si">}</span><span class="s2">)"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
||||
<span class="s2">"Failed to check/submit the dkim_results_combined/"</span>
|
||||
<span class="sa">f</span><span class="s2">"spf_results_combined backfill for </span><span class="si">{</span><span class="n">pattern</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">. This "</span>
|
||||
<span class="s2">"will be retried at the next startup; the manual "</span>
|
||||
<span class="s2">"_update_by_query command in the documentation remains "</span>
|
||||
<span class="s2">"available in the meantime."</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">smtp_tls_indexes</span> <span class="ow">or</span> <span class="p">[]:</span>
|
||||
<span class="n">pattern</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">name</span><span class="si">}</span><span class="s2">*"</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">count_response</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">count</span><span class="p">(</span>
|
||||
<span class="n">index</span><span class="o">=</span><span class="n">pattern</span><span class="p">,</span>
|
||||
<span class="n">body</span><span class="o">=</span><span class="p">{</span><span class="s2">"query"</span><span class="p">:</span> <span class="n">_SMTP_TLS_COMBINED_BACKFILL_QUERY</span><span class="p">},</span>
|
||||
<span class="n">ignore_unavailable</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="n">allow_no_indices</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">count</span> <span class="o">=</span> <span class="n">count_response</span><span class="p">[</span><span class="s2">"count"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">count</span><span class="p">:</span>
|
||||
<span class="k">continue</span>
|
||||
<span class="n">update_response</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">update_by_query</span><span class="p">(</span>
|
||||
<span class="n">index</span><span class="o">=</span><span class="n">pattern</span><span class="p">,</span>
|
||||
<span class="n">body</span><span class="o">=</span><span class="p">{</span>
|
||||
<span class="s2">"query"</span><span class="p">:</span> <span class="n">_SMTP_TLS_COMBINED_BACKFILL_QUERY</span><span class="p">,</span>
|
||||
<span class="s2">"script"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"source"</span><span class="p">:</span> <span class="n">_SMTP_TLS_COMBINED_BACKFILL_SCRIPT</span><span class="p">,</span>
|
||||
<span class="s2">"lang"</span><span class="p">:</span> <span class="s2">"painless"</span><span class="p">,</span>
|
||||
<span class="p">},</span>
|
||||
<span class="p">},</span>
|
||||
<span class="n">conflicts</span><span class="o">=</span><span class="s2">"proceed"</span><span class="p">,</span>
|
||||
<span class="n">wait_for_completion</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">ignore_unavailable</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="n">allow_no_indices</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">task_id</span> <span class="o">=</span> <span class="n">update_response</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"task"</span><span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
|
||||
<span class="s2">"Backfilling policies_combined/failure_details_combined on "</span>
|
||||
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">count</span><span class="si">}</span><span class="s2"> existing documents in </span><span class="si">{</span><span class="n">pattern</span><span class="si">}</span><span class="s2"> (task </span><span class="si">{</span><span class="n">task_id</span><span class="si">}</span><span class="s2">)"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
||||
<span class="s2">"Failed to check/submit the policies_combined/"</span>
|
||||
<span class="sa">f</span><span class="s2">"failure_details_combined backfill for </span><span class="si">{</span><span class="n">pattern</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">. "</span>
|
||||
<span class="s2">"This will be retried at the next startup; the manual "</span>
|
||||
<span class="s2">"_update_by_query command in the documentation remains "</span>
|
||||
<span class="s2">"available in the meantime."</span>
|
||||
<span class="p">)</span></div>
|
||||
|
||||
|
||||
|
||||
@@ -589,11 +915,11 @@
|
||||
<span class="n">end_date_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="nb">range</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">date_end</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">lte</span><span class="o">=</span><span class="n">end_date</span><span class="p">))))</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">index_suffix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"dmarc_aggregate_</span><span class="si">{0}</span><span class="s2">*"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"dmarc_aggregate_</span><span class="si">{</span><span class="n">index_suffix</span><span class="si">}</span><span class="s2">*"</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"dmarc_aggregate*"</span>
|
||||
<span class="k">if</span> <span class="n">index_prefix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">search_index</span><span class="p">)</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">index_prefix</span><span class="si">}{</span><span class="n">search_index</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="n">search_index</span><span class="p">)</span>
|
||||
<span class="n">query</span> <span class="o">=</span> <span class="n">org_name_query</span> <span class="o">&</span> <span class="n">report_id_query</span> <span class="o">&</span> <span class="n">domain_query</span>
|
||||
<span class="n">query</span> <span class="o">=</span> <span class="n">query</span> <span class="o">&</span> <span class="n">begin_date_query</span> <span class="o">&</span> <span class="n">end_date_query</span>
|
||||
@@ -605,18 +931,15 @@
|
||||
<span class="n">existing</span> <span class="o">=</span> <span class="n">search</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error_</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span>
|
||||
<span class="s2">"OpenSearch's search for existing report </span><span class="se">\</span>
|
||||
<span class="s2"> error: </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error_</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="sa">f</span><span class="s2">"OpenSearch's search for existing report error: </span><span class="si">{</span><span class="n">error_</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">existing</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">AlreadySaved</span><span class="p">(</span>
|
||||
<span class="s2">"An aggregate report ID </span><span class="si">{0}</span><span class="s2"> from </span><span class="si">{1}</span><span class="s2"> about </span><span class="si">{2}</span><span class="s2"> "</span>
|
||||
<span class="s2">"with a date range of </span><span class="si">{3}</span><span class="s2"> UTC to </span><span class="si">{4}</span><span class="s2"> UTC already "</span>
|
||||
<span class="sa">f</span><span class="s2">"An aggregate report ID </span><span class="si">{</span><span class="n">report_id</span><span class="si">}</span><span class="s2"> from </span><span class="si">{</span><span class="n">org_name</span><span class="si">}</span><span class="s2"> about </span><span class="si">{</span><span class="n">domain</span><span class="si">}</span><span class="s2"> "</span>
|
||||
<span class="sa">f</span><span class="s2">"with a date range of </span><span class="si">{</span><span class="n">begin_date_human</span><span class="si">}</span><span class="s2"> UTC to </span><span class="si">{</span><span class="n">end_date_human</span><span class="si">}</span><span class="s2"> UTC already "</span>
|
||||
<span class="s2">"exists in "</span>
|
||||
<span class="s2">"OpenSearch"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">report_id</span><span class="p">,</span> <span class="n">org_name</span><span class="p">,</span> <span class="n">domain</span><span class="p">,</span> <span class="n">begin_date_human</span><span class="p">,</span> <span class="n">end_date_human</span>
|
||||
<span class="p">)</span>
|
||||
<span class="s2">"OpenSearch"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">published_policy</span> <span class="o">=</span> <span class="n">_PublishedPolicy</span><span class="p">(</span>
|
||||
<span class="n">domain</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"domain"</span><span class="p">],</span>
|
||||
@@ -709,11 +1032,11 @@
|
||||
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"dmarc_aggregate"</span>
|
||||
<span class="k">if</span> <span class="n">index_suffix</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">_</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">index</span><span class="si">}</span><span class="s2">_</span><span class="si">{</span><span class="n">index_suffix</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="k">if</span> <span class="n">index_prefix</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">index</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">index_prefix</span><span class="si">}{</span><span class="n">index</span><span class="si">}</span><span class="s2">"</span>
|
||||
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">-</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_date</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">index</span><span class="si">}</span><span class="s2">-</span><span class="si">{</span><span class="n">index_date</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span> <span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span>
|
||||
<span class="p">)</span>
|
||||
@@ -723,7 +1046,7 @@
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">agg_doc</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span><span class="s2">"OpenSearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"OpenSearch error: </span><span class="si">{</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
|
||||
@@ -774,12 +1097,12 @@
|
||||
<span class="n">arrival_date_epoch_milliseconds</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">arrival_date</span><span class="o">.</span><span class="n">timestamp</span><span class="p">()</span> <span class="o">*</span> <span class="mi">1000</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">index_suffix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"dmarc_failure_</span><span class="si">{0}</span><span class="s2">*,dmarc_forensic_</span><span class="si">{0}</span><span class="s2">*"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"dmarc_failure_</span><span class="si">{</span><span class="n">index_suffix</span><span class="si">}</span><span class="s2">*,dmarc_forensic_</span><span class="si">{</span><span class="n">index_suffix</span><span class="si">}</span><span class="s2">*"</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"dmarc_failure*,dmarc_forensic*"</span>
|
||||
<span class="k">if</span> <span class="n">index_prefix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">","</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
|
||||
<span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">part</span><span class="p">)</span> <span class="k">for</span> <span class="n">part</span> <span class="ow">in</span> <span class="n">search_index</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">","</span><span class="p">)</span>
|
||||
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">index_prefix</span><span class="si">}{</span><span class="n">part</span><span class="si">}</span><span class="s2">"</span> <span class="k">for</span> <span class="n">part</span> <span class="ow">in</span> <span class="n">search_index</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">","</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="n">search_index</span><span class="p">)</span>
|
||||
<span class="n">q</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">arrival_date</span><span class="o">=</span><span class="n">arrival_date_epoch_milliseconds</span><span class="p">)))</span>
|
||||
@@ -831,8 +1154,8 @@
|
||||
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">existing</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">AlreadySaved</span><span class="p">(</span>
|
||||
<span class="s2">"A failure sample to </span><span class="si">{0}</span><span class="s2"> from </span><span class="si">{1}</span><span class="s2"> "</span>
|
||||
<span class="s2">"with a subject of </span><span class="si">{2}</span><span class="s2"> and arrival date of </span><span class="si">{3}</span><span class="s2"> "</span>
|
||||
<span class="s2">"A failure sample to </span><span class="si">{}</span><span class="s2"> from </span><span class="si">{}</span><span class="s2"> "</span>
|
||||
<span class="s2">"with a subject of </span><span class="si">{}</span><span class="s2"> and arrival date of </span><span class="si">{}</span><span class="s2"> "</span>
|
||||
<span class="s2">"already exists in "</span>
|
||||
<span class="s2">"OpenSearch"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">to_</span><span class="p">,</span> <span class="n">from_</span><span class="p">,</span> <span class="n">subject</span><span class="p">,</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"arrival_date_utc"</span><span class="p">])</span>
|
||||
<span class="p">)</span>
|
||||
@@ -891,14 +1214,14 @@
|
||||
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"dmarc_failure"</span>
|
||||
<span class="k">if</span> <span class="n">index_suffix</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">_</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">index</span><span class="si">}</span><span class="s2">_</span><span class="si">{</span><span class="n">index_suffix</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="k">if</span> <span class="n">index_prefix</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">index</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">index_prefix</span><span class="si">}{</span><span class="n">index</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="k">if</span> <span class="n">monthly_indexes</span><span class="p">:</span>
|
||||
<span class="n">index_date</span> <span class="o">=</span> <span class="n">arrival_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m"</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">index_date</span> <span class="o">=</span> <span class="n">arrival_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">-</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_date</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">index</span><span class="si">}</span><span class="s2">-</span><span class="si">{</span><span class="n">index_date</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span> <span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span>
|
||||
<span class="p">)</span>
|
||||
@@ -907,10 +1230,10 @@
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">failure_doc</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span><span class="s2">"OpenSearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"OpenSearch error: </span><span class="si">{</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="k">except</span> <span class="ne">KeyError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidFailureReport</span><span class="p">(</span>
|
||||
<span class="s2">"Failure report missing required field: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="sa">f</span><span class="s2">"Failure report missing required field: </span><span class="si">{</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="p">)</span></div>
|
||||
|
||||
|
||||
@@ -960,11 +1283,11 @@
|
||||
<span class="n">end_date_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">date_end</span><span class="o">=</span><span class="n">end_date</span><span class="p">)))</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">index_suffix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"smtp_tls_</span><span class="si">{0}</span><span class="s2">*"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"smtp_tls_</span><span class="si">{</span><span class="n">index_suffix</span><span class="si">}</span><span class="s2">*"</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"smtp_tls*"</span>
|
||||
<span class="k">if</span> <span class="n">index_prefix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">search_index</span><span class="p">)</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">index_prefix</span><span class="si">}{</span><span class="n">search_index</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="n">search_index</span><span class="p">)</span>
|
||||
<span class="n">query</span> <span class="o">=</span> <span class="n">org_name_query</span> <span class="o">&</span> <span class="n">report_id_query</span>
|
||||
<span class="n">query</span> <span class="o">=</span> <span class="n">query</span> <span class="o">&</span> <span class="n">begin_date_query</span> <span class="o">&</span> <span class="n">end_date_query</span>
|
||||
@@ -974,8 +1297,7 @@
|
||||
<span class="n">existing</span> <span class="o">=</span> <span class="n">search</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error_</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span>
|
||||
<span class="s2">"OpenSearch's search for existing report </span><span class="se">\</span>
|
||||
<span class="s2"> error: </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error_</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="sa">f</span><span class="s2">"OpenSearch's search for existing report error: </span><span class="si">{</span><span class="n">error_</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">existing</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||
@@ -989,10 +1311,10 @@
|
||||
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"smtp_tls"</span>
|
||||
<span class="k">if</span> <span class="n">index_suffix</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">_</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">index</span><span class="si">}</span><span class="s2">_</span><span class="si">{</span><span class="n">index_suffix</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="k">if</span> <span class="n">index_prefix</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">index</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">-</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_date</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">index_prefix</span><span class="si">}{</span><span class="n">index</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">index</span><span class="si">}</span><span class="s2">-</span><span class="si">{</span><span class="n">index_date</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span> <span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span>
|
||||
<span class="p">)</span>
|
||||
@@ -1013,6 +1335,16 @@
|
||||
<span class="n">policy_strings</span> <span class="o">=</span> <span class="n">policy</span><span class="p">[</span><span class="s2">"policy_strings"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="s2">"mx_host_patterns"</span> <span class="ow">in</span> <span class="n">policy</span><span class="p">:</span>
|
||||
<span class="n">mx_host_patterns</span> <span class="o">=</span> <span class="n">policy</span><span class="p">[</span><span class="s2">"mx_host_patterns"</span><span class="p">]</span>
|
||||
<span class="c1"># policies_combined/failure_details_combined: see the field</span>
|
||||
<span class="c1"># declarations on _SMTPTLSReportDoc and issue #169. policies and</span>
|
||||
<span class="c1"># their failure_details are object arrays with the same</span>
|
||||
<span class="c1"># cross-product problem as dkim_results/spf_results, so dashboards</span>
|
||||
<span class="c1"># aggregate these composed strings instead of the raw subfields.</span>
|
||||
<span class="n">policy_domain_combined</span> <span class="o">=</span> <span class="n">policy</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"policy_domain"</span><span class="p">)</span> <span class="ow">or</span> <span class="s2">"none"</span>
|
||||
<span class="n">policy_type_combined</span> <span class="o">=</span> <span class="n">policy</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"policy_type"</span><span class="p">)</span> <span class="ow">or</span> <span class="s2">"none"</span>
|
||||
<span class="n">smtp_tls_doc</span><span class="o">.</span><span class="n">policies_combined</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">policy_domain_combined</span><span class="si">}</span><span class="s2"> / </span><span class="si">{</span><span class="n">policy_type_combined</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">policy_doc</span> <span class="o">=</span> <span class="n">_SMTPTLSPolicyDoc</span><span class="p">(</span>
|
||||
<span class="n">policy_domain</span><span class="o">=</span><span class="n">policy</span><span class="p">[</span><span class="s2">"policy_domain"</span><span class="p">],</span>
|
||||
<span class="n">policy_type</span><span class="o">=</span><span class="n">policy</span><span class="p">[</span><span class="s2">"policy_type"</span><span class="p">],</span>
|
||||
@@ -1033,7 +1365,12 @@
|
||||
|
||||
<span class="k">if</span> <span class="s2">"receiving_mx_hostname"</span> <span class="ow">in</span> <span class="n">failure_detail</span><span class="p">:</span>
|
||||
<span class="n">receiving_mx_hostname</span> <span class="o">=</span> <span class="n">failure_detail</span><span class="p">[</span><span class="s2">"receiving_mx_hostname"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="s2">"additional_information_uri"</span> <span class="ow">in</span> <span class="n">failure_detail</span><span class="p">:</span>
|
||||
<span class="c1"># The parser's key is additional_info_uri (see</span>
|
||||
<span class="c1"># SMTPTLSFailureDetailsOptional in types.py); accept the</span>
|
||||
<span class="c1"># long-form key too for dicts built by other callers.</span>
|
||||
<span class="k">if</span> <span class="s2">"additional_info_uri"</span> <span class="ow">in</span> <span class="n">failure_detail</span><span class="p">:</span>
|
||||
<span class="n">additional_information_uri</span> <span class="o">=</span> <span class="n">failure_detail</span><span class="p">[</span><span class="s2">"additional_info_uri"</span><span class="p">]</span>
|
||||
<span class="k">elif</span> <span class="s2">"additional_information_uri"</span> <span class="ow">in</span> <span class="n">failure_detail</span><span class="p">:</span>
|
||||
<span class="n">additional_information_uri</span> <span class="o">=</span> <span class="n">failure_detail</span><span class="p">[</span>
|
||||
<span class="s2">"additional_information_uri"</span>
|
||||
<span class="p">]</span>
|
||||
@@ -1058,6 +1395,16 @@
|
||||
<span class="n">additional_information_uri</span><span class="o">=</span><span class="n">additional_information_uri</span><span class="p">,</span>
|
||||
<span class="n">failure_reason_code</span><span class="o">=</span><span class="n">failure_reason_code</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">smtp_tls_doc</span><span class="o">.</span><span class="n">failure_details_combined</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="s2">"</span><span class="si">{}</span><span class="s2"> / </span><span class="si">{}</span><span class="s2"> / </span><span class="si">{}</span><span class="s2"> / </span><span class="si">{}</span><span class="s2"> / </span><span class="si">{}</span><span class="s2"> / </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">policy_domain_combined</span><span class="p">,</span>
|
||||
<span class="n">policy_type_combined</span><span class="p">,</span>
|
||||
<span class="n">failure_detail</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"result_type"</span><span class="p">)</span> <span class="ow">or</span> <span class="s2">"none"</span><span class="p">,</span>
|
||||
<span class="n">sending_mta_ip</span> <span class="ow">or</span> <span class="s2">"none"</span><span class="p">,</span>
|
||||
<span class="n">receiving_ip</span> <span class="ow">or</span> <span class="s2">"none"</span><span class="p">,</span>
|
||||
<span class="n">receiving_mx_hostname</span> <span class="ow">or</span> <span class="s2">"none"</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">smtp_tls_doc</span><span class="o">.</span><span class="n">policies</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">policy_doc</span><span class="p">)</span>
|
||||
|
||||
<span class="n">create_indexes</span><span class="p">([</span><span class="n">index</span><span class="p">],</span> <span class="n">index_settings</span><span class="p">)</span>
|
||||
@@ -1066,7 +1413,7 @@
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">smtp_tls_doc</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span><span class="s2">"OpenSearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"OpenSearch error: </span><span class="si">{</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -5,14 +5,14 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc.splunk — parsedmarc 10.2.4 documentation</title>
|
||||
<title>parsedmarc.splunk — parsedmarc 10.3.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=9edc463e" />
|
||||
|
||||
|
||||
<script src="../../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../../_static/documentation_options.js?v=2c8206f1"></script>
|
||||
<script src="../../_static/documentation_options.js?v=a126560f"></script>
|
||||
<script src="../../_static/doctools.js?v=fd6eb6e6"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=6ffebe34"></script>
|
||||
<script src="../../_static/js/theme.js"></script>
|
||||
@@ -133,8 +133,8 @@
|
||||
<span class="sd"> data before giving up</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">parsed_url</span> <span class="o">=</span> <span class="n">urlparse</span><span class="p">(</span><span class="n">url</span><span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">url</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">://</span><span class="si">{1}</span><span class="s2">/services/collector/event/1.0"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">parsed_url</span><span class="o">.</span><span class="n">scheme</span><span class="p">,</span> <span class="n">parsed_url</span><span class="o">.</span><span class="n">netloc</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">url</span> <span class="o">=</span> <span class="p">(</span>
|
||||
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">parsed_url</span><span class="o">.</span><span class="n">scheme</span><span class="si">}</span><span class="s2">://</span><span class="si">{</span><span class="n">parsed_url</span><span class="o">.</span><span class="n">netloc</span><span class="si">}</span><span class="s2">/services/collector/event/1.0"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">access_token</span> <span class="o">=</span> <span class="n">access_token</span><span class="o">.</span><span class="n">lstrip</span><span class="p">(</span><span class="s2">"Splunk "</span><span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">index</span>
|
||||
@@ -149,7 +149,7 @@
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">session</span> <span class="o">=</span> <span class="n">httpx</span><span class="o">.</span><span class="n">Client</span><span class="p">(</span>
|
||||
<span class="n">headers</span><span class="o">=</span><span class="p">{</span>
|
||||
<span class="s2">"User-Agent"</span><span class="p">:</span> <span class="n">USER_AGENT</span><span class="p">,</span>
|
||||
<span class="s2">"Authorization"</span><span class="p">:</span> <span class="s2">"Splunk </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">access_token</span><span class="p">),</span>
|
||||
<span class="s2">"Authorization"</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"Splunk </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">access_token</span><span class="si">}</span><span class="s2">"</span><span class="p">,</span>
|
||||
<span class="p">},</span>
|
||||
<span class="n">verify</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">verify</span><span class="p">,</span>
|
||||
<span class="n">follow_redirects</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
@@ -177,7 +177,7 @@
|
||||
<span class="k">return</span>
|
||||
|
||||
<span class="n">data</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_common_data</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
<span class="n">json_str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||
<span class="n">json_lines</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="k">for</span> <span class="n">report</span> <span class="ow">in</span> <span class="n">aggregate_reports</span><span class="p">:</span>
|
||||
<span class="k">for</span> <span class="n">record</span> <span class="ow">in</span> <span class="n">report</span><span class="p">[</span><span class="s2">"records"</span><span class="p">]:</span>
|
||||
<span class="n">new_report</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span> <span class="o">|</span> <span class="nb">int</span> <span class="o">|</span> <span class="nb">float</span> <span class="o">|</span> <span class="nb">dict</span><span class="p">]</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
|
||||
@@ -216,13 +216,13 @@
|
||||
<span class="p">)</span>
|
||||
<span class="n">data</span><span class="p">[</span><span class="s2">"time"</span><span class="p">]</span> <span class="o">=</span> <span class="n">timestamp</span>
|
||||
<span class="n">data</span><span class="p">[</span><span class="s2">"event"</span><span class="p">]</span> <span class="o">=</span> <span class="n">new_report</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
<span class="n">json_str</span> <span class="o">+=</span> <span class="s2">"</span><span class="si">{0}</span><span class="se">\n</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">data</span><span class="p">))</span>
|
||||
<span class="n">json_lines</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">data</span><span class="p">)</span><span class="si">}</span><span class="se">\n</span><span class="s2">"</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">verify</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Skipping certificate verification for Splunk HEC"</span><span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">post</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">url</span><span class="p">,</span> <span class="n">content</span><span class="o">=</span><span class="n">json_str</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">timeout</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">url</span><span class="p">,</span> <span class="n">content</span><span class="o">=</span><span class="s2">""</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">json_lines</span><span class="p">),</span> <span class="n">timeout</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">timeout</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">response</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
@@ -251,7 +251,7 @@
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">failure_reports</span><span class="p">)</span> <span class="o"><</span> <span class="mi">1</span><span class="p">:</span>
|
||||
<span class="k">return</span>
|
||||
|
||||
<span class="n">json_str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||
<span class="n">json_lines</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="k">for</span> <span class="n">report</span> <span class="ow">in</span> <span class="n">failure_reports</span><span class="p">:</span>
|
||||
<span class="n">data</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_common_data</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
<span class="n">data</span><span class="p">[</span><span class="s2">"sourcetype"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"dmarc:failure"</span>
|
||||
@@ -262,13 +262,13 @@
|
||||
<span class="p">)</span>
|
||||
<span class="n">data</span><span class="p">[</span><span class="s2">"time"</span><span class="p">]</span> <span class="o">=</span> <span class="n">timestamp</span>
|
||||
<span class="n">data</span><span class="p">[</span><span class="s2">"event"</span><span class="p">]</span> <span class="o">=</span> <span class="n">report</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
<span class="n">json_str</span> <span class="o">+=</span> <span class="s2">"</span><span class="si">{0}</span><span class="se">\n</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">data</span><span class="p">))</span>
|
||||
<span class="n">json_lines</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">data</span><span class="p">)</span><span class="si">}</span><span class="se">\n</span><span class="s2">"</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">verify</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Skipping certificate verification for Splunk HEC"</span><span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">post</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">url</span><span class="p">,</span> <span class="n">content</span><span class="o">=</span><span class="n">json_str</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">timeout</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">url</span><span class="p">,</span> <span class="n">content</span><span class="o">=</span><span class="s2">""</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">json_lines</span><span class="p">),</span> <span class="n">timeout</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">timeout</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">response</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
@@ -298,19 +298,19 @@
|
||||
<span class="k">return</span>
|
||||
|
||||
<span class="n">data</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_common_data</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
<span class="n">json_str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||
<span class="n">json_lines</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="k">for</span> <span class="n">report</span> <span class="ow">in</span> <span class="n">reports</span><span class="p">:</span>
|
||||
<span class="n">data</span><span class="p">[</span><span class="s2">"sourcetype"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"smtp:tls"</span>
|
||||
<span class="n">timestamp</span> <span class="o">=</span> <span class="n">human_timestamp_to_unix_timestamp</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">])</span>
|
||||
<span class="n">data</span><span class="p">[</span><span class="s2">"time"</span><span class="p">]</span> <span class="o">=</span> <span class="n">timestamp</span>
|
||||
<span class="n">data</span><span class="p">[</span><span class="s2">"event"</span><span class="p">]</span> <span class="o">=</span> <span class="n">report</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
<span class="n">json_str</span> <span class="o">+=</span> <span class="s2">"</span><span class="si">{0}</span><span class="se">\n</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">data</span><span class="p">))</span>
|
||||
<span class="n">json_lines</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">data</span><span class="p">)</span><span class="si">}</span><span class="se">\n</span><span class="s2">"</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">verify</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Skipping certificate verification for Splunk HEC"</span><span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">post</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">url</span><span class="p">,</span> <span class="n">content</span><span class="o">=</span><span class="n">json_str</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">timeout</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">url</span><span class="p">,</span> <span class="n">content</span><span class="o">=</span><span class="s2">""</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">json_lines</span><span class="p">),</span> <span class="n">timeout</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">timeout</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">response</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
|
||||
@@ -5,14 +5,14 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc.types — parsedmarc 10.2.4 documentation</title>
|
||||
<title>parsedmarc.types — parsedmarc 10.3.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=9edc463e" />
|
||||
|
||||
|
||||
<script src="../../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../../_static/documentation_options.js?v=2c8206f1"></script>
|
||||
<script src="../../_static/documentation_options.js?v=a126560f"></script>
|
||||
<script src="../../_static/doctools.js?v=fd6eb6e6"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=6ffebe34"></script>
|
||||
<script src="../../_static/js/theme.js"></script>
|
||||
|
||||
@@ -5,14 +5,14 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc.utils — parsedmarc 10.2.4 documentation</title>
|
||||
<title>parsedmarc.utils — parsedmarc 10.3.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=9edc463e" />
|
||||
|
||||
|
||||
<script src="../../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../../_static/documentation_options.js?v=2c8206f1"></script>
|
||||
<script src="../../_static/documentation_options.js?v=a126560f"></script>
|
||||
<script src="../../_static/doctools.js?v=fd6eb6e6"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=6ffebe34"></script>
|
||||
<script src="../../_static/js/theme.js"></script>
|
||||
@@ -340,7 +340,7 @@
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">domain</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">domain</span><span class="p">)</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
|
||||
<span class="n">record_type</span> <span class="o">=</span> <span class="n">record_type</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span>
|
||||
<span class="n">cache_key</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">_</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">domain</span><span class="p">,</span> <span class="n">record_type</span><span class="p">)</span>
|
||||
<span class="n">cache_key</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">domain</span><span class="si">}</span><span class="s2">_</span><span class="si">{</span><span class="n">record_type</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="k">if</span> <span class="n">cache</span><span class="p">:</span>
|
||||
<span class="n">cached_records</span> <span class="o">=</span> <span class="n">cache</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">cache_key</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">cached_records</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
|
||||
@@ -1227,7 +1227,7 @@
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">mbox</span><span class="o">.</span><span class="n">keys</span><span class="p">())</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="n">_is_mbox</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Error checking for MBOX file: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Error checking for MBOX file: </span><span class="si">{</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">_is_mbox</span></div>
|
||||
|
||||
@@ -1388,7 +1388,7 @@
|
||||
<span class="n">payload</span> <span class="o">=</span> <span class="nb">str</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="n">payload</span><span class="p">)</span>
|
||||
<span class="n">attachment</span><span class="p">[</span><span class="s2">"sha256"</span><span class="p">]</span> <span class="o">=</span> <span class="n">hashlib</span><span class="o">.</span><span class="n">sha256</span><span class="p">(</span><span class="n">payload</span><span class="p">)</span><span class="o">.</span><span class="n">hexdigest</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Unable to decode attachment: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Unable to decode attachment: </span><span class="si">{</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">strip_attachment_payloads</span><span class="p">:</span>
|
||||
<span class="k">for</span> <span class="n">attachment</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">"attachments"</span><span class="p">]:</span>
|
||||
<span class="k">if</span> <span class="s2">"payload"</span> <span class="ow">in</span> <span class="n">attachment</span><span class="p">:</span>
|
||||
|
||||
+108
-10
@@ -235,14 +235,32 @@ result paired, which the dashboards' alignment-detail tables aggregate on.
|
||||
Reports saved by older versions lack these fields and will not appear in
|
||||
those tables.
|
||||
|
||||
Running the following once per cluster backfills the fields on existing
|
||||
documents. It is idempotent (documents that already have the fields are
|
||||
skipped), so it is safe to re-run. It works identically on OpenSearch;
|
||||
just adjust the URL and credentials. The query matches only documents
|
||||
that have at least one DKIM or SPF auth result and lack the corresponding
|
||||
combined field; documents with no auth results are skipped, because an
|
||||
`exists` query cannot see an empty array, and for search purposes an
|
||||
empty `dkim_results_combined` is identical to an absent one.
|
||||
parsedmarc now backfills this automatically. On startup, it runs a cheap
|
||||
count query against each configured aggregate index pattern to check for
|
||||
documents that have DKIM or SPF results but are missing the corresponding
|
||||
combined field. If any are found, it submits the backfill as a background
|
||||
`_update_by_query` task (`wait_for_completion=false`), so startup is never
|
||||
blocked on it; progress is logged, including the task ID. The check itself
|
||||
is idempotent — once an index is fully backfilled, later startups see a
|
||||
count of 0 and log nothing further — and it works the same way on
|
||||
OpenSearch. Any error talking to the cluster (for example, no indexes yet
|
||||
on a fresh install) is logged as a warning and retried on the next startup,
|
||||
rather than aborting parsedmarc.
|
||||
|
||||
If you upgrade the dashboards without pointing the new parsedmarc version
|
||||
at the cluster, or you'd rather control when the write load happens, you
|
||||
can still run the backfill manually. It is idempotent (documents that
|
||||
already have the fields are skipped), so it is safe to re-run. It works
|
||||
identically on OpenSearch; just adjust the URL and credentials. The query
|
||||
matches only documents that have at least one DKIM or SPF auth result and
|
||||
lack the corresponding combined field; documents with no auth results are
|
||||
skipped, because an `exists` query cannot see an empty array, and for
|
||||
search purposes an empty `dkim_results_combined` is identical to an
|
||||
absent one. Each result is matched on either its `domain` or its `result`
|
||||
subfield as defense in depth: an empty string indexes no text tokens and
|
||||
is invisible to `exists`, and the storage shape of every historical
|
||||
parsedmarc version can't be audited, so matching either subfield ensures
|
||||
no backfillable document is skipped.
|
||||
|
||||
```bash
|
||||
curl -X POST "http://localhost:9200/dmarc_aggregate*/_update_by_query?conflicts=proceed&wait_for_completion=false" \
|
||||
@@ -254,13 +272,33 @@ curl -X POST "http://localhost:9200/dmarc_aggregate*/_update_by_query?conflicts=
|
||||
"should": [
|
||||
{
|
||||
"bool": {
|
||||
"must": [{"exists": {"field": "dkim_results.domain"}}],
|
||||
"must": [
|
||||
{
|
||||
"bool": {
|
||||
"minimum_should_match": 1,
|
||||
"should": [
|
||||
{"exists": {"field": "dkim_results.domain"}},
|
||||
{"exists": {"field": "dkim_results.result"}}
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"must_not": [{"exists": {"field": "dkim_results_combined"}}]
|
||||
}
|
||||
},
|
||||
{
|
||||
"bool": {
|
||||
"must": [{"exists": {"field": "spf_results.domain"}}],
|
||||
"must": [
|
||||
{
|
||||
"bool": {
|
||||
"minimum_should_match": 1,
|
||||
"should": [
|
||||
{"exists": {"field": "spf_results.domain"}},
|
||||
{"exists": {"field": "spf_results.result"}}
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"must_not": [{"exists": {"field": "spf_results_combined"}}]
|
||||
}
|
||||
}
|
||||
@@ -280,6 +318,66 @@ curl -X POST "http://localhost:9200/dmarc_aggregate*/_update_by_query?conflicts=
|
||||
dashboards ndjson (the index pattern saved object changed too) per the
|
||||
import instructions above.
|
||||
|
||||
SMTP TLS documents have the same class of defect one level deeper:
|
||||
`policies` is an object array, and each policy's `failure_details` is an
|
||||
object array inside it. SMTP TLS documents now also carry
|
||||
`policies_combined` and `failure_details_combined`, backfilled
|
||||
automatically at startup the same way, and the equivalent manual command
|
||||
is:
|
||||
|
||||
```bash
|
||||
curl -X POST "http://localhost:9200/smtp_tls*/_update_by_query?conflicts=proceed&wait_for_completion=false" \
|
||||
-H "Content-Type: application/json" -d '
|
||||
{
|
||||
"query": {
|
||||
"bool": {
|
||||
"minimum_should_match": 1,
|
||||
"should": [
|
||||
{
|
||||
"bool": {
|
||||
"must": [
|
||||
{
|
||||
"bool": {
|
||||
"minimum_should_match": 1,
|
||||
"should": [
|
||||
{"exists": {"field": "policies.policy_domain"}},
|
||||
{"exists": {"field": "policies.policy_type"}}
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"must_not": [{"exists": {"field": "policies_combined"}}]
|
||||
}
|
||||
},
|
||||
{
|
||||
"bool": {
|
||||
"must": [
|
||||
{
|
||||
"bool": {
|
||||
"minimum_should_match": 1,
|
||||
"should": [
|
||||
{"exists": {"field": "policies.failure_details.result_type"}},
|
||||
{"exists": {"field": "policies.failure_details.sending_mta_ip"}}
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"must_not": [{"exists": {"field": "failure_details_combined"}}]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"script": {
|
||||
"lang": "painless",
|
||||
"source": "List pols = new ArrayList(); List dets = new ArrayList(); def ps = ctx._source.policies; if (ps != null) { if (!(ps instanceof List)) { ps = [ps]; } for (p in ps) { if (p == null) { continue; } def dom = p.policy_domain != null ? p.policy_domain : \"none\"; def typ = p.policy_type != null ? p.policy_type : \"none\"; pols.add(dom + \" / \" + typ); def fds = p.failure_details; if (fds != null) { if (!(fds instanceof List)) { fds = [fds]; } for (f in fds) { if (f == null) { continue; } def rt = f.result_type != null ? f.result_type : \"none\"; def smi = f.sending_mta_ip != null ? f.sending_mta_ip : \"none\"; def ri = f.receiving_ip != null ? f.receiving_ip : \"none\"; def rmh = f.receiving_mx_hostname != null ? f.receiving_mx_hostname : \"none\"; dets.add(dom + \" / \" + typ + \" / \" + rt + \" / \" + smi + \" / \" + ri + \" / \" + rmh); } } } } ctx._source.policies_combined = pols; ctx._source.failure_details_combined = dets;"
|
||||
}
|
||||
}'
|
||||
```
|
||||
|
||||
It works identically on OpenSearch; just adjust the URL and credentials, same
|
||||
as the aggregate command above.
|
||||
|
||||
## Records retention
|
||||
|
||||
Starting in version 5.0.0, `parsedmarc` stores data in a separate
|
||||
|
||||
+25
-1
@@ -86,7 +86,19 @@ table. Each row of the DKIM details table is one real DKIM signature, shown
|
||||
as a combined `selector / domain / result` value; the SPF details table
|
||||
shows `scope / domain / result` the same way. Combining the values into one
|
||||
column keeps each signature's selector, domain, and result paired together,
|
||||
rather than aggregating them as separate columns.
|
||||
rather than aggregating them as separate columns. Because a message that
|
||||
carries multiple DKIM signatures appears once per signature, summing the
|
||||
messages column across rows can exceed the total number of messages.
|
||||
|
||||
The "Auth result filters" panel above the details tables
|
||||
provides dropdowns for the individual auth-result components — DKIM
|
||||
selector, DKIM domain, DKIM result, SPF scope, SPF domain, and SPF
|
||||
result — and filters the whole dashboard by them. Because components from
|
||||
different signatures of the same message are indexed together, combining
|
||||
two of these component filters matches documents where any signature
|
||||
satisfies each condition individually, not necessarily the same signature;
|
||||
the combined `selector / domain / result` (`scope / domain / result`)
|
||||
column remains the per-signature source of truth.
|
||||
|
||||
:::{note}
|
||||
The alignment tables (SPF details, DKIM details) and the per-IP source
|
||||
@@ -116,3 +128,15 @@ reporting organizations, the policy domains they report on, and the
|
||||
specific failure types — certificate expiry, STARTTLS not supported,
|
||||
STS policy fetch errors, validation failures, and similar — together with
|
||||
the sending and receiving MTA addresses involved.
|
||||
|
||||
Like the DKIM and SPF details tables above, the "SMTP TLS domains" and
|
||||
"SMTP TLS failure details" tables show one row per policy and one row per
|
||||
failure detail, respectively, using combined `policy (domain / type)` and
|
||||
`failure detail (domain / type / result / sending mta / receiving ip / mx)`
|
||||
columns so that each policy's or failure detail's fields stay paired
|
||||
together, rather than aggregating them as separate columns. The
|
||||
`successful_sessions` and `failed_sessions` columns are summed per report
|
||||
document, though, not per policy: when a single report carries multiple
|
||||
policies, a row's session sums include the sibling policies from that
|
||||
report as well as its own. Fully attributing session counts to a single
|
||||
policy would require restructuring the stored documents.
|
||||
|
||||
+18
-6
@@ -3,23 +3,25 @@
|
||||
## CLI help
|
||||
|
||||
```text
|
||||
usage: parsedmarc [-h] [-c CONFIG_FILE] [--strip-attachment-payloads] [-o OUTPUT]
|
||||
usage: parsedmarc [-h] [-c CONFIG_FILE] [-r] [--strip-attachment-payloads] [-o OUTPUT]
|
||||
[--aggregate-json-filename AGGREGATE_JSON_FILENAME] [--failure-json-filename FAILURE_JSON_FILENAME]
|
||||
[--smtp-tls-json-filename SMTP_TLS_JSON_FILENAME] [--aggregate-csv-filename AGGREGATE_CSV_FILENAME]
|
||||
[--failure-csv-filename FAILURE_CSV_FILENAME] [--smtp-tls-csv-filename SMTP_TLS_CSV_FILENAME]
|
||||
[-n NAMESERVERS [NAMESERVERS ...]] [-t DNS_TIMEOUT] [--offline] [-s] [-w] [--verbose] [--debug]
|
||||
[--log-file LOG_FILE] [--no-prettify-json] [-v]
|
||||
[-n NAMESERVERS [NAMESERVERS ...]] [-t DNS_TIMEOUT] [--dns-retries DNS_RETRIES] [--offline] [-s]
|
||||
[-w] [--verbose] [--debug] [--log-file LOG_FILE] [--no-prettify-json] [-v]
|
||||
[file_path ...]
|
||||
|
||||
Parses DMARC reports
|
||||
|
||||
positional arguments:
|
||||
file_path one or more paths to aggregate or failure report files, emails, or mbox files'
|
||||
file_path one or more paths to aggregate or failure report files, emails, mbox files, or directories
|
||||
containing them
|
||||
|
||||
options:
|
||||
-h, --help show this help message and exit
|
||||
-c CONFIG_FILE, --config-file CONFIG_FILE
|
||||
a path to a configuration file (--silent implied)
|
||||
-r, --recursive search directories given as file_path recursively, and enable '**' recursion in glob patterns
|
||||
--strip-attachment-payloads
|
||||
remove attachment payloads from failure report output
|
||||
-o OUTPUT, --output OUTPUT
|
||||
@@ -40,6 +42,8 @@ options:
|
||||
nameservers to query
|
||||
-t DNS_TIMEOUT, --dns_timeout DNS_TIMEOUT
|
||||
number of seconds to wait for an answer from DNS (default: 2.0)
|
||||
--dns-retries DNS_RETRIES
|
||||
number of times to retry DNS queries on timeout or other transient errors (default: 0)
|
||||
--offline do not make online queries for geolocation or DNS
|
||||
-s, --silent only print errors
|
||||
-w, --warnings print warnings in addition to errors
|
||||
@@ -165,14 +169,22 @@ The full set of configuration options are:
|
||||
any configured output destination fails while saving/publishing
|
||||
reports (Default: `False`)
|
||||
- `log_file` - str: Write log messages to a file at this path
|
||||
- `n_procs` - int: Number of process to run in parallel when
|
||||
parsing in CLI mode (Default: `1`)
|
||||
- `n_procs` - int: Number of processes to run in parallel when
|
||||
parsing report files passed directly as CLI arguments
|
||||
(Default: `1`)
|
||||
|
||||
:::{note}
|
||||
Setting this to a number larger than one can improve
|
||||
performance when processing thousands of files
|
||||
:::
|
||||
|
||||
:::{note}
|
||||
`n_procs` only applies to report files passed directly on the
|
||||
command line. Messages from mbox files and from mailbox
|
||||
connections (IMAP, Microsoft Graph, Gmail API, Maildir) are
|
||||
always processed sequentially.
|
||||
:::
|
||||
|
||||
- `mailbox`
|
||||
- `reports_folder` - str: The mailbox folder (or label for
|
||||
Gmail) where the incoming reports can be found
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
const DOCUMENTATION_OPTIONS = {
|
||||
VERSION: '10.2.4',
|
||||
VERSION: '10.3.0',
|
||||
LANGUAGE: 'en',
|
||||
COLLAPSE_INDEX: false,
|
||||
BUILDER: 'html',
|
||||
|
||||
@@ -6,14 +6,14 @@
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>API reference — parsedmarc 10.2.4 documentation</title>
|
||||
<title>API reference — parsedmarc 10.3.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=9edc463e" />
|
||||
|
||||
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=2c8206f1"></script>
|
||||
<script src="_static/documentation_options.js?v=a126560f"></script>
|
||||
<script src="_static/doctools.js?v=fd6eb6e6"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=6ffebe34"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
@@ -872,23 +872,30 @@ remaining keys are passed through; defaults are skipped entirely.</p></li>
|
||||
|
||||
<dl class="py function">
|
||||
<dt class="sig sig-object py" id="parsedmarc.elastic.migrate_indexes">
|
||||
<span class="sig-prename descclassname"><span class="pre">parsedmarc.elastic.</span></span><span class="sig-name descname"><span class="pre">migrate_indexes</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">aggregate_indexes</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">failure_indexes</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc/elastic.html#migrate_indexes"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#parsedmarc.elastic.migrate_indexes" title="Link to this definition"></a></dt>
|
||||
<dd><p>Updates index mappings</p>
|
||||
<p>This is a no-op kept for API compatibility (<code class="docutils literal notranslate"><span class="pre">cli.py</span></code> calls it on
|
||||
startup). The only migration this function ever performed was
|
||||
re-typing <code class="docutils literal notranslate"><span class="pre">published_policy.fo</span></code> from <code class="docutils literal notranslate"><span class="pre">long</span></code> to <code class="docutils literal notranslate"><span class="pre">text</span></code>, which
|
||||
applied exclusively to indices still carrying the legacy
|
||||
Elasticsearch 6-era <code class="docutils literal notranslate"><span class="pre">"doc"</span></code> mapping type. The 8.x client can only
|
||||
reach servers (Elasticsearch 8.x/9.x) whose indices were created on
|
||||
Elasticsearch 7.x or later and are therefore typeless, so that
|
||||
migration path is unreachable and has been removed.</p>
|
||||
<span class="sig-prename descclassname"><span class="pre">parsedmarc.elastic.</span></span><span class="sig-name descname"><span class="pre">migrate_indexes</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">aggregate_indexes</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">failure_indexes</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">smtp_tls_indexes</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc/elastic.html#migrate_indexes"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#parsedmarc.elastic.migrate_indexes" title="Link to this definition"></a></dt>
|
||||
<dd><p>Backfills the <code class="docutils literal notranslate"><span class="pre">dkim_results_combined</span></code>/<code class="docutils literal notranslate"><span class="pre">spf_results_combined</span></code> fields
|
||||
(added for issue #169) on aggregate report documents, and the
|
||||
<code class="docutils literal notranslate"><span class="pre">policies_combined</span></code>/<code class="docutils literal notranslate"><span class="pre">failure_details_combined</span></code> fields on SMTP TLS
|
||||
report documents, that were saved before those fields existed.</p>
|
||||
<p>For each name in <code class="docutils literal notranslate"><span class="pre">aggregate_indexes</span></code>/<code class="docutils literal notranslate"><span class="pre">smtp_tls_indexes</span></code>, this
|
||||
submits an <code class="docutils literal notranslate"><span class="pre">update_by_query</span></code> against the <code class="docutils literal notranslate"><span class="pre">f"{name}*"</span></code> index pattern
|
||||
(the real indexes are date-suffixed) as a non-blocking background task
|
||||
(<code class="docutils literal notranslate"><span class="pre">wait_for_completion=False</span></code>), so it never delays parsedmarc startup.
|
||||
Submission is guarded by a cheap <code class="docutils literal notranslate"><span class="pre">count</span></code> query that only matches
|
||||
documents with DKIM/SPF results (or policies/failure details) but no
|
||||
combined field, so once an index is fully backfilled, later calls are a
|
||||
fast no-op. Any error talking to the cluster (e.g. no indexes yet on a
|
||||
fresh install, or a transient connection issue) is caught and logged as
|
||||
a warning rather than raised; the backfill is simply retried on the
|
||||
next startup, and the manual <code class="docutils literal notranslate"><span class="pre">_update_by_query</span></code> commands documented in
|
||||
<code class="docutils literal notranslate"><span class="pre">docs/source/elasticsearch.md</span></code> remain available in the meantime.</p>
|
||||
<dl class="field-list simple">
|
||||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||||
<dd class="field-odd"><ul class="simple">
|
||||
<li><p><strong>aggregate_indexes</strong> (<em>list</em>) – A list of aggregate index names
|
||||
(accepted for API compatibility; unused)</p></li>
|
||||
<li><p><strong>aggregate_indexes</strong> (<em>list</em>) – A list of aggregate index names</p></li>
|
||||
<li><p><strong>failure_indexes</strong> (<em>list</em>) – A list of failure index names
|
||||
(accepted for API compatibility; unused)</p></li>
|
||||
<li><p><strong>smtp_tls_indexes</strong> (<em>list</em>) – A list of SMTP TLS index names</p></li>
|
||||
</ul>
|
||||
</dd>
|
||||
</dl>
|
||||
@@ -1039,8 +1046,29 @@ any other settings through unchanged.</p></li>
|
||||
|
||||
<dl class="py function">
|
||||
<dt class="sig sig-object py" id="parsedmarc.opensearch.migrate_indexes">
|
||||
<span class="sig-prename descclassname"><span class="pre">parsedmarc.opensearch.</span></span><span class="sig-name descname"><span class="pre">migrate_indexes</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">aggregate_indexes</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">failure_indexes</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc/opensearch.html#migrate_indexes"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#parsedmarc.opensearch.migrate_indexes" title="Link to this definition"></a></dt>
|
||||
<dd><p>Updates index mappings</p>
|
||||
<span class="sig-prename descclassname"><span class="pre">parsedmarc.opensearch.</span></span><span class="sig-name descname"><span class="pre">migrate_indexes</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">aggregate_indexes</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">failure_indexes</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">smtp_tls_indexes</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc/opensearch.html#migrate_indexes"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#parsedmarc.opensearch.migrate_indexes" title="Link to this definition"></a></dt>
|
||||
<dd><p>Runs index migrations and backfills.</p>
|
||||
<p>First, the legacy <code class="docutils literal notranslate"><span class="pre">published_policy.fo</span></code> migration: indexes where that
|
||||
field was mapped as <code class="docutils literal notranslate"><span class="pre">long</span></code> (data indexed by very old parsedmarc
|
||||
releases under the Elasticsearch 6-era <code class="docutils literal notranslate"><span class="pre">doc</span></code> mapping type) are rebuilt
|
||||
as a <code class="docutils literal notranslate"><span class="pre">-v2</span></code> index with the text/keyword shape.</p>
|
||||
<p>Second, the <code class="docutils literal notranslate"><span class="pre">dkim_results_combined</span></code>/<code class="docutils literal notranslate"><span class="pre">spf_results_combined</span></code> backfill
|
||||
(added for issue #169) for aggregate report documents that were saved
|
||||
before those fields existed. For each name in <code class="docutils literal notranslate"><span class="pre">aggregate_indexes</span></code>,
|
||||
this submits an <code class="docutils literal notranslate"><span class="pre">update_by_query</span></code> against the <code class="docutils literal notranslate"><span class="pre">f"{name}*"</span></code> index
|
||||
pattern (the real indexes are date-suffixed) as a non-blocking
|
||||
background task (<code class="docutils literal notranslate"><span class="pre">wait_for_completion=False</span></code>), so it never delays
|
||||
parsedmarc startup. Submission is guarded by a cheap <code class="docutils literal notranslate"><span class="pre">count</span></code> query
|
||||
that only matches documents with DKIM/SPF results but no combined
|
||||
field, so once an index is fully backfilled, later calls are a fast
|
||||
no-op. Any error talking to the cluster (e.g. no indexes yet on a
|
||||
fresh install, or a transient connection issue) is caught and logged
|
||||
as a warning rather than raised; the backfill is simply retried on the
|
||||
next startup, and the manual <code class="docutils literal notranslate"><span class="pre">_update_by_query</span></code> command documented
|
||||
in <code class="docutils literal notranslate"><span class="pre">docs/source/elasticsearch.md</span></code> remains available in the meantime.</p>
|
||||
<p>Third, the same treatment for the <code class="docutils literal notranslate"><span class="pre">policies_combined</span></code>/
|
||||
<code class="docutils literal notranslate"><span class="pre">failure_details_combined</span></code> fields (same issue #169) on SMTP TLS
|
||||
report documents, for each name in <code class="docutils literal notranslate"><span class="pre">smtp_tls_indexes</span></code>.</p>
|
||||
<dl class="field-list simple">
|
||||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||||
<dd class="field-odd"><ul class="simple">
|
||||
@@ -1048,6 +1076,7 @@ any other settings through unchanged.</p></li>
|
||||
<li><p><strong>failure_indexes</strong> (<em>list</em>) – A list of failure index names
|
||||
(accepted for API compatibility; no migrations are
|
||||
currently needed for failure indexes)</p></li>
|
||||
<li><p><strong>smtp_tls_indexes</strong> (<em>list</em>) – A list of SMTP TLS index names</p></li>
|
||||
</ul>
|
||||
</dd>
|
||||
</dl>
|
||||
|
||||
+2
-2
@@ -6,14 +6,14 @@
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Contributing to parsedmarc — parsedmarc 10.2.4 documentation</title>
|
||||
<title>Contributing to parsedmarc — parsedmarc 10.3.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=9edc463e" />
|
||||
|
||||
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=2c8206f1"></script>
|
||||
<script src="_static/documentation_options.js?v=a126560f"></script>
|
||||
<script src="_static/doctools.js?v=fd6eb6e6"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=6ffebe34"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
|
||||
+2
-2
@@ -6,14 +6,14 @@
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Accessing an inbox using OWA/EWS — parsedmarc 10.2.4 documentation</title>
|
||||
<title>Accessing an inbox using OWA/EWS — parsedmarc 10.3.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=9edc463e" />
|
||||
|
||||
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=2c8206f1"></script>
|
||||
<script src="_static/documentation_options.js?v=a126560f"></script>
|
||||
<script src="_static/doctools.js?v=fd6eb6e6"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=6ffebe34"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
|
||||
+2
-2
@@ -6,14 +6,14 @@
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Understanding DMARC — parsedmarc 10.2.4 documentation</title>
|
||||
<title>Understanding DMARC — parsedmarc 10.3.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=9edc463e" />
|
||||
|
||||
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=2c8206f1"></script>
|
||||
<script src="_static/documentation_options.js?v=a126560f"></script>
|
||||
<script src="_static/doctools.js?v=fd6eb6e6"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=6ffebe34"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
|
||||
+106
-12
@@ -6,14 +6,14 @@
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Elasticsearch and Kibana — parsedmarc 10.2.4 documentation</title>
|
||||
<title>Elasticsearch and Kibana — parsedmarc 10.3.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=9edc463e" />
|
||||
|
||||
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=2c8206f1"></script>
|
||||
<script src="_static/documentation_options.js?v=a126560f"></script>
|
||||
<script src="_static/doctools.js?v=fd6eb6e6"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=6ffebe34"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
@@ -273,14 +273,31 @@ scalar string arrays that keep each auth result’s selector/scope, domain, and
|
||||
result paired, which the dashboards’ alignment-detail tables aggregate on.
|
||||
Reports saved by older versions lack these fields and will not appear in
|
||||
those tables.</p>
|
||||
<p>Running the following once per cluster backfills the fields on existing
|
||||
documents. It is idempotent (documents that already have the fields are
|
||||
skipped), so it is safe to re-run. It works identically on OpenSearch;
|
||||
just adjust the URL and credentials. The query matches only documents
|
||||
that have at least one DKIM or SPF auth result and lack the corresponding
|
||||
combined field; documents with no auth results are skipped, because an
|
||||
<code class="docutils literal notranslate"><span class="pre">exists</span></code> query cannot see an empty array, and for search purposes an
|
||||
empty <code class="docutils literal notranslate"><span class="pre">dkim_results_combined</span></code> is identical to an absent one.</p>
|
||||
<p>parsedmarc now backfills this automatically. On startup, it runs a cheap
|
||||
count query against each configured aggregate index pattern to check for
|
||||
documents that have DKIM or SPF results but are missing the corresponding
|
||||
combined field. If any are found, it submits the backfill as a background
|
||||
<code class="docutils literal notranslate"><span class="pre">_update_by_query</span></code> task (<code class="docutils literal notranslate"><span class="pre">wait_for_completion=false</span></code>), so startup is never
|
||||
blocked on it; progress is logged, including the task ID. The check itself
|
||||
is idempotent — once an index is fully backfilled, later startups see a
|
||||
count of 0 and log nothing further — and it works the same way on
|
||||
OpenSearch. Any error talking to the cluster (for example, no indexes yet
|
||||
on a fresh install) is logged as a warning and retried on the next startup,
|
||||
rather than aborting parsedmarc.</p>
|
||||
<p>If you upgrade the dashboards without pointing the new parsedmarc version
|
||||
at the cluster, or you’d rather control when the write load happens, you
|
||||
can still run the backfill manually. It is idempotent (documents that
|
||||
already have the fields are skipped), so it is safe to re-run. It works
|
||||
identically on OpenSearch; just adjust the URL and credentials. The query
|
||||
matches only documents that have at least one DKIM or SPF auth result and
|
||||
lack the corresponding combined field; documents with no auth results are
|
||||
skipped, because an <code class="docutils literal notranslate"><span class="pre">exists</span></code> query cannot see an empty array, and for
|
||||
search purposes an empty <code class="docutils literal notranslate"><span class="pre">dkim_results_combined</span></code> is identical to an
|
||||
absent one. Each result is matched on either its <code class="docutils literal notranslate"><span class="pre">domain</span></code> or its <code class="docutils literal notranslate"><span class="pre">result</span></code>
|
||||
subfield as defense in depth: an empty string indexes no text tokens and
|
||||
is invisible to <code class="docutils literal notranslate"><span class="pre">exists</span></code>, and the storage shape of every historical
|
||||
parsedmarc version can’t be audited, so matching either subfield ensures
|
||||
no backfillable document is skipped.</p>
|
||||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>curl<span class="w"> </span>-X<span class="w"> </span>POST<span class="w"> </span><span class="s2">"http://localhost:9200/dmarc_aggregate*/_update_by_query?conflicts=proceed&wait_for_completion=false"</span><span class="w"> </span><span class="se">\</span>
|
||||
<span class="w"> </span>-H<span class="w"> </span><span class="s2">"Content-Type: application/json"</span><span class="w"> </span>-d<span class="w"> </span><span class="s1">'</span>
|
||||
<span class="s1">{</span>
|
||||
@@ -290,13 +307,33 @@ empty <code class="docutils literal notranslate"><span class="pre">dkim_results_
|
||||
<span class="s1"> "should": [</span>
|
||||
<span class="s1"> {</span>
|
||||
<span class="s1"> "bool": {</span>
|
||||
<span class="s1"> "must": [{"exists": {"field": "dkim_results.domain"}}],</span>
|
||||
<span class="s1"> "must": [</span>
|
||||
<span class="s1"> {</span>
|
||||
<span class="s1"> "bool": {</span>
|
||||
<span class="s1"> "minimum_should_match": 1,</span>
|
||||
<span class="s1"> "should": [</span>
|
||||
<span class="s1"> {"exists": {"field": "dkim_results.domain"}},</span>
|
||||
<span class="s1"> {"exists": {"field": "dkim_results.result"}}</span>
|
||||
<span class="s1"> ]</span>
|
||||
<span class="s1"> }</span>
|
||||
<span class="s1"> }</span>
|
||||
<span class="s1"> ],</span>
|
||||
<span class="s1"> "must_not": [{"exists": {"field": "dkim_results_combined"}}]</span>
|
||||
<span class="s1"> }</span>
|
||||
<span class="s1"> },</span>
|
||||
<span class="s1"> {</span>
|
||||
<span class="s1"> "bool": {</span>
|
||||
<span class="s1"> "must": [{"exists": {"field": "spf_results.domain"}}],</span>
|
||||
<span class="s1"> "must": [</span>
|
||||
<span class="s1"> {</span>
|
||||
<span class="s1"> "bool": {</span>
|
||||
<span class="s1"> "minimum_should_match": 1,</span>
|
||||
<span class="s1"> "should": [</span>
|
||||
<span class="s1"> {"exists": {"field": "spf_results.domain"}},</span>
|
||||
<span class="s1"> {"exists": {"field": "spf_results.result"}}</span>
|
||||
<span class="s1"> ]</span>
|
||||
<span class="s1"> }</span>
|
||||
<span class="s1"> }</span>
|
||||
<span class="s1"> ],</span>
|
||||
<span class="s1"> "must_not": [{"exists": {"field": "spf_results_combined"}}]</span>
|
||||
<span class="s1"> }</span>
|
||||
<span class="s1"> }</span>
|
||||
@@ -315,6 +352,63 @@ empty <code class="docutils literal notranslate"><span class="pre">dkim_results_
|
||||
<code class="docutils literal notranslate"><span class="pre">index_prefix</span></code>/<code class="docutils literal notranslate"><span class="pre">index_suffix</span></code>. After backfilling, re-import the updated
|
||||
dashboards ndjson (the index pattern saved object changed too) per the
|
||||
import instructions above.</p>
|
||||
<p>SMTP TLS documents have the same class of defect one level deeper:
|
||||
<code class="docutils literal notranslate"><span class="pre">policies</span></code> is an object array, and each policy’s <code class="docutils literal notranslate"><span class="pre">failure_details</span></code> is an
|
||||
object array inside it. SMTP TLS documents now also carry
|
||||
<code class="docutils literal notranslate"><span class="pre">policies_combined</span></code> and <code class="docutils literal notranslate"><span class="pre">failure_details_combined</span></code>, backfilled
|
||||
automatically at startup the same way, and the equivalent manual command
|
||||
is:</p>
|
||||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>curl<span class="w"> </span>-X<span class="w"> </span>POST<span class="w"> </span><span class="s2">"http://localhost:9200/smtp_tls*/_update_by_query?conflicts=proceed&wait_for_completion=false"</span><span class="w"> </span><span class="se">\</span>
|
||||
<span class="w"> </span>-H<span class="w"> </span><span class="s2">"Content-Type: application/json"</span><span class="w"> </span>-d<span class="w"> </span><span class="s1">'</span>
|
||||
<span class="s1">{</span>
|
||||
<span class="s1"> "query": {</span>
|
||||
<span class="s1"> "bool": {</span>
|
||||
<span class="s1"> "minimum_should_match": 1,</span>
|
||||
<span class="s1"> "should": [</span>
|
||||
<span class="s1"> {</span>
|
||||
<span class="s1"> "bool": {</span>
|
||||
<span class="s1"> "must": [</span>
|
||||
<span class="s1"> {</span>
|
||||
<span class="s1"> "bool": {</span>
|
||||
<span class="s1"> "minimum_should_match": 1,</span>
|
||||
<span class="s1"> "should": [</span>
|
||||
<span class="s1"> {"exists": {"field": "policies.policy_domain"}},</span>
|
||||
<span class="s1"> {"exists": {"field": "policies.policy_type"}}</span>
|
||||
<span class="s1"> ]</span>
|
||||
<span class="s1"> }</span>
|
||||
<span class="s1"> }</span>
|
||||
<span class="s1"> ],</span>
|
||||
<span class="s1"> "must_not": [{"exists": {"field": "policies_combined"}}]</span>
|
||||
<span class="s1"> }</span>
|
||||
<span class="s1"> },</span>
|
||||
<span class="s1"> {</span>
|
||||
<span class="s1"> "bool": {</span>
|
||||
<span class="s1"> "must": [</span>
|
||||
<span class="s1"> {</span>
|
||||
<span class="s1"> "bool": {</span>
|
||||
<span class="s1"> "minimum_should_match": 1,</span>
|
||||
<span class="s1"> "should": [</span>
|
||||
<span class="s1"> {"exists": {"field": "policies.failure_details.result_type"}},</span>
|
||||
<span class="s1"> {"exists": {"field": "policies.failure_details.sending_mta_ip"}}</span>
|
||||
<span class="s1"> ]</span>
|
||||
<span class="s1"> }</span>
|
||||
<span class="s1"> }</span>
|
||||
<span class="s1"> ],</span>
|
||||
<span class="s1"> "must_not": [{"exists": {"field": "failure_details_combined"}}]</span>
|
||||
<span class="s1"> }</span>
|
||||
<span class="s1"> }</span>
|
||||
<span class="s1"> ]</span>
|
||||
<span class="s1"> }</span>
|
||||
<span class="s1"> },</span>
|
||||
<span class="s1"> "script": {</span>
|
||||
<span class="s1"> "lang": "painless",</span>
|
||||
<span class="s1"> "source": "List pols = new ArrayList(); List dets = new ArrayList(); def ps = ctx._source.policies; if (ps != null) { if (!(ps instanceof List)) { ps = [ps]; } for (p in ps) { if (p == null) { continue; } def dom = p.policy_domain != null ? p.policy_domain : \"none\"; def typ = p.policy_type != null ? p.policy_type : \"none\"; pols.add(dom + \" / \" + typ); def fds = p.failure_details; if (fds != null) { if (!(fds instanceof List)) { fds = [fds]; } for (f in fds) { if (f == null) { continue; } def rt = f.result_type != null ? f.result_type : \"none\"; def smi = f.sending_mta_ip != null ? f.sending_mta_ip : \"none\"; def ri = f.receiving_ip != null ? f.receiving_ip : \"none\"; def rmh = f.receiving_mx_hostname != null ? f.receiving_mx_hostname : \"none\"; dets.add(dom + \" / \" + typ + \" / \" + rt + \" / \" + smi + \" / \" + ri + \" / \" + rmh); } } } } ctx._source.policies_combined = pols; ctx._source.failure_details_combined = dets;"</span>
|
||||
<span class="s1"> }</span>
|
||||
<span class="s1">}'</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
<p>It works identically on OpenSearch; just adjust the URL and credentials, same
|
||||
as the aggregate command above.</p>
|
||||
</section>
|
||||
<section id="records-retention">
|
||||
<h2>Records retention<a class="headerlink" href="#records-retention" title="Link to this heading"></a></h2>
|
||||
|
||||
+2
-2
@@ -5,14 +5,14 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Index — parsedmarc 10.2.4 documentation</title>
|
||||
<title>Index — parsedmarc 10.3.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=9edc463e" />
|
||||
|
||||
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=2c8206f1"></script>
|
||||
<script src="_static/documentation_options.js?v=a126560f"></script>
|
||||
<script src="_static/doctools.js?v=fd6eb6e6"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=6ffebe34"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
|
||||
+2
-2
@@ -6,14 +6,14 @@
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc documentation - Open source DMARC report analyzer and visualizer — parsedmarc 10.2.4 documentation</title>
|
||||
<title>parsedmarc documentation - Open source DMARC report analyzer and visualizer — parsedmarc 10.3.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=9edc463e" />
|
||||
|
||||
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=2c8206f1"></script>
|
||||
<script src="_static/documentation_options.js?v=a126560f"></script>
|
||||
<script src="_static/doctools.js?v=fd6eb6e6"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=6ffebe34"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
|
||||
+2
-2
@@ -6,14 +6,14 @@
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Installation — parsedmarc 10.2.4 documentation</title>
|
||||
<title>Installation — parsedmarc 10.3.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=9edc463e" />
|
||||
|
||||
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=2c8206f1"></script>
|
||||
<script src="_static/documentation_options.js?v=a126560f"></script>
|
||||
<script src="_static/doctools.js?v=fd6eb6e6"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=6ffebe34"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
|
||||
+25
-3
@@ -6,14 +6,14 @@
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Using the Kibana dashboards — parsedmarc 10.2.4 documentation</title>
|
||||
<title>Using the Kibana dashboards — parsedmarc 10.3.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=9edc463e" />
|
||||
|
||||
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=2c8206f1"></script>
|
||||
<script src="_static/documentation_options.js?v=a126560f"></script>
|
||||
<script src="_static/doctools.js?v=fd6eb6e6"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=6ffebe34"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
@@ -164,7 +164,18 @@ table. Each row of the DKIM details table is one real DKIM signature, shown
|
||||
as a combined <code class="docutils literal notranslate"><span class="pre">selector</span> <span class="pre">/</span> <span class="pre">domain</span> <span class="pre">/</span> <span class="pre">result</span></code> value; the SPF details table
|
||||
shows <code class="docutils literal notranslate"><span class="pre">scope</span> <span class="pre">/</span> <span class="pre">domain</span> <span class="pre">/</span> <span class="pre">result</span></code> the same way. Combining the values into one
|
||||
column keeps each signature’s selector, domain, and result paired together,
|
||||
rather than aggregating them as separate columns.</p>
|
||||
rather than aggregating them as separate columns. Because a message that
|
||||
carries multiple DKIM signatures appears once per signature, summing the
|
||||
messages column across rows can exceed the total number of messages.</p>
|
||||
<p>The “Auth result filters” panel above the details tables
|
||||
provides dropdowns for the individual auth-result components — DKIM
|
||||
selector, DKIM domain, DKIM result, SPF scope, SPF domain, and SPF
|
||||
result — and filters the whole dashboard by them. Because components from
|
||||
different signatures of the same message are indexed together, combining
|
||||
two of these component filters matches documents where any signature
|
||||
satisfies each condition individually, not necessarily the same signature;
|
||||
the combined <code class="docutils literal notranslate"><span class="pre">selector</span> <span class="pre">/</span> <span class="pre">domain</span> <span class="pre">/</span> <span class="pre">result</span></code> (<code class="docutils literal notranslate"><span class="pre">scope</span> <span class="pre">/</span> <span class="pre">domain</span> <span class="pre">/</span> <span class="pre">result</span></code>)
|
||||
column remains the per-signature source of truth.</p>
|
||||
<div class="admonition note">
|
||||
<p class="admonition-title">Note</p>
|
||||
<p>The alignment tables (SPF details, DKIM details) and the per-IP source
|
||||
@@ -193,6 +204,17 @@ reporting organizations, the policy domains they report on, and the
|
||||
specific failure types — certificate expiry, STARTTLS not supported,
|
||||
STS policy fetch errors, validation failures, and similar — together with
|
||||
the sending and receiving MTA addresses involved.</p>
|
||||
<p>Like the DKIM and SPF details tables above, the “SMTP TLS domains” and
|
||||
“SMTP TLS failure details” tables show one row per policy and one row per
|
||||
failure detail, respectively, using combined <code class="docutils literal notranslate"><span class="pre">policy</span> <span class="pre">(domain</span> <span class="pre">/</span> <span class="pre">type)</span></code> and
|
||||
<code class="docutils literal notranslate"><span class="pre">failure</span> <span class="pre">detail</span> <span class="pre">(domain</span> <span class="pre">/</span> <span class="pre">type</span> <span class="pre">/</span> <span class="pre">result</span> <span class="pre">/</span> <span class="pre">sending</span> <span class="pre">mta</span> <span class="pre">/</span> <span class="pre">receiving</span> <span class="pre">ip</span> <span class="pre">/</span> <span class="pre">mx)</span></code>
|
||||
columns so that each policy’s or failure detail’s fields stay paired
|
||||
together, rather than aggregating them as separate columns. The
|
||||
<code class="docutils literal notranslate"><span class="pre">successful_sessions</span></code> and <code class="docutils literal notranslate"><span class="pre">failed_sessions</span></code> columns are summed per report
|
||||
document, though, not per policy: when a single report carries multiple
|
||||
policies, a row’s session sums include the sibling policies from that
|
||||
report as well as its own. Fully attributing session counts to a single
|
||||
policy would require restructuring the stored documents.</p>
|
||||
</section>
|
||||
</section>
|
||||
|
||||
|
||||
+2
-2
@@ -6,14 +6,14 @@
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>What about mailing lists? — parsedmarc 10.2.4 documentation</title>
|
||||
<title>What about mailing lists? — parsedmarc 10.3.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=9edc463e" />
|
||||
|
||||
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=2c8206f1"></script>
|
||||
<script src="_static/documentation_options.js?v=a126560f"></script>
|
||||
<script src="_static/doctools.js?v=fd6eb6e6"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=6ffebe34"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
|
||||
BIN
Binary file not shown.
+2
-2
@@ -6,14 +6,14 @@
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>OpenSearch and Grafana — parsedmarc 10.2.4 documentation</title>
|
||||
<title>OpenSearch and Grafana — parsedmarc 10.3.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=9edc463e" />
|
||||
|
||||
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=2c8206f1"></script>
|
||||
<script src="_static/documentation_options.js?v=a126560f"></script>
|
||||
<script src="_static/doctools.js?v=fd6eb6e6"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=6ffebe34"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
|
||||
+2
-2
@@ -6,14 +6,14 @@
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Sample outputs — parsedmarc 10.2.4 documentation</title>
|
||||
<title>Sample outputs — parsedmarc 10.3.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=9edc463e" />
|
||||
|
||||
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=2c8206f1"></script>
|
||||
<script src="_static/documentation_options.js?v=a126560f"></script>
|
||||
<script src="_static/doctools.js?v=fd6eb6e6"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=6ffebe34"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
|
||||
+2
-2
@@ -5,14 +5,14 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Python Module Index — parsedmarc 10.2.4 documentation</title>
|
||||
<title>Python Module Index — parsedmarc 10.3.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=9edc463e" />
|
||||
|
||||
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=2c8206f1"></script>
|
||||
<script src="_static/documentation_options.js?v=a126560f"></script>
|
||||
<script src="_static/doctools.js?v=fd6eb6e6"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=6ffebe34"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
|
||||
+2
-2
@@ -5,7 +5,7 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Search — parsedmarc 10.2.4 documentation</title>
|
||||
<title>Search — parsedmarc 10.3.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=9edc463e" />
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=2c8206f1"></script>
|
||||
<script src="_static/documentation_options.js?v=a126560f"></script>
|
||||
<script src="_static/doctools.js?v=fd6eb6e6"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=6ffebe34"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
|
||||
+1
-1
File diff suppressed because one or more lines are too long
+2
-2
@@ -6,14 +6,14 @@
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Splunk — parsedmarc 10.2.4 documentation</title>
|
||||
<title>Splunk — parsedmarc 10.3.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=9edc463e" />
|
||||
|
||||
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=2c8206f1"></script>
|
||||
<script src="_static/documentation_options.js?v=a126560f"></script>
|
||||
<script src="_static/doctools.js?v=fd6eb6e6"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=6ffebe34"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
|
||||
+20
-8
@@ -6,14 +6,14 @@
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Using parsedmarc — parsedmarc 10.2.4 documentation</title>
|
||||
<title>Using parsedmarc — parsedmarc 10.3.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=9edc463e" />
|
||||
|
||||
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=2c8206f1"></script>
|
||||
<script src="_static/documentation_options.js?v=a126560f"></script>
|
||||
<script src="_static/doctools.js?v=fd6eb6e6"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=6ffebe34"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
@@ -104,23 +104,25 @@
|
||||
<h1>Using parsedmarc<a class="headerlink" href="#using-parsedmarc" title="Link to this heading"></a></h1>
|
||||
<section id="cli-help">
|
||||
<h2>CLI help<a class="headerlink" href="#cli-help" title="Link to this heading"></a></h2>
|
||||
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span>usage: parsedmarc [-h] [-c CONFIG_FILE] [--strip-attachment-payloads] [-o OUTPUT]
|
||||
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span>usage: parsedmarc [-h] [-c CONFIG_FILE] [-r] [--strip-attachment-payloads] [-o OUTPUT]
|
||||
[--aggregate-json-filename AGGREGATE_JSON_FILENAME] [--failure-json-filename FAILURE_JSON_FILENAME]
|
||||
[--smtp-tls-json-filename SMTP_TLS_JSON_FILENAME] [--aggregate-csv-filename AGGREGATE_CSV_FILENAME]
|
||||
[--failure-csv-filename FAILURE_CSV_FILENAME] [--smtp-tls-csv-filename SMTP_TLS_CSV_FILENAME]
|
||||
[-n NAMESERVERS [NAMESERVERS ...]] [-t DNS_TIMEOUT] [--offline] [-s] [-w] [--verbose] [--debug]
|
||||
[--log-file LOG_FILE] [--no-prettify-json] [-v]
|
||||
[-n NAMESERVERS [NAMESERVERS ...]] [-t DNS_TIMEOUT] [--dns-retries DNS_RETRIES] [--offline] [-s]
|
||||
[-w] [--verbose] [--debug] [--log-file LOG_FILE] [--no-prettify-json] [-v]
|
||||
[file_path ...]
|
||||
|
||||
Parses DMARC reports
|
||||
|
||||
positional arguments:
|
||||
file_path one or more paths to aggregate or failure report files, emails, or mbox files'
|
||||
file_path one or more paths to aggregate or failure report files, emails, mbox files, or directories
|
||||
containing them
|
||||
|
||||
options:
|
||||
-h, --help show this help message and exit
|
||||
-c CONFIG_FILE, --config-file CONFIG_FILE
|
||||
a path to a configuration file (--silent implied)
|
||||
-r, --recursive search directories given as file_path recursively, and enable '**' recursion in glob patterns
|
||||
--strip-attachment-payloads
|
||||
remove attachment payloads from failure report output
|
||||
-o OUTPUT, --output OUTPUT
|
||||
@@ -141,6 +143,8 @@ options:
|
||||
nameservers to query
|
||||
-t DNS_TIMEOUT, --dns_timeout DNS_TIMEOUT
|
||||
number of seconds to wait for an answer from DNS (default: 2.0)
|
||||
--dns-retries DNS_RETRIES
|
||||
number of times to retry DNS queries on timeout or other transient errors (default: 0)
|
||||
--offline do not make online queries for geolocation or DNS
|
||||
-s, --silent only print errors
|
||||
-w, --warnings print warnings in addition to errors
|
||||
@@ -264,13 +268,21 @@ DNS resolvers (Default: <code class="docutils literal notranslate"><span class="
|
||||
any configured output destination fails while saving/publishing
|
||||
reports (Default: <code class="docutils literal notranslate"><span class="pre">False</span></code>)</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">log_file</span></code> - str: Write log messages to a file at this path</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">n_procs</span></code> - int: Number of process to run in parallel when
|
||||
parsing in CLI mode (Default: <code class="docutils literal notranslate"><span class="pre">1</span></code>)</p>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">n_procs</span></code> - int: Number of processes to run in parallel when
|
||||
parsing report files passed directly as CLI arguments
|
||||
(Default: <code class="docutils literal notranslate"><span class="pre">1</span></code>)</p>
|
||||
<div class="admonition note">
|
||||
<p class="admonition-title">Note</p>
|
||||
<p>Setting this to a number larger than one can improve
|
||||
performance when processing thousands of files</p>
|
||||
</div>
|
||||
<div class="admonition note">
|
||||
<p class="admonition-title">Note</p>
|
||||
<p><code class="docutils literal notranslate"><span class="pre">n_procs</span></code> only applies to report files passed directly on the
|
||||
command line. Messages from mbox files and from mailbox
|
||||
connections (IMAP, Microsoft Graph, Gmail API, Maildir) are
|
||||
always processed sequentially.</p>
|
||||
</div>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
|
||||
Reference in New Issue
Block a user