mirror of
https://github.com/domainaware/parsedmarc.git
synced 2026-09-21 21:28:32 +00:00
Update docs
This commit is contained in:
Sean Whalen
+2
-2
@@ -5,14 +5,14 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Overview: module code — parsedmarc 10.2.4 documentation</title>
|
||||
<title>Overview: module code — parsedmarc 10.3.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="../_static/css/theme.css?v=9edc463e" />
|
||||
|
||||
|
||||
<script src="../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../_static/documentation_options.js?v=2c8206f1"></script>
|
||||
<script src="../_static/documentation_options.js?v=a126560f"></script>
|
||||
<script src="../_static/doctools.js?v=fd6eb6e6"></script>
|
||||
<script src="../_static/sphinx_highlight.js?v=6ffebe34"></script>
|
||||
<script src="../_static/js/theme.js"></script>
|
||||
|
||||
+77
-124
@@ -5,14 +5,14 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc — parsedmarc 10.2.4 documentation</title>
|
||||
<title>parsedmarc — parsedmarc 10.3.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="../_static/css/theme.css?v=9edc463e" />
|
||||
|
||||
|
||||
<script src="../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../_static/documentation_options.js?v=2c8206f1"></script>
|
||||
<script src="../_static/documentation_options.js?v=a126560f"></script>
|
||||
<script src="../_static/doctools.js?v=fd6eb6e6"></script>
|
||||
<script src="../_static/sphinx_highlight.js?v=6ffebe34"></script>
|
||||
<script src="../_static/js/theme.js"></script>
|
||||
@@ -115,6 +115,7 @@
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">xmltodict</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">expiringdict</span><span class="w"> </span><span class="kn">import</span> <span class="n">ExpiringDict</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">mailsuite.smtp</span><span class="w"> </span><span class="kn">import</span> <span class="n">send_email</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">tqdm</span><span class="w"> </span><span class="kn">import</span> <span class="n">tqdm</span>
|
||||
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.constants</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
|
||||
<span class="n">DEFAULT_DNS_MAX_RETRIES</span><span class="p">,</span>
|
||||
@@ -147,7 +148,7 @@
|
||||
<span class="n">timestamp_to_human</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"parsedmarc v</span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">__version__</span><span class="p">))</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">"parsedmarc v</span><span class="si">{</span><span class="n">__version__</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
|
||||
<span class="n">feedback_report_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">"^([\w\-]+): (.+)$"</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">MULTILINE</span><span class="p">)</span>
|
||||
<span class="n">xml_header_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">"^<\?xml .*?>"</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">MULTILINE</span><span class="p">)</span>
|
||||
@@ -254,7 +255,7 @@
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">frames</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="s2">""</span>
|
||||
<span class="n">last</span> <span class="o">=</span> <span class="n">frames</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
|
||||
<span class="k">return</span> <span class="s2">" (raised at </span><span class="si">{0}</span><span class="s2">:</span><span class="si">{1}</span><span class="s2">)"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">last</span><span class="o">.</span><span class="n">filename</span><span class="p">,</span> <span class="n">last</span><span class="o">.</span><span class="n">lineno</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="sa">f</span><span class="s2">" (raised at </span><span class="si">{</span><span class="n">last</span><span class="o">.</span><span class="n">filename</span><span class="si">}</span><span class="s2">:</span><span class="si">{</span><span class="n">last</span><span class="o">.</span><span class="n">lineno</span><span class="si">}</span><span class="s2">)"</span>
|
||||
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">_text</span><span class="p">(</span><span class="n">value</span><span class="p">:</span> <span class="n">Any</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">:</span>
|
||||
@@ -930,7 +931,7 @@
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">xmltodict</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">xml</span><span class="p">)[</span><span class="s2">"feedback"</span><span class="p">]</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">errors</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">"Invalid XML: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
||||
<span class="n">errors</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Invalid XML: </span><span class="si">{</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">tree</span> <span class="o">=</span> <span class="n">etree</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span>
|
||||
<span class="n">BytesIO</span><span class="p">(</span><span class="n">xml</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s2">"utf-8"</span><span class="p">)),</span>
|
||||
@@ -978,13 +979,10 @@
|
||||
<span class="k">if</span> <span class="n">new_org_name</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">org_name</span> <span class="o">=</span> <span class="n">new_org_name</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">org_name</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
||||
<span class="s2">"Could not parse org_name from XML.</span><span class="se">\r\n</span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">report</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Could not parse org_name from XML.</span><span class="se">\r\n</span><span class="si">{</span><span class="n">report</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="k">raise</span> <span class="ne">KeyError</span><span class="p">(</span>
|
||||
<span class="s2">"Organization name is missing. </span><span class="se">\</span>
|
||||
<span class="s2"> This field is a requirement for </span><span class="se">\</span>
|
||||
<span class="s2"> saving the report"</span>
|
||||
<span class="s2">"Organization name is missing. This field is a requirement "</span>
|
||||
<span class="s2">"for saving the report"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"org_name"</span><span class="p">]</span> <span class="o">=</span> <span class="n">org_name</span>
|
||||
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"org_email"</span><span class="p">]</span> <span class="o">=</span> <span class="n">report_metadata</span><span class="p">[</span><span class="s2">"email"</span><span class="p">]</span>
|
||||
@@ -1083,14 +1081,14 @@
|
||||
<span class="k">if</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"np"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">np_</span> <span class="o">=</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"np"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="n">np_</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">(</span><span class="s2">"none"</span><span class="p">,</span> <span class="s2">"quarantine"</span><span class="p">,</span> <span class="s2">"reject"</span><span class="p">):</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"Invalid np value: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">np_</span><span class="p">))</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Invalid np value: </span><span class="si">{</span><span class="n">np_</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="n">new_policy_published</span><span class="p">[</span><span class="s2">"np"</span><span class="p">]</span> <span class="o">=</span> <span class="n">np_</span>
|
||||
<span class="n">testing</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="k">if</span> <span class="s2">"testing"</span> <span class="ow">in</span> <span class="n">policy_published</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"testing"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">testing</span> <span class="o">=</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"testing"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="n">testing</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">(</span><span class="s2">"n"</span><span class="p">,</span> <span class="s2">"y"</span><span class="p">):</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"Invalid testing value: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">testing</span><span class="p">))</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Invalid testing value: </span><span class="si">{</span><span class="n">testing</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="n">new_policy_published</span><span class="p">[</span><span class="s2">"testing"</span><span class="p">]</span> <span class="o">=</span> <span class="n">testing</span>
|
||||
<span class="n">discovery_method</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="k">if</span> <span class="s2">"discovery_method"</span> <span class="ow">in</span> <span class="n">policy_published</span><span class="p">:</span>
|
||||
@@ -1098,7 +1096,7 @@
|
||||
<span class="n">discovery_method</span> <span class="o">=</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"discovery_method"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="n">discovery_method</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">(</span><span class="s2">"psl"</span><span class="p">,</span> <span class="s2">"treewalk"</span><span class="p">):</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
||||
<span class="s2">"Invalid discovery_method value: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">discovery_method</span><span class="p">)</span>
|
||||
<span class="sa">f</span><span class="s2">"Invalid discovery_method value: </span><span class="si">{</span><span class="n">discovery_method</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">new_policy_published</span><span class="p">[</span><span class="s2">"discovery_method"</span><span class="p">]</span> <span class="o">=</span> <span class="n">discovery_method</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">]</span> <span class="o">=</span> <span class="n">new_policy_published</span>
|
||||
@@ -1108,7 +1106,7 @@
|
||||
<span class="k">if</span> <span class="n">keep_alive</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">i</span> <span class="o">></span> <span class="mi">0</span> <span class="ow">and</span> <span class="n">i</span> <span class="o">%</span> <span class="mi">20</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Sending keepalive cmd"</span><span class="p">)</span>
|
||||
<span class="n">keep_alive</span><span class="p">()</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Processed </span><span class="si">{0}</span><span class="s2">/</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">i</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">"record"</span><span class="p">])))</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Processed </span><span class="si">{}</span><span class="s2">/</span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">i</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">"record"</span><span class="p">])))</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">report_record</span> <span class="o">=</span> <span class="n">_parse_report_record</span><span class="p">(</span>
|
||||
<span class="n">report</span><span class="p">[</span><span class="s2">"record"</span><span class="p">][</span><span class="n">i</span><span class="p">],</span>
|
||||
@@ -1130,7 +1128,7 @@
|
||||
<span class="n">normalize</span><span class="o">=</span><span class="n">normalize_timespan</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"Could not parse record: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="p">))</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Could not parse record: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">report_record</span> <span class="o">=</span> <span class="n">_parse_report_record</span><span class="p">(</span>
|
||||
@@ -1158,20 +1156,16 @@
|
||||
<span class="k">return</span> <span class="n">cast</span><span class="p">(</span><span class="n">AggregateReport</span><span class="p">,</span> <span class="n">new_report</span><span class="p">)</span>
|
||||
|
||||
<span class="k">except</span> <span class="n">expat</span><span class="o">.</span><span class="n">ExpatError</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidAggregateReport</span><span class="p">(</span>
|
||||
<span class="s2">"Invalid XML: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">error</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidAggregateReport</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Invalid XML: </span><span class="si">{</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">error</span>
|
||||
|
||||
<span class="k">except</span> <span class="ne">KeyError</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidAggregateReport</span><span class="p">(</span>
|
||||
<span class="s2">"Missing field: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">error</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidAggregateReport</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Missing field: </span><span class="si">{</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">error</span>
|
||||
<span class="k">except</span> <span class="ne">AttributeError</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidAggregateReport</span><span class="p">(</span><span class="s2">"Report missing required section"</span><span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">error</span>
|
||||
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidAggregateReport</span><span class="p">(</span>
|
||||
<span class="s2">"Unexpected error: </span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">(),</span> <span class="n">_exc_origin</span><span class="p">(</span><span class="n">error</span><span class="p">))</span>
|
||||
<span class="sa">f</span><span class="s2">"Unexpected error: </span><span class="si">{</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}{</span><span class="n">_exc_origin</span><span class="p">(</span><span class="n">error</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">error</span></div>
|
||||
|
||||
|
||||
@@ -1252,7 +1246,7 @@
|
||||
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">ParserError</span><span class="p">(</span>
|
||||
<span class="s2">"Invalid archive file: </span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">(),</span> <span class="n">_exc_origin</span><span class="p">(</span><span class="n">error</span><span class="p">))</span>
|
||||
<span class="sa">f</span><span class="s2">"Invalid archive file: </span><span class="si">{</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}{</span><span class="n">_exc_origin</span><span class="p">(</span><span class="n">error</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">error</span>
|
||||
<span class="k">finally</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">file_object</span><span class="p">:</span>
|
||||
@@ -1731,13 +1725,11 @@
|
||||
<span class="k">return</span> <span class="n">cast</span><span class="p">(</span><span class="n">FailureReport</span><span class="p">,</span> <span class="n">parsed_report</span><span class="p">)</span>
|
||||
|
||||
<span class="k">except</span> <span class="ne">KeyError</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidFailureReport</span><span class="p">(</span>
|
||||
<span class="s2">"Missing value: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">error</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidFailureReport</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Missing value: </span><span class="si">{</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">error</span>
|
||||
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidFailureReport</span><span class="p">(</span>
|
||||
<span class="s2">"Unexpected error: </span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">(),</span> <span class="n">_exc_origin</span><span class="p">(</span><span class="n">error</span><span class="p">))</span>
|
||||
<span class="sa">f</span><span class="s2">"Unexpected error: </span><span class="si">{</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}{</span><span class="n">_exc_origin</span><span class="p">(</span><span class="n">error</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">error</span></div>
|
||||
|
||||
|
||||
@@ -1922,7 +1914,7 @@
|
||||
<span class="n">sample</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="n">is_feedback_report</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
<span class="k">if</span> <span class="s2">"From"</span> <span class="ow">in</span> <span class="n">msg_headers</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Parsing mail from </span><span class="si">{0}</span><span class="s2"> on </span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">msg_headers</span><span class="p">[</span><span class="s2">"From"</span><span class="p">],</span> <span class="n">date</span><span class="p">))</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Parsing mail from </span><span class="si">{}</span><span class="s2"> on </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">msg_headers</span><span class="p">[</span><span class="s2">"From"</span><span class="p">],</span> <span class="n">date</span><span class="p">))</span>
|
||||
<span class="k">if</span> <span class="s2">"Subject"</span> <span class="ow">in</span> <span class="n">msg_headers</span><span class="p">:</span>
|
||||
<span class="n">subject</span> <span class="o">=</span> <span class="n">msg_headers</span><span class="p">[</span><span class="s2">"Subject"</span><span class="p">]</span>
|
||||
<span class="k">for</span> <span class="n">part</span> <span class="ow">in</span> <span class="n">msg</span><span class="o">.</span><span class="n">walk</span><span class="p">():</span>
|
||||
@@ -1972,9 +1964,7 @@
|
||||
<span class="n">fields</span><span class="p">[</span><span class="s2">"received-date"</span><span class="p">],</span> <span class="n">fields</span><span class="p">[</span><span class="s2">"sender-ip-address"</span><span class="p">]</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">error</span> <span class="o">=</span> <span class="s1">'Unable to parse message with subject "</span><span class="si">{0}</span><span class="s1">": </span><span class="si">{1}{2}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">subject</span><span class="p">,</span> <span class="n">e</span><span class="p">,</span> <span class="n">_exc_origin</span><span class="p">(</span><span class="n">e</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">error</span> <span class="o">=</span> <span class="sa">f</span><span class="s1">'Unable to parse message with subject "</span><span class="si">{</span><span class="n">subject</span><span class="si">}</span><span class="s1">": </span><span class="si">{</span><span class="n">e</span><span class="si">}{</span><span class="n">_exc_origin</span><span class="p">(</span><span class="n">e</span><span class="p">)</span><span class="si">}</span><span class="s1">'</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidDMARCReport</span><span class="p">(</span><span class="n">error</span><span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">e</span>
|
||||
|
||||
<span class="n">sample</span> <span class="o">=</span> <span class="n">parts</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">lstrip</span><span class="p">()</span>
|
||||
@@ -2016,15 +2006,12 @@
|
||||
|
||||
<span class="k">except</span> <span class="n">InvalidDMARCReport</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">error</span> <span class="o">=</span> <span class="p">(</span>
|
||||
<span class="s1">'Message with subject "</span><span class="si">{0}</span><span class="s1">" is not a valid '</span>
|
||||
<span class="s2">"DMARC report: </span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">subject</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
|
||||
<span class="sa">f</span><span class="s1">'Message with subject "</span><span class="si">{</span><span class="n">subject</span><span class="si">}</span><span class="s1">" is not a valid DMARC report: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s1">'</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">raise</span> <span class="n">ParserError</span><span class="p">(</span><span class="n">error</span><span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">e</span>
|
||||
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">error</span> <span class="o">=</span> <span class="s1">'Unable to parse message with subject "</span><span class="si">{0}</span><span class="s1">": </span><span class="si">{1}{2}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">subject</span><span class="p">,</span> <span class="n">e</span><span class="p">,</span> <span class="n">_exc_origin</span><span class="p">(</span><span class="n">e</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">error</span> <span class="o">=</span> <span class="sa">f</span><span class="s1">'Unable to parse message with subject "</span><span class="si">{</span><span class="n">subject</span><span class="si">}</span><span class="s1">": </span><span class="si">{</span><span class="n">e</span><span class="si">}{</span><span class="n">_exc_origin</span><span class="p">(</span><span class="n">e</span><span class="p">)</span><span class="si">}</span><span class="s1">'</span>
|
||||
<span class="k">raise</span> <span class="n">ParserError</span><span class="p">(</span><span class="n">error</span><span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">e</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">feedback_report</span> <span class="ow">and</span> <span class="n">sample</span><span class="p">:</span>
|
||||
@@ -2045,9 +2032,9 @@
|
||||
<span class="p">)</span>
|
||||
<span class="k">except</span> <span class="n">InvalidFailureReport</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">error</span> <span class="o">=</span> <span class="p">(</span>
|
||||
<span class="s1">'Message with subject "</span><span class="si">{0}</span><span class="s1">" '</span>
|
||||
<span class="sa">f</span><span class="s1">'Message with subject "</span><span class="si">{</span><span class="n">subject</span><span class="si">}</span><span class="s1">" '</span>
|
||||
<span class="s2">"is not a valid "</span>
|
||||
<span class="s2">"failure DMARC report: </span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">subject</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
|
||||
<span class="sa">f</span><span class="s2">"failure DMARC report: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidFailureReport</span><span class="p">(</span><span class="n">error</span><span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">e</span>
|
||||
|
||||
@@ -2055,7 +2042,7 @@
|
||||
<span class="k">return</span> <span class="n">result</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">result</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">error</span> <span class="o">=</span> <span class="s1">'Message with subject "</span><span class="si">{0}</span><span class="s1">" is not a valid report'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">subject</span><span class="p">)</span>
|
||||
<span class="n">error</span> <span class="o">=</span> <span class="sa">f</span><span class="s1">'Message with subject "</span><span class="si">{</span><span class="n">subject</span><span class="si">}</span><span class="s1">" is not a valid report'</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidDMARCReport</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">result</span></div>
|
||||
@@ -2099,11 +2086,11 @@
|
||||
<span class="n">sniff</span> <span class="o">=</span> <span class="n">sniff</span><span class="o">.</span><span class="n">lstrip</span><span class="p">()</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">sniff</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">"<"</span><span class="p">):</span>
|
||||
<span class="k">return</span> <span class="s2">"Invalid aggregate report: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">aggregate_error</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="sa">f</span><span class="s2">"Invalid aggregate report: </span><span class="si">{</span><span class="n">aggregate_error</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="k">if</span> <span class="n">sniff</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">"{"</span><span class="p">):</span>
|
||||
<span class="k">return</span> <span class="s2">"Invalid SMTP TLS report: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">smtp_tls_error</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="sa">f</span><span class="s2">"Invalid SMTP TLS report: </span><span class="si">{</span><span class="n">smtp_tls_error</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="k">if</span> <span class="n">_looks_like_email</span><span class="p">(</span><span class="n">sniff</span><span class="p">):</span>
|
||||
<span class="k">return</span> <span class="s2">"Invalid report email: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">email_error</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="sa">f</span><span class="s2">"Invalid report email: </span><span class="si">{</span><span class="n">email_error</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="k">return</span> <span class="p">(</span>
|
||||
<span class="s2">"Not a recognized report format (not a DMARC aggregate XML report, "</span>
|
||||
<span class="s2">"an SMTP TLS JSON report, or a DMARC report email)"</span>
|
||||
@@ -2153,7 +2140,7 @@
|
||||
<span class="n">file_object</span><span class="p">:</span> <span class="n">BinaryIO</span>
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">input_</span><span class="p">,</span> <span class="p">(</span><span class="nb">str</span><span class="p">,</span> <span class="n">os</span><span class="o">.</span><span class="n">PathLike</span><span class="p">)):</span>
|
||||
<span class="n">file_path</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">fspath</span><span class="p">(</span><span class="n">input_</span><span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Parsing </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">file_path</span><span class="p">))</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Parsing </span><span class="si">{</span><span class="n">file_path</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="n">file_object</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="n">file_path</span><span class="p">,</span> <span class="s2">"rb"</span><span class="p">)</span>
|
||||
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">input_</span><span class="p">,</span> <span class="p">(</span><span class="nb">bytes</span><span class="p">,</span> <span class="nb">bytearray</span><span class="p">,</span> <span class="nb">memoryview</span><span class="p">)):</span>
|
||||
<span class="n">file_object</span> <span class="o">=</span> <span class="n">BytesIO</span><span class="p">(</span><span class="nb">bytes</span><span class="p">(</span><span class="n">input_</span><span class="p">))</span>
|
||||
@@ -2265,10 +2252,10 @@
|
||||
<span class="n">mbox</span> <span class="o">=</span> <span class="n">mailbox</span><span class="o">.</span><span class="n">mbox</span><span class="p">(</span><span class="n">input_</span><span class="p">)</span>
|
||||
<span class="n">message_keys</span> <span class="o">=</span> <span class="n">mbox</span><span class="o">.</span><span class="n">keys</span><span class="p">()</span>
|
||||
<span class="n">total_messages</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">message_keys</span><span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Found </span><span class="si">{0}</span><span class="s2"> messages in </span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">total_messages</span><span class="p">,</span> <span class="n">input_</span><span class="p">))</span>
|
||||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">message_keys</span><span class="p">)):</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Found </span><span class="si">{</span><span class="n">total_messages</span><span class="si">}</span><span class="s2"> messages in </span><span class="si">{</span><span class="n">input_</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">tqdm</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="n">total_messages</span><span class="p">),</span> <span class="n">disable</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
<span class="n">message_key</span> <span class="o">=</span> <span class="n">message_keys</span><span class="p">[</span><span class="n">i</span><span class="p">]</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Processing message </span><span class="si">{0}</span><span class="s2"> of </span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">i</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="n">total_messages</span><span class="p">))</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Processing message </span><span class="si">{</span><span class="n">i</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="mi">1</span><span class="si">}</span><span class="s2"> of </span><span class="si">{</span><span class="n">total_messages</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="n">msg_content</span> <span class="o">=</span> <span class="n">mbox</span><span class="o">.</span><span class="n">get_string</span><span class="p">(</span><span class="n">message_key</span><span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">sa</span> <span class="o">=</span> <span class="n">strip_attachment_payloads</span>
|
||||
@@ -2304,7 +2291,7 @@
|
||||
<span class="k">except</span> <span class="n">InvalidDMARCReport</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="k">except</span> <span class="n">mailbox</span><span class="o">.</span><span class="n">NoSuchMailboxError</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidDMARCReport</span><span class="p">(</span><span class="s2">"Mailbox </span><span class="si">{0}</span><span class="s2"> does not exist"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">input_</span><span class="p">))</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidDMARCReport</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Mailbox </span><span class="si">{</span><span class="n">input_</span><span class="si">}</span><span class="s2"> does not exist"</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="p">{</span>
|
||||
<span class="s2">"aggregate_reports"</span><span class="p">:</span> <span class="n">aggregate_reports</span><span class="p">,</span>
|
||||
<span class="s2">"failure_reports"</span><span class="p">:</span> <span class="n">failure_reports</span><span class="p">,</span>
|
||||
@@ -2331,8 +2318,8 @@
|
||||
<span class="sd"> (warn, don't crash). Uses the folder-management API added in mailsuite</span>
|
||||
<span class="sd"> 2.1.0 (``folder_exists`` / ``rename_folder`` / ``merge_folders``).</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">old_folder</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">/Forensic"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">archive_folder</span><span class="p">)</span>
|
||||
<span class="n">new_folder</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">/Failure"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">archive_folder</span><span class="p">)</span>
|
||||
<span class="n">old_folder</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">archive_folder</span><span class="si">}</span><span class="s2">/Forensic"</span>
|
||||
<span class="n">new_folder</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">archive_folder</span><span class="si">}</span><span class="s2">/Failure"</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">connection</span><span class="o">.</span><span class="n">folder_exists</span><span class="p">(</span><span class="n">old_folder</span><span class="p">):</span>
|
||||
<span class="k">return</span>
|
||||
@@ -2341,23 +2328,13 @@
|
||||
<span class="c1"># created Failure folder): move the legacy folder's messages into</span>
|
||||
<span class="c1"># the new one and drop the now-empty legacy folder.</span>
|
||||
<span class="n">connection</span><span class="o">.</span><span class="n">merge_folders</span><span class="p">(</span><span class="n">old_folder</span><span class="p">,</span> <span class="n">new_folder</span><span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
|
||||
<span class="s2">"Merged legacy archive folder </span><span class="si">{0}</span><span class="s2"> into </span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">old_folder</span><span class="p">,</span> <span class="n">new_folder</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Merged legacy archive folder </span><span class="si">{</span><span class="n">old_folder</span><span class="si">}</span><span class="s2"> into </span><span class="si">{</span><span class="n">new_folder</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">connection</span><span class="o">.</span><span class="n">rename_folder</span><span class="p">(</span><span class="n">old_folder</span><span class="p">,</span> <span class="n">new_folder</span><span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
|
||||
<span class="s2">"Renamed legacy archive folder </span><span class="si">{0}</span><span class="s2"> to </span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">old_folder</span><span class="p">,</span> <span class="n">new_folder</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Renamed legacy archive folder </span><span class="si">{</span><span class="n">old_folder</span><span class="si">}</span><span class="s2"> to </span><span class="si">{</span><span class="n">new_folder</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
||||
<span class="s2">"Could not migrate legacy archive folder </span><span class="si">{0}</span><span class="s2"> to </span><span class="si">{1}</span><span class="s2">: </span><span class="si">{2}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">old_folder</span><span class="p">,</span> <span class="n">new_folder</span><span class="p">,</span> <span class="n">error</span>
|
||||
<span class="p">)</span>
|
||||
<span class="sa">f</span><span class="s2">"Could not migrate legacy archive folder </span><span class="si">{</span><span class="n">old_folder</span><span class="si">}</span><span class="s2"> to </span><span class="si">{</span><span class="n">new_folder</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="n">error</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
|
||||
@@ -2432,10 +2409,10 @@
|
||||
<span class="n">aggregate_report_msg_uids</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="n">failure_report_msg_uids</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="n">smtp_tls_msg_uids</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="n">aggregate_reports_folder</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">/Aggregate"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">archive_folder</span><span class="p">)</span>
|
||||
<span class="n">failure_reports_folder</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">/Failure"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">archive_folder</span><span class="p">)</span>
|
||||
<span class="n">smtp_tls_reports_folder</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">/SMTP-TLS"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">archive_folder</span><span class="p">)</span>
|
||||
<span class="n">invalid_reports_folder</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">/Invalid"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">archive_folder</span><span class="p">)</span>
|
||||
<span class="n">aggregate_reports_folder</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">archive_folder</span><span class="si">}</span><span class="s2">/Aggregate"</span>
|
||||
<span class="n">failure_reports_folder</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">archive_folder</span><span class="si">}</span><span class="s2">/Failure"</span>
|
||||
<span class="n">smtp_tls_reports_folder</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">archive_folder</span><span class="si">}</span><span class="s2">/SMTP-TLS"</span>
|
||||
<span class="n">invalid_reports_folder</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">archive_folder</span><span class="si">}</span><span class="s2">/Invalid"</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">results</span><span class="p">:</span>
|
||||
<span class="n">aggregate_reports</span> <span class="o">=</span> <span class="n">results</span><span class="p">[</span><span class="s2">"aggregate_reports"</span><span class="p">]</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
@@ -2464,17 +2441,17 @@
|
||||
<span class="n">_since</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">s</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span> <span class="o">*</span> <span class="mi">60</span> <span class="o">*</span> <span class="mi">24</span> <span class="o">*</span> <span class="mi">7</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
||||
<span class="s2">"Incorrect format for 'since' option. </span><span class="se">\</span>
|
||||
<span class="s2"> Provided value:</span><span class="si">{0}</span><span class="s2">, Expected values:(5m|3h|2d|1w). </span><span class="se">\</span>
|
||||
<span class="s2"> Ignoring option, fetching messages for last 24hrs"</span>
|
||||
<span class="s2">"SMTP does not support a time or timezone in since."</span>
|
||||
<span class="s2">"See https://www.rfc-editor.org/rfc/rfc3501#page-52"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">since</span><span class="p">)</span>
|
||||
<span class="sa">f</span><span class="s2">"Incorrect format for 'since' option. Provided value: </span><span class="si">{</span><span class="n">since</span><span class="si">}</span><span class="s2">, "</span>
|
||||
<span class="s2">"expected values: (5m|3h|2d|1w). Ignoring option, fetching "</span>
|
||||
<span class="s2">"messages for last 24hrs. SMTP does not support a time or "</span>
|
||||
<span class="s2">"timezone in since. See "</span>
|
||||
<span class="s2">"https://www.rfc-editor.org/rfc/rfc3501#page-52"</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">connection</span><span class="p">,</span> <span class="n">IMAPConnection</span><span class="p">):</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
||||
<span class="s2">"Only days and weeks values in 'since' option are </span><span class="se">\</span>
|
||||
<span class="s2"> considered for IMAP connections. Examples: 2d or 1w"</span>
|
||||
<span class="s2">"Only days and weeks values in 'since' option are considered "</span>
|
||||
<span class="s2">"for IMAP connections. Examples: 2d or 1w"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">since</span> <span class="o">=</span> <span class="p">(</span><span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">(</span><span class="n">timezone</span><span class="o">.</span><span class="n">utc</span><span class="p">)</span> <span class="o">-</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">minutes</span><span class="o">=</span><span class="n">_since</span><span class="p">))</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span>
|
||||
<span class="s2">"</span><span class="si">%d</span><span class="s2">-%b-%Y"</span>
|
||||
@@ -2495,22 +2472,18 @@
|
||||
<span class="n">reports_folder</span><span class="p">,</span> <span class="n">batch_size</span><span class="o">=</span><span class="n">batch_size</span><span class="p">,</span> <span class="n">since</span><span class="o">=</span><span class="n">since</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">total_messages</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">messages</span><span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Found </span><span class="si">{0}</span><span class="s2"> messages in </span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">messages</span><span class="p">),</span> <span class="n">reports_folder</span><span class="p">))</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Found </span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="n">messages</span><span class="p">)</span><span class="si">}</span><span class="s2"> messages in </span><span class="si">{</span><span class="n">reports_folder</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">batch_size</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">since</span><span class="p">:</span>
|
||||
<span class="n">message_limit</span> <span class="o">=</span> <span class="nb">min</span><span class="p">(</span><span class="n">total_messages</span><span class="p">,</span> <span class="n">batch_size</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">message_limit</span> <span class="o">=</span> <span class="n">total_messages</span>
|
||||
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Processing </span><span class="si">{0}</span><span class="s2"> messages"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">message_limit</span><span class="p">))</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Processing </span><span class="si">{</span><span class="n">message_limit</span><span class="si">}</span><span class="s2"> messages"</span><span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">message_limit</span><span class="p">):</span>
|
||||
<span class="n">msg_uid</span> <span class="o">=</span> <span class="n">messages</span><span class="p">[</span><span class="n">i</span><span class="p">]</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
||||
<span class="s2">"Processing message </span><span class="si">{0}</span><span class="s2"> of </span><span class="si">{1}</span><span class="s2">: UID </span><span class="si">{2}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">i</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="n">message_limit</span><span class="p">,</span> <span class="n">msg_uid</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Processing message </span><span class="si">{</span><span class="n">i</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="mi">1</span><span class="si">}</span><span class="s2"> of </span><span class="si">{</span><span class="n">message_limit</span><span class="si">}</span><span class="s2">: UID </span><span class="si">{</span><span class="n">msg_uid</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="n">message_id</span><span class="p">:</span> <span class="nb">int</span> <span class="o">|</span> <span class="nb">str</span>
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">connection</span><span class="p">,</span> <span class="n">IMAPConnection</span><span class="p">):</span>
|
||||
<span class="n">message_id</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">msg_uid</span><span class="p">)</span>
|
||||
@@ -2562,16 +2535,14 @@
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">test</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">delete</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Deleting message UID </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">msg_uid</span><span class="p">))</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Deleting message UID </span><span class="si">{</span><span class="n">msg_uid</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">connection</span><span class="p">,</span> <span class="n">IMAPConnection</span><span class="p">):</span>
|
||||
<span class="n">connection</span><span class="o">.</span><span class="n">delete_message</span><span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="n">message_id</span><span class="p">))</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">connection</span><span class="o">.</span><span class="n">delete_message</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">message_id</span><span class="p">))</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
||||
<span class="s2">"Moving message UID </span><span class="si">{0}</span><span class="s2"> to </span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">msg_uid</span><span class="p">,</span> <span class="n">invalid_reports_folder</span>
|
||||
<span class="p">)</span>
|
||||
<span class="sa">f</span><span class="s2">"Moving message UID </span><span class="si">{</span><span class="n">msg_uid</span><span class="si">}</span><span class="s2"> to </span><span class="si">{</span><span class="n">invalid_reports_folder</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">connection</span><span class="p">,</span> <span class="n">IMAPConnection</span><span class="p">):</span>
|
||||
<span class="n">connection</span><span class="o">.</span><span class="n">move_message</span><span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="n">message_id</span><span class="p">),</span> <span class="n">invalid_reports_folder</span><span class="p">)</span>
|
||||
@@ -2588,81 +2559,63 @@
|
||||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">number_of_processed_msgs</span><span class="p">):</span>
|
||||
<span class="n">msg_uid</span> <span class="o">=</span> <span class="n">processed_messages</span><span class="p">[</span><span class="n">i</span><span class="p">]</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
||||
<span class="s2">"Deleting message </span><span class="si">{0}</span><span class="s2"> of </span><span class="si">{1}</span><span class="s2">: UID </span><span class="si">{2}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">i</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="n">number_of_processed_msgs</span><span class="p">,</span> <span class="n">msg_uid</span>
|
||||
<span class="p">)</span>
|
||||
<span class="sa">f</span><span class="s2">"Deleting message </span><span class="si">{</span><span class="n">i</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="mi">1</span><span class="si">}</span><span class="s2"> of </span><span class="si">{</span><span class="n">number_of_processed_msgs</span><span class="si">}</span><span class="s2">: UID </span><span class="si">{</span><span class="n">msg_uid</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">connection</span><span class="o">.</span><span class="n">delete_message</span><span class="p">(</span><span class="n">msg_uid</span><span class="p">)</span>
|
||||
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">message</span> <span class="o">=</span> <span class="s2">"Error deleting message UID"</span>
|
||||
<span class="n">e</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2"> </span><span class="si">{1}</span><span class="s2">: </span><span class="si">{2}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">message</span><span class="p">,</span> <span class="n">msg_uid</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Mailbox error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="p">))</span>
|
||||
<span class="n">e</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">message</span><span class="si">}</span><span class="s2"> </span><span class="si">{</span><span class="n">msg_uid</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Mailbox error: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">aggregate_report_msg_uids</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="n">log_message</span> <span class="o">=</span> <span class="s2">"Moving aggregate report messages from"</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
||||
<span class="s2">"</span><span class="si">{0}</span><span class="s2"> </span><span class="si">{1}</span><span class="s2"> to </span><span class="si">{2}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">log_message</span><span class="p">,</span> <span class="n">reports_folder</span><span class="p">,</span> <span class="n">aggregate_reports_folder</span>
|
||||
<span class="p">)</span>
|
||||
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">log_message</span><span class="si">}</span><span class="s2"> </span><span class="si">{</span><span class="n">reports_folder</span><span class="si">}</span><span class="s2"> to </span><span class="si">{</span><span class="n">aggregate_reports_folder</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">number_of_agg_report_msgs</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">aggregate_report_msg_uids</span><span class="p">)</span>
|
||||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">number_of_agg_report_msgs</span><span class="p">):</span>
|
||||
<span class="n">msg_uid</span> <span class="o">=</span> <span class="n">aggregate_report_msg_uids</span><span class="p">[</span><span class="n">i</span><span class="p">]</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
||||
<span class="s2">"Moving message </span><span class="si">{0}</span><span class="s2"> of </span><span class="si">{1}</span><span class="s2">: UID </span><span class="si">{2}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">i</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="n">number_of_agg_report_msgs</span><span class="p">,</span> <span class="n">msg_uid</span>
|
||||
<span class="p">)</span>
|
||||
<span class="sa">f</span><span class="s2">"Moving message </span><span class="si">{</span><span class="n">i</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="mi">1</span><span class="si">}</span><span class="s2"> of </span><span class="si">{</span><span class="n">number_of_agg_report_msgs</span><span class="si">}</span><span class="s2">: UID </span><span class="si">{</span><span class="n">msg_uid</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">connection</span><span class="o">.</span><span class="n">move_message</span><span class="p">(</span><span class="n">msg_uid</span><span class="p">,</span> <span class="n">aggregate_reports_folder</span><span class="p">)</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">message</span> <span class="o">=</span> <span class="s2">"Error moving message UID"</span>
|
||||
<span class="n">e</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2"> </span><span class="si">{1}</span><span class="s2">: </span><span class="si">{2}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">message</span><span class="p">,</span> <span class="n">msg_uid</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Mailbox error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="p">))</span>
|
||||
<span class="n">e</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">message</span><span class="si">}</span><span class="s2"> </span><span class="si">{</span><span class="n">msg_uid</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Mailbox error: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">failure_report_msg_uids</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="n">message</span> <span class="o">=</span> <span class="s2">"Moving failure report messages from"</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
||||
<span class="s2">"</span><span class="si">{0}</span><span class="s2"> </span><span class="si">{1}</span><span class="s2"> to </span><span class="si">{2}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">message</span><span class="p">,</span> <span class="n">reports_folder</span><span class="p">,</span> <span class="n">failure_reports_folder</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">message</span><span class="si">}</span><span class="s2"> </span><span class="si">{</span><span class="n">reports_folder</span><span class="si">}</span><span class="s2"> to </span><span class="si">{</span><span class="n">failure_reports_folder</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="n">number_of_failure_msgs</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">failure_report_msg_uids</span><span class="p">)</span>
|
||||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">number_of_failure_msgs</span><span class="p">):</span>
|
||||
<span class="n">msg_uid</span> <span class="o">=</span> <span class="n">failure_report_msg_uids</span><span class="p">[</span><span class="n">i</span><span class="p">]</span>
|
||||
<span class="n">message</span> <span class="o">=</span> <span class="s2">"Moving message"</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
||||
<span class="s2">"</span><span class="si">{0}</span><span class="s2"> </span><span class="si">{1}</span><span class="s2"> of </span><span class="si">{2}</span><span class="s2">: UID </span><span class="si">{3}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">message</span><span class="p">,</span> <span class="n">i</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="n">number_of_failure_msgs</span><span class="p">,</span> <span class="n">msg_uid</span>
|
||||
<span class="p">)</span>
|
||||
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">message</span><span class="si">}</span><span class="s2"> </span><span class="si">{</span><span class="n">i</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="mi">1</span><span class="si">}</span><span class="s2"> of </span><span class="si">{</span><span class="n">number_of_failure_msgs</span><span class="si">}</span><span class="s2">: UID </span><span class="si">{</span><span class="n">msg_uid</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">connection</span><span class="o">.</span><span class="n">move_message</span><span class="p">(</span><span class="n">msg_uid</span><span class="p">,</span> <span class="n">failure_reports_folder</span><span class="p">)</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">e</span> <span class="o">=</span> <span class="s2">"Error moving message UID </span><span class="si">{0}</span><span class="s2">: </span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">msg_uid</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Mailbox error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="p">))</span>
|
||||
<span class="n">e</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"Error moving message UID </span><span class="si">{</span><span class="n">msg_uid</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Mailbox error: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">smtp_tls_msg_uids</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="n">message</span> <span class="o">=</span> <span class="s2">"Moving SMTP TLS report messages from"</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
||||
<span class="s2">"</span><span class="si">{0}</span><span class="s2"> </span><span class="si">{1}</span><span class="s2"> to </span><span class="si">{2}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">message</span><span class="p">,</span> <span class="n">reports_folder</span><span class="p">,</span> <span class="n">smtp_tls_reports_folder</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">message</span><span class="si">}</span><span class="s2"> </span><span class="si">{</span><span class="n">reports_folder</span><span class="si">}</span><span class="s2"> to </span><span class="si">{</span><span class="n">smtp_tls_reports_folder</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="n">number_of_smtp_tls_uids</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">smtp_tls_msg_uids</span><span class="p">)</span>
|
||||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">number_of_smtp_tls_uids</span><span class="p">):</span>
|
||||
<span class="n">msg_uid</span> <span class="o">=</span> <span class="n">smtp_tls_msg_uids</span><span class="p">[</span><span class="n">i</span><span class="p">]</span>
|
||||
<span class="n">message</span> <span class="o">=</span> <span class="s2">"Moving message"</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
||||
<span class="s2">"</span><span class="si">{0}</span><span class="s2"> </span><span class="si">{1}</span><span class="s2"> of </span><span class="si">{2}</span><span class="s2">: UID </span><span class="si">{3}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">message</span><span class="p">,</span> <span class="n">i</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="n">number_of_smtp_tls_uids</span><span class="p">,</span> <span class="n">msg_uid</span>
|
||||
<span class="p">)</span>
|
||||
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">message</span><span class="si">}</span><span class="s2"> </span><span class="si">{</span><span class="n">i</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="mi">1</span><span class="si">}</span><span class="s2"> of </span><span class="si">{</span><span class="n">number_of_smtp_tls_uids</span><span class="si">}</span><span class="s2">: UID </span><span class="si">{</span><span class="n">msg_uid</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">connection</span><span class="o">.</span><span class="n">move_message</span><span class="p">(</span><span class="n">msg_uid</span><span class="p">,</span> <span class="n">smtp_tls_reports_folder</span><span class="p">)</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">e</span> <span class="o">=</span> <span class="s2">"Error moving message UID </span><span class="si">{0}</span><span class="s2">: </span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">msg_uid</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Mailbox error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="p">))</span>
|
||||
<span class="n">e</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"Error moving message UID </span><span class="si">{</span><span class="n">msg_uid</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Mailbox error: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="n">results</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s2">"aggregate_reports"</span><span class="p">:</span> <span class="n">aggregate_reports</span><span class="p">,</span>
|
||||
<span class="s2">"failure_reports"</span><span class="p">:</span> <span class="n">failure_reports</span><span class="p">,</span>
|
||||
@@ -2888,7 +2841,7 @@
|
||||
|
||||
<span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">output_directory</span><span class="p">):</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isdir</span><span class="p">(</span><span class="n">output_directory</span><span class="p">):</span>
|
||||
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"</span><span class="si">{0}</span><span class="s2"> is not a directory"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">output_directory</span><span class="p">))</span>
|
||||
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">output_directory</span><span class="si">}</span><span class="s2"> is not a directory"</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">os</span><span class="o">.</span><span class="n">makedirs</span><span class="p">(</span><span class="n">output_directory</span><span class="p">)</span>
|
||||
|
||||
@@ -2935,11 +2888,11 @@
|
||||
|
||||
<span class="k">while</span> <span class="n">filename</span> <span class="ow">in</span> <span class="n">sample_filenames</span><span class="p">:</span>
|
||||
<span class="n">message_count</span> <span class="o">+=</span> <span class="mi">1</span>
|
||||
<span class="n">filename</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2"> (</span><span class="si">{1}</span><span class="s2">)"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">subject</span><span class="p">,</span> <span class="n">message_count</span><span class="p">)</span>
|
||||
<span class="n">filename</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">subject</span><span class="si">}</span><span class="s2"> (</span><span class="si">{</span><span class="n">message_count</span><span class="si">}</span><span class="s2">)"</span>
|
||||
|
||||
<span class="n">sample_filenames</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">filename</span><span class="p">)</span>
|
||||
|
||||
<span class="n">filename</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">.eml"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">filename</span><span class="p">)</span>
|
||||
<span class="n">filename</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">filename</span><span class="si">}</span><span class="s2">.eml"</span>
|
||||
<span class="n">path</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">samples_directory</span><span class="p">,</span> <span class="n">filename</span><span class="p">)</span>
|
||||
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="s2">"w"</span><span class="p">,</span> <span class="n">newline</span><span class="o">=</span><span class="s2">"</span><span class="se">\n</span><span class="s2">"</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s2">"utf-8"</span><span class="p">)</span> <span class="k">as</span> <span class="n">sample_file</span><span class="p">:</span>
|
||||
<span class="n">sample_file</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">sample</span><span class="p">)</span></div>
|
||||
@@ -3013,12 +2966,12 @@
|
||||
<span class="n">attachment_filename</span> <span class="o">+=</span> <span class="s2">".zip"</span>
|
||||
<span class="n">filename</span> <span class="o">=</span> <span class="n">attachment_filename</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">filename</span> <span class="o">=</span> <span class="s2">"DMARC-</span><span class="si">{0}</span><span class="s2">.zip"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">date_string</span><span class="p">)</span>
|
||||
<span class="n">filename</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"DMARC-</span><span class="si">{</span><span class="n">date_string</span><span class="si">}</span><span class="s2">.zip"</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">subject</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">subject</span> <span class="o">=</span> <span class="s2">"DMARC results for </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">date_string</span><span class="p">)</span>
|
||||
<span class="n">subject</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"DMARC results for </span><span class="si">{</span><span class="n">date_string</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="k">if</span> <span class="n">message</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">message</span> <span class="o">=</span> <span class="s2">"DMARC results for </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">date_string</span><span class="p">)</span>
|
||||
<span class="n">message</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"DMARC results for </span><span class="si">{</span><span class="n">date_string</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="n">zip_bytes</span> <span class="o">=</span> <span class="n">get_report_zip</span><span class="p">(</span><span class="n">results</span><span class="p">)</span>
|
||||
<span class="n">attachments</span> <span class="o">=</span> <span class="p">[(</span><span class="n">filename</span><span class="p">,</span> <span class="n">zip_bytes</span><span class="p">)]</span>
|
||||
|
||||
|
||||
@@ -5,14 +5,14 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc.elastic — parsedmarc 10.2.4 documentation</title>
|
||||
<title>parsedmarc.elastic — parsedmarc 10.3.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=9edc463e" />
|
||||
|
||||
|
||||
<script src="../../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../../_static/documentation_options.js?v=2c8206f1"></script>
|
||||
<script src="../../_static/documentation_options.js?v=a126560f"></script>
|
||||
<script src="../../_static/doctools.js?v=fd6eb6e6"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=6ffebe34"></script>
|
||||
<script src="../../_static/js/theme.js"></script>
|
||||
@@ -126,6 +126,181 @@
|
||||
<span class="c1"># settings (e.g. ``refresh_interval``) are accepted and pass through.</span>
|
||||
<span class="n">_SERVERLESS_REJECTED_SETTINGS</span> <span class="o">=</span> <span class="nb">frozenset</span><span class="p">({</span><span class="s2">"number_of_shards"</span><span class="p">,</span> <span class="s2">"number_of_replicas"</span><span class="p">})</span>
|
||||
|
||||
<span class="c1"># Guard query for the dkim_results_combined/spf_results_combined backfill</span>
|
||||
<span class="c1"># (see ``migrate_indexes``). Matches only documents that have at least one</span>
|
||||
<span class="c1"># DKIM or SPF auth result and are missing the corresponding combined field.</span>
|
||||
<span class="c1"># Empty arrays are invisible to ``exists``, so documents with zero</span>
|
||||
<span class="c1"># DKIM/SPF results are correctly skipped (verified against real data;</span>
|
||||
<span class="c1"># this also makes the query idempotent — a backfilled document no longer</span>
|
||||
<span class="c1"># matches). Each result is matched on an OR of its ``domain``/``result``</span>
|
||||
<span class="c1"># subfields as defense in depth: the parsers we audited never store a</span>
|
||||
<span class="c1"># result without both, but an empty string indexes no text tokens and is</span>
|
||||
<span class="c1"># invisible to ``exists``, and the storage shape of every historical</span>
|
||||
<span class="c1"># parsedmarc version can't be audited — matching either subfield costs</span>
|
||||
<span class="c1"># nothing and cannot skip a document that has something to backfill.</span>
|
||||
<span class="n">_COMBINED_BACKFILL_QUERY</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s2">"bool"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"minimum_should_match"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
|
||||
<span class="s2">"should"</span><span class="p">:</span> <span class="p">[</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s2">"bool"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"must"</span><span class="p">:</span> <span class="p">[</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s2">"bool"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"minimum_should_match"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
|
||||
<span class="s2">"should"</span><span class="p">:</span> <span class="p">[</span>
|
||||
<span class="p">{</span><span class="s2">"exists"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"field"</span><span class="p">:</span> <span class="s2">"dkim_results.domain"</span><span class="p">}},</span>
|
||||
<span class="p">{</span><span class="s2">"exists"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"field"</span><span class="p">:</span> <span class="s2">"dkim_results.result"</span><span class="p">}},</span>
|
||||
<span class="p">],</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">],</span>
|
||||
<span class="s2">"must_not"</span><span class="p">:</span> <span class="p">[{</span><span class="s2">"exists"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"field"</span><span class="p">:</span> <span class="s2">"dkim_results_combined"</span><span class="p">}}],</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">},</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s2">"bool"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"must"</span><span class="p">:</span> <span class="p">[</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s2">"bool"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"minimum_should_match"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
|
||||
<span class="s2">"should"</span><span class="p">:</span> <span class="p">[</span>
|
||||
<span class="p">{</span><span class="s2">"exists"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"field"</span><span class="p">:</span> <span class="s2">"spf_results.domain"</span><span class="p">}},</span>
|
||||
<span class="p">{</span><span class="s2">"exists"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"field"</span><span class="p">:</span> <span class="s2">"spf_results.result"</span><span class="p">}},</span>
|
||||
<span class="p">],</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">],</span>
|
||||
<span class="s2">"must_not"</span><span class="p">:</span> <span class="p">[{</span><span class="s2">"exists"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"field"</span><span class="p">:</span> <span class="s2">"spf_results_combined"</span><span class="p">}}],</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">},</span>
|
||||
<span class="p">],</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">}</span>
|
||||
|
||||
<span class="c1"># Painless script that (re)derives dkim_results_combined/spf_results_combined</span>
|
||||
<span class="c1"># from dkim_results/spf_results, matching the format written by</span>
|
||||
<span class="c1"># save_aggregate_report_to_elasticsearch(): "{selector} / {domain} / {result}"</span>
|
||||
<span class="c1"># per DKIM result and "{scope} / {domain} / {result}" per SPF result.</span>
|
||||
<span class="n">_COMBINED_BACKFILL_SCRIPT</span> <span class="o">=</span> <span class="p">(</span>
|
||||
<span class="s2">"List dk = new ArrayList(); "</span>
|
||||
<span class="s2">"def dr = ctx._source.dkim_results; "</span>
|
||||
<span class="s2">"if (dr != null) { "</span>
|
||||
<span class="s2">"if (!(dr instanceof List)) { dr = [dr]; } "</span>
|
||||
<span class="s2">"for (e in dr) { "</span>
|
||||
<span class="s2">"if (e == null) { continue; } "</span>
|
||||
<span class="s1">'def sel = e.selector != null ? e.selector : "none"; '</span>
|
||||
<span class="s1">'def dom = e.domain != null ? e.domain : "none"; '</span>
|
||||
<span class="s1">'def res = e.result != null ? e.result : "none"; '</span>
|
||||
<span class="s1">'dk.add(sel + " / " + dom + " / " + res); '</span>
|
||||
<span class="s2">"} } "</span>
|
||||
<span class="s2">"ctx._source.dkim_results_combined = dk; "</span>
|
||||
<span class="s2">"List sp = new ArrayList(); "</span>
|
||||
<span class="s2">"def sr = ctx._source.spf_results; "</span>
|
||||
<span class="s2">"if (sr != null) { "</span>
|
||||
<span class="s2">"if (!(sr instanceof List)) { sr = [sr]; } "</span>
|
||||
<span class="s2">"for (e in sr) { "</span>
|
||||
<span class="s2">"if (e == null) { continue; } "</span>
|
||||
<span class="s1">'def sc = e.scope != null ? e.scope : "mfrom"; '</span>
|
||||
<span class="s1">'def dom = e.domain != null ? e.domain : "none"; '</span>
|
||||
<span class="s1">'def res = e.result != null ? e.result : (e.results != null ? e.results : "none"); '</span>
|
||||
<span class="s1">'sp.add(sc + " / " + dom + " / " + res); '</span>
|
||||
<span class="s2">"} } "</span>
|
||||
<span class="s2">"ctx._source.spf_results_combined = sp;"</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="c1"># Guard query for the policies_combined/failure_details_combined backfill</span>
|
||||
<span class="c1"># (see ``migrate_indexes``). Matches only SMTP TLS documents that have at</span>
|
||||
<span class="c1"># least one policy or failure detail and are missing the corresponding</span>
|
||||
<span class="c1"># combined field. Empty arrays are invisible to ``exists``, so documents</span>
|
||||
<span class="c1"># with zero policies/failure details are correctly skipped (this also</span>
|
||||
<span class="c1"># makes the query idempotent — a backfilled document no longer matches).</span>
|
||||
<span class="c1"># Each result is matched on an OR of its relevant subfields as defense in</span>
|
||||
<span class="c1"># depth: the parsers we audited never store a policy/failure detail</span>
|
||||
<span class="c1"># without these fields, but an empty string indexes no text tokens and is</span>
|
||||
<span class="c1"># invisible to ``exists``, and the storage shape of every historical</span>
|
||||
<span class="c1"># parsedmarc version can't be audited — matching either subfield costs</span>
|
||||
<span class="c1"># nothing and cannot skip a document that has something to backfill.</span>
|
||||
<span class="n">_SMTP_TLS_COMBINED_BACKFILL_QUERY</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s2">"bool"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"minimum_should_match"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
|
||||
<span class="s2">"should"</span><span class="p">:</span> <span class="p">[</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s2">"bool"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"must"</span><span class="p">:</span> <span class="p">[</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s2">"bool"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"minimum_should_match"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
|
||||
<span class="s2">"should"</span><span class="p">:</span> <span class="p">[</span>
|
||||
<span class="p">{</span><span class="s2">"exists"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"field"</span><span class="p">:</span> <span class="s2">"policies.policy_domain"</span><span class="p">}},</span>
|
||||
<span class="p">{</span><span class="s2">"exists"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"field"</span><span class="p">:</span> <span class="s2">"policies.policy_type"</span><span class="p">}},</span>
|
||||
<span class="p">],</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">],</span>
|
||||
<span class="s2">"must_not"</span><span class="p">:</span> <span class="p">[{</span><span class="s2">"exists"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"field"</span><span class="p">:</span> <span class="s2">"policies_combined"</span><span class="p">}}],</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">},</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s2">"bool"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"must"</span><span class="p">:</span> <span class="p">[</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s2">"bool"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"minimum_should_match"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
|
||||
<span class="s2">"should"</span><span class="p">:</span> <span class="p">[</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s2">"exists"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"field"</span><span class="p">:</span> <span class="s2">"policies.failure_details.result_type"</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">},</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s2">"exists"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"field"</span><span class="p">:</span> <span class="s2">"policies.failure_details.sending_mta_ip"</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">},</span>
|
||||
<span class="p">],</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">],</span>
|
||||
<span class="s2">"must_not"</span><span class="p">:</span> <span class="p">[{</span><span class="s2">"exists"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"field"</span><span class="p">:</span> <span class="s2">"failure_details_combined"</span><span class="p">}}],</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">},</span>
|
||||
<span class="p">],</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">}</span>
|
||||
|
||||
<span class="c1"># Painless script that (re)derives policies_combined/failure_details_combined</span>
|
||||
<span class="c1"># from policies/policies.failure_details, matching the format written by</span>
|
||||
<span class="c1"># save_smtp_tls_report_to_elasticsearch(): "{policy_domain} / {policy_type}"</span>
|
||||
<span class="c1"># per policy and "{policy_domain} / {policy_type} / {result_type} /</span>
|
||||
<span class="c1"># {sending_mta_ip} / {receiving_ip} / {receiving_mx_hostname}" per failure</span>
|
||||
<span class="c1"># detail.</span>
|
||||
<span class="n">_SMTP_TLS_COMBINED_BACKFILL_SCRIPT</span> <span class="o">=</span> <span class="p">(</span>
|
||||
<span class="s2">"List pols = new ArrayList(); "</span>
|
||||
<span class="s2">"List dets = new ArrayList(); "</span>
|
||||
<span class="s2">"def ps = ctx._source.policies; "</span>
|
||||
<span class="s2">"if (ps != null) { "</span>
|
||||
<span class="s2">"if (!(ps instanceof List)) { ps = [ps]; } "</span>
|
||||
<span class="s2">"for (p in ps) { "</span>
|
||||
<span class="s2">"if (p == null) { continue; } "</span>
|
||||
<span class="s1">'def dom = p.policy_domain != null ? p.policy_domain : "none"; '</span>
|
||||
<span class="s1">'def typ = p.policy_type != null ? p.policy_type : "none"; '</span>
|
||||
<span class="s1">'pols.add(dom + " / " + typ); '</span>
|
||||
<span class="s2">"def fds = p.failure_details; "</span>
|
||||
<span class="s2">"if (fds != null) { "</span>
|
||||
<span class="s2">"if (!(fds instanceof List)) { fds = [fds]; } "</span>
|
||||
<span class="s2">"for (f in fds) { "</span>
|
||||
<span class="s2">"if (f == null) { continue; } "</span>
|
||||
<span class="s1">'def rt = f.result_type != null ? f.result_type : "none"; '</span>
|
||||
<span class="s1">'def smi = f.sending_mta_ip != null ? f.sending_mta_ip : "none"; '</span>
|
||||
<span class="s1">'def ri = f.receiving_ip != null ? f.receiving_ip : "none"; '</span>
|
||||
<span class="s1">'def rmh = f.receiving_mx_hostname != null ? f.receiving_mx_hostname : "none"; '</span>
|
||||
<span class="s1">'dets.add(dom + " / " + typ + " / " + rt + " / " + smi + " / " + ri + " / " + rmh); '</span>
|
||||
<span class="s2">"} } } } "</span>
|
||||
<span class="s2">"ctx._source.policies_combined = pols; "</span>
|
||||
<span class="s2">"ctx._source.failure_details_combined = dets;"</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_PolicyOverride</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="c1"># The elasticsearch.dsl 8.x type stubs use dataclass_transform and only</span>
|
||||
@@ -229,6 +404,11 @@
|
||||
<span class="n">header_from</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">envelope_from</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">envelope_to</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="c1"># Nested(...) on the two auth-result fields below is only the DSL's</span>
|
||||
<span class="c1"># in-memory document shape; it is never installed as a mapping.</span>
|
||||
<span class="c1"># create_indexes() deliberately skips Index.document() registration so</span>
|
||||
<span class="c1"># these fields stay dynamic-mapped as plain `object` in the cluster</span>
|
||||
<span class="c1"># (see the comment there and issue #169).</span>
|
||||
<span class="n">dkim_results</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_DKIMResult</span><span class="p">)</span>
|
||||
<span class="n">spf_results</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SPFResult</span><span class="p">)</span>
|
||||
<span class="c1"># One "{selector} / {domain} / {result}" (DKIM) or "{scope} / {domain} /</span>
|
||||
@@ -266,9 +446,7 @@
|
||||
<span class="n">human_result</span><span class="o">=</span><span class="n">human_result</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">dkim_results_combined</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="s2">"</span><span class="si">{0}</span><span class="s2"> / </span><span class="si">{1}</span><span class="s2"> / </span><span class="si">{2}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">selector</span><span class="p">,</span> <span class="n">domain</span><span class="p">,</span> <span class="n">result</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">dkim_results_combined</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">selector</span><span class="si">}</span><span class="s2"> / </span><span class="si">{</span><span class="n">domain</span><span class="si">}</span><span class="s2"> / </span><span class="si">{</span><span class="n">result</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_spf_result</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="p">,</span>
|
||||
@@ -285,9 +463,7 @@
|
||||
<span class="n">human_result</span><span class="o">=</span><span class="n">human_result</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">spf_results_combined</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="s2">"</span><span class="si">{0}</span><span class="s2"> / </span><span class="si">{1}</span><span class="s2"> / </span><span class="si">{2}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">scope</span><span class="p">,</span> <span class="n">domain</span><span class="p">,</span> <span class="n">result</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">spf_results_combined</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">scope</span><span class="si">}</span><span class="s2"> / </span><span class="si">{</span><span class="n">domain</span><span class="si">}</span><span class="s2"> / </span><span class="si">{</span><span class="n">result</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">save</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span> <span class="c1"># pyright: ignore[reportIncompatibleMethodOverride]</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">passed_dmarc</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
@@ -404,6 +580,7 @@
|
||||
<span class="n">result_type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">sending_mta_ip</span> <span class="o">=</span> <span class="n">Ip</span><span class="p">()</span>
|
||||
<span class="n">receiving_mx_helo</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">receiving_mx_hostname</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">receiving_ip</span> <span class="o">=</span> <span class="n">Ip</span><span class="p">()</span>
|
||||
<span class="n">failed_session_count</span> <span class="o">=</span> <span class="n">Integer</span><span class="p">()</span>
|
||||
<span class="n">additional_information_uri</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
@@ -445,7 +622,7 @@
|
||||
<span class="n">receiving_mx_helo</span><span class="o">=</span><span class="n">receiving_mx_helo</span><span class="p">,</span>
|
||||
<span class="n">receiving_ip</span><span class="o">=</span><span class="n">receiving_ip</span><span class="p">,</span>
|
||||
<span class="n">failed_session_count</span><span class="o">=</span><span class="n">failed_session_count</span><span class="p">,</span>
|
||||
<span class="n">additional_information</span><span class="o">=</span><span class="n">additional_information_uri</span><span class="p">,</span>
|
||||
<span class="n">additional_information_uri</span><span class="o">=</span><span class="n">additional_information_uri</span><span class="p">,</span>
|
||||
<span class="n">failure_reason_code</span><span class="o">=</span><span class="n">failure_reason_code</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">failure_details</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_details</span><span class="p">)</span>
|
||||
@@ -468,6 +645,19 @@
|
||||
<span class="n">contact_info</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">report_id</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">policies</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SMTPTLSPolicyDoc</span><span class="p">)</span>
|
||||
<span class="c1"># One "{policy_domain} / {policy_type}" string per policy. Kibana/</span>
|
||||
<span class="c1"># Grafana tables cannot terms-aggregate the subfields of an object</span>
|
||||
<span class="c1"># array without producing a cross-product of values (issue #169), so</span>
|
||||
<span class="c1"># dashboards aggregate these composed keywords instead. Declared to</span>
|
||||
<span class="c1"># match what dynamic mapping produces for a string array (text +</span>
|
||||
<span class="c1"># .keyword).</span>
|
||||
<span class="n">policies_combined</span> <span class="o">=</span> <span class="n">Text</span><span class="p">(</span><span class="n">multi</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">fields</span><span class="o">=</span><span class="p">{</span><span class="s2">"keyword"</span><span class="p">:</span> <span class="n">Keyword</span><span class="p">(</span><span class="n">ignore_above</span><span class="o">=</span><span class="mi">256</span><span class="p">)})</span>
|
||||
<span class="c1"># One "{policy_domain} / {policy_type} / {result_type} /</span>
|
||||
<span class="c1"># {sending_mta_ip} / {receiving_ip} / {receiving_mx_hostname}" string</span>
|
||||
<span class="c1"># per failure detail, across all policies.</span>
|
||||
<span class="n">failure_details_combined</span> <span class="o">=</span> <span class="n">Text</span><span class="p">(</span>
|
||||
<span class="n">multi</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">fields</span><span class="o">=</span><span class="p">{</span><span class="s2">"keyword"</span><span class="p">:</span> <span class="n">Keyword</span><span class="p">(</span><span class="n">ignore_above</span><span class="o">=</span><span class="mi">256</span><span class="p">)}</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="AlreadySaved">
|
||||
@@ -516,9 +706,7 @@
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">hosts</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
|
||||
<span class="n">hosts</span> <span class="o">=</span> <span class="p">[</span><span class="n">hosts</span><span class="p">]</span>
|
||||
<span class="n">scheme</span> <span class="o">=</span> <span class="s2">"https://"</span> <span class="k">if</span> <span class="n">use_ssl</span> <span class="k">else</span> <span class="s2">"http://"</span>
|
||||
<span class="n">normalized_hosts</span> <span class="o">=</span> <span class="p">[</span>
|
||||
<span class="n">host</span> <span class="k">if</span> <span class="s2">"://"</span> <span class="ow">in</span> <span class="n">host</span> <span class="k">else</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">scheme</span><span class="p">,</span> <span class="n">host</span><span class="p">)</span> <span class="k">for</span> <span class="n">host</span> <span class="ow">in</span> <span class="n">hosts</span>
|
||||
<span class="p">]</span>
|
||||
<span class="n">normalized_hosts</span> <span class="o">=</span> <span class="p">[</span><span class="n">host</span> <span class="k">if</span> <span class="s2">"://"</span> <span class="ow">in</span> <span class="n">host</span> <span class="k">else</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">scheme</span><span class="si">}{</span><span class="n">host</span><span class="si">}</span><span class="s2">"</span> <span class="k">for</span> <span class="n">host</span> <span class="ow">in</span> <span class="n">hosts</span><span class="p">]</span>
|
||||
<span class="n">conn_params</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"hosts"</span><span class="p">:</span> <span class="n">normalized_hosts</span><span class="p">,</span> <span class="s2">"request_timeout"</span><span class="p">:</span> <span class="n">timeout</span><span class="p">}</span>
|
||||
<span class="k">if</span> <span class="n">use_ssl</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">ssl_cert_path</span><span class="p">:</span>
|
||||
@@ -561,18 +749,25 @@
|
||||
<span class="k">for</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">names</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="n">Index</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="c1"># Deliberately no Index.document() registration: Kibana/OpenSearch</span>
|
||||
<span class="c1"># Dashboards/Grafana cannot terms-aggregate fields inside a</span>
|
||||
<span class="c1"># `nested` mapping, so the dynamic `object` mapping produced by a</span>
|
||||
<span class="c1"># bare create is load-bearing for the shipped dashboards (issue</span>
|
||||
<span class="c1"># #169; see the *_combined fields on _AggregateReportDoc).</span>
|
||||
<span class="c1"># Deliberately no Index.document() registration: the shipped</span>
|
||||
<span class="c1"># dashboards cannot rebuild their detail tables on a `nested`</span>
|
||||
<span class="c1"># mapping — Kibana/OSD visual editors do not support nested</span>
|
||||
<span class="c1"># fields, Vega can run nested aggregations but does not render</span>
|
||||
<span class="c1"># tables, and Grafana's nested bucket aggregation (9.4+) lacks</span>
|
||||
<span class="c1"># reverse_nested for parent-level metrics like message_count —</span>
|
||||
<span class="c1"># so the dynamic `object` mapping produced by a bare create is</span>
|
||||
<span class="c1"># load-bearing for the shipped dashboards. _AggregateReportDoc</span>
|
||||
<span class="c1"># still declares dkim_results/spf_results with Nested(...), but</span>
|
||||
<span class="c1"># that is only the DSL's in-memory shape for building documents</span>
|
||||
<span class="c1"># — it is never installed as a mapping. See issue #169 and the</span>
|
||||
<span class="c1"># *_combined fields on _AggregateReportDoc.</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">index</span><span class="o">.</span><span class="n">exists</span><span class="p">():</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Creating Elasticsearch index: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">name</span><span class="p">))</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Creating Elasticsearch index: </span><span class="si">{</span><span class="n">name</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">effective_settings</span><span class="p">:</span>
|
||||
<span class="n">index</span><span class="o">.</span><span class="n">settings</span><span class="p">(</span><span class="o">**</span><span class="n">effective_settings</span><span class="p">)</span>
|
||||
<span class="n">index</span><span class="o">.</span><span class="n">create</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="s2">"Elasticsearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Elasticsearch error: </span><span class="si">{</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
|
||||
@@ -581,25 +776,118 @@
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">migrate_indexes</span><span class="p">(</span>
|
||||
<span class="n">aggregate_indexes</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">failure_indexes</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">smtp_tls_indexes</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Updates index mappings</span>
|
||||
<span class="sd"> Backfills the ``dkim_results_combined``/``spf_results_combined`` fields</span>
|
||||
<span class="sd"> (added for issue #169) on aggregate report documents, and the</span>
|
||||
<span class="sd"> ``policies_combined``/``failure_details_combined`` fields on SMTP TLS</span>
|
||||
<span class="sd"> report documents, that were saved before those fields existed.</span>
|
||||
|
||||
<span class="sd"> This is a no-op kept for API compatibility (``cli.py`` calls it on</span>
|
||||
<span class="sd"> startup). The only migration this function ever performed was</span>
|
||||
<span class="sd"> re-typing ``published_policy.fo`` from ``long`` to ``text``, which</span>
|
||||
<span class="sd"> applied exclusively to indices still carrying the legacy</span>
|
||||
<span class="sd"> Elasticsearch 6-era ``"doc"`` mapping type. The 8.x client can only</span>
|
||||
<span class="sd"> reach servers (Elasticsearch 8.x/9.x) whose indices were created on</span>
|
||||
<span class="sd"> Elasticsearch 7.x or later and are therefore typeless, so that</span>
|
||||
<span class="sd"> migration path is unreachable and has been removed.</span>
|
||||
<span class="sd"> For each name in ``aggregate_indexes``/``smtp_tls_indexes``, this</span>
|
||||
<span class="sd"> submits an ``update_by_query`` against the ``f"{name}*"`` index pattern</span>
|
||||
<span class="sd"> (the real indexes are date-suffixed) as a non-blocking background task</span>
|
||||
<span class="sd"> (``wait_for_completion=False``), so it never delays parsedmarc startup.</span>
|
||||
<span class="sd"> Submission is guarded by a cheap ``count`` query that only matches</span>
|
||||
<span class="sd"> documents with DKIM/SPF results (or policies/failure details) but no</span>
|
||||
<span class="sd"> combined field, so once an index is fully backfilled, later calls are a</span>
|
||||
<span class="sd"> fast no-op. Any error talking to the cluster (e.g. no indexes yet on a</span>
|
||||
<span class="sd"> fresh install, or a transient connection issue) is caught and logged as</span>
|
||||
<span class="sd"> a warning rather than raised; the backfill is simply retried on the</span>
|
||||
<span class="sd"> next startup, and the manual ``_update_by_query`` commands documented in</span>
|
||||
<span class="sd"> ``docs/source/elasticsearch.md`` remain available in the meantime.</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> aggregate_indexes (list): A list of aggregate index names</span>
|
||||
<span class="sd"> (accepted for API compatibility; unused)</span>
|
||||
<span class="sd"> failure_indexes (list): A list of failure index names</span>
|
||||
<span class="sd"> (accepted for API compatibility; unused)</span>
|
||||
<span class="sd"> """</span></div>
|
||||
<span class="sd"> smtp_tls_indexes (list): A list of SMTP TLS index names</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">aggregate_indexes</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">smtp_tls_indexes</span><span class="p">:</span>
|
||||
<span class="k">return</span>
|
||||
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">client</span> <span class="o">=</span> <span class="n">connections</span><span class="o">.</span><span class="n">get_connection</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
||||
<span class="s2">"Skipping the dkim_results_combined/spf_results_combined/"</span>
|
||||
<span class="s2">"policies_combined/failure_details_combined backfill: could "</span>
|
||||
<span class="sa">f</span><span class="s2">"not get an Elasticsearch connection: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">. This will be "</span>
|
||||
<span class="s2">"retried at the next startup."</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">return</span>
|
||||
<span class="k">for</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">aggregate_indexes</span> <span class="ow">or</span> <span class="p">[]:</span>
|
||||
<span class="n">pattern</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">name</span><span class="si">}</span><span class="s2">*"</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">count_response</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">count</span><span class="p">(</span>
|
||||
<span class="n">index</span><span class="o">=</span><span class="n">pattern</span><span class="p">,</span>
|
||||
<span class="n">query</span><span class="o">=</span><span class="n">_COMBINED_BACKFILL_QUERY</span><span class="p">,</span>
|
||||
<span class="n">ignore_unavailable</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="n">allow_no_indices</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">count</span> <span class="o">=</span> <span class="n">count_response</span><span class="p">[</span><span class="s2">"count"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">count</span><span class="p">:</span>
|
||||
<span class="k">continue</span>
|
||||
<span class="n">update_response</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">update_by_query</span><span class="p">(</span>
|
||||
<span class="n">index</span><span class="o">=</span><span class="n">pattern</span><span class="p">,</span>
|
||||
<span class="n">query</span><span class="o">=</span><span class="n">_COMBINED_BACKFILL_QUERY</span><span class="p">,</span>
|
||||
<span class="n">script</span><span class="o">=</span><span class="p">{</span><span class="s2">"source"</span><span class="p">:</span> <span class="n">_COMBINED_BACKFILL_SCRIPT</span><span class="p">,</span> <span class="s2">"lang"</span><span class="p">:</span> <span class="s2">"painless"</span><span class="p">},</span>
|
||||
<span class="n">conflicts</span><span class="o">=</span><span class="s2">"proceed"</span><span class="p">,</span>
|
||||
<span class="n">wait_for_completion</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">ignore_unavailable</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="n">allow_no_indices</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">task_id</span> <span class="o">=</span> <span class="n">update_response</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"task"</span><span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
|
||||
<span class="s2">"Backfilling dkim_results_combined/spf_results_combined on "</span>
|
||||
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">count</span><span class="si">}</span><span class="s2"> existing documents in </span><span class="si">{</span><span class="n">pattern</span><span class="si">}</span><span class="s2"> (task </span><span class="si">{</span><span class="n">task_id</span><span class="si">}</span><span class="s2">)"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
||||
<span class="s2">"Failed to check/submit the dkim_results_combined/"</span>
|
||||
<span class="sa">f</span><span class="s2">"spf_results_combined backfill for </span><span class="si">{</span><span class="n">pattern</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">. This "</span>
|
||||
<span class="s2">"will be retried at the next startup; the manual "</span>
|
||||
<span class="s2">"_update_by_query command in the documentation remains "</span>
|
||||
<span class="s2">"available in the meantime."</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">smtp_tls_indexes</span> <span class="ow">or</span> <span class="p">[]:</span>
|
||||
<span class="n">pattern</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">name</span><span class="si">}</span><span class="s2">*"</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">count_response</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">count</span><span class="p">(</span>
|
||||
<span class="n">index</span><span class="o">=</span><span class="n">pattern</span><span class="p">,</span>
|
||||
<span class="n">query</span><span class="o">=</span><span class="n">_SMTP_TLS_COMBINED_BACKFILL_QUERY</span><span class="p">,</span>
|
||||
<span class="n">ignore_unavailable</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="n">allow_no_indices</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">count</span> <span class="o">=</span> <span class="n">count_response</span><span class="p">[</span><span class="s2">"count"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">count</span><span class="p">:</span>
|
||||
<span class="k">continue</span>
|
||||
<span class="n">update_response</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">update_by_query</span><span class="p">(</span>
|
||||
<span class="n">index</span><span class="o">=</span><span class="n">pattern</span><span class="p">,</span>
|
||||
<span class="n">query</span><span class="o">=</span><span class="n">_SMTP_TLS_COMBINED_BACKFILL_QUERY</span><span class="p">,</span>
|
||||
<span class="n">script</span><span class="o">=</span><span class="p">{</span>
|
||||
<span class="s2">"source"</span><span class="p">:</span> <span class="n">_SMTP_TLS_COMBINED_BACKFILL_SCRIPT</span><span class="p">,</span>
|
||||
<span class="s2">"lang"</span><span class="p">:</span> <span class="s2">"painless"</span><span class="p">,</span>
|
||||
<span class="p">},</span>
|
||||
<span class="n">conflicts</span><span class="o">=</span><span class="s2">"proceed"</span><span class="p">,</span>
|
||||
<span class="n">wait_for_completion</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">ignore_unavailable</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="n">allow_no_indices</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">task_id</span> <span class="o">=</span> <span class="n">update_response</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"task"</span><span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
|
||||
<span class="s2">"Backfilling policies_combined/failure_details_combined on "</span>
|
||||
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">count</span><span class="si">}</span><span class="s2"> existing documents in </span><span class="si">{</span><span class="n">pattern</span><span class="si">}</span><span class="s2"> (task </span><span class="si">{</span><span class="n">task_id</span><span class="si">}</span><span class="s2">)"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
||||
<span class="s2">"Failed to check/submit the policies_combined/"</span>
|
||||
<span class="sa">f</span><span class="s2">"failure_details_combined backfill for </span><span class="si">{</span><span class="n">pattern</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">. "</span>
|
||||
<span class="s2">"This will be retried at the next startup; the manual "</span>
|
||||
<span class="s2">"_update_by_query command in the documentation remains "</span>
|
||||
<span class="s2">"available in the meantime."</span>
|
||||
<span class="p">)</span></div>
|
||||
|
||||
|
||||
|
||||
@@ -648,11 +936,11 @@
|
||||
<span class="n">end_date_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="nb">range</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">date_end</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">lte</span><span class="o">=</span><span class="n">end_date</span><span class="p">))))</span> <span class="c1"># pyright: ignore[reportArgumentType]</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">index_suffix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"dmarc_aggregate_</span><span class="si">{0}</span><span class="s2">*"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"dmarc_aggregate_</span><span class="si">{</span><span class="n">index_suffix</span><span class="si">}</span><span class="s2">*"</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"dmarc_aggregate*"</span>
|
||||
<span class="k">if</span> <span class="n">index_prefix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">search_index</span><span class="p">)</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">index_prefix</span><span class="si">}{</span><span class="n">search_index</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="n">search_index</span><span class="p">)</span>
|
||||
<span class="n">query</span> <span class="o">=</span> <span class="n">org_name_query</span> <span class="o">&</span> <span class="n">report_id_query</span> <span class="o">&</span> <span class="n">domain_query</span>
|
||||
<span class="n">query</span> <span class="o">=</span> <span class="n">query</span> <span class="o">&</span> <span class="n">begin_date_query</span> <span class="o">&</span> <span class="n">end_date_query</span>
|
||||
@@ -667,18 +955,15 @@
|
||||
<span class="n">existing</span> <span class="o">=</span> <span class="n">search</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error_</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span>
|
||||
<span class="s2">"Elasticsearch's search for existing report </span><span class="se">\</span>
|
||||
<span class="s2"> error: </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error_</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="sa">f</span><span class="s2">"Elasticsearch's search for existing report error: </span><span class="si">{</span><span class="n">error_</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">existing</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">AlreadySaved</span><span class="p">(</span>
|
||||
<span class="s2">"An aggregate report ID </span><span class="si">{0}</span><span class="s2"> from </span><span class="si">{1}</span><span class="s2"> about </span><span class="si">{2}</span><span class="s2"> "</span>
|
||||
<span class="s2">"with a date range of </span><span class="si">{3}</span><span class="s2"> UTC to </span><span class="si">{4}</span><span class="s2"> UTC already "</span>
|
||||
<span class="sa">f</span><span class="s2">"An aggregate report ID </span><span class="si">{</span><span class="n">report_id</span><span class="si">}</span><span class="s2"> from </span><span class="si">{</span><span class="n">org_name</span><span class="si">}</span><span class="s2"> about </span><span class="si">{</span><span class="n">domain</span><span class="si">}</span><span class="s2"> "</span>
|
||||
<span class="sa">f</span><span class="s2">"with a date range of </span><span class="si">{</span><span class="n">begin_date_human</span><span class="si">}</span><span class="s2"> UTC to </span><span class="si">{</span><span class="n">end_date_human</span><span class="si">}</span><span class="s2"> UTC already "</span>
|
||||
<span class="s2">"exists in "</span>
|
||||
<span class="s2">"Elasticsearch"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">report_id</span><span class="p">,</span> <span class="n">org_name</span><span class="p">,</span> <span class="n">domain</span><span class="p">,</span> <span class="n">begin_date_human</span><span class="p">,</span> <span class="n">end_date_human</span>
|
||||
<span class="p">)</span>
|
||||
<span class="s2">"Elasticsearch"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">published_policy</span> <span class="o">=</span> <span class="n">_PublishedPolicy</span><span class="p">(</span>
|
||||
<span class="n">domain</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"domain"</span><span class="p">],</span>
|
||||
@@ -771,11 +1056,11 @@
|
||||
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"dmarc_aggregate"</span>
|
||||
<span class="k">if</span> <span class="n">index_suffix</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">_</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">index</span><span class="si">}</span><span class="s2">_</span><span class="si">{</span><span class="n">index_suffix</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="k">if</span> <span class="n">index_prefix</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">index</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">index_prefix</span><span class="si">}{</span><span class="n">index</span><span class="si">}</span><span class="s2">"</span>
|
||||
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">-</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_date</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">index</span><span class="si">}</span><span class="s2">-</span><span class="si">{</span><span class="n">index_date</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span> <span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span>
|
||||
<span class="p">)</span>
|
||||
@@ -785,7 +1070,7 @@
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">agg_doc</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="s2">"Elasticsearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Elasticsearch error: </span><span class="si">{</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
|
||||
@@ -836,12 +1121,12 @@
|
||||
<span class="n">arrival_date_epoch_milliseconds</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">arrival_date</span><span class="o">.</span><span class="n">timestamp</span><span class="p">()</span> <span class="o">*</span> <span class="mi">1000</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">index_suffix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"dmarc_failure_</span><span class="si">{0}</span><span class="s2">*,dmarc_forensic_</span><span class="si">{0}</span><span class="s2">*"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"dmarc_failure_</span><span class="si">{</span><span class="n">index_suffix</span><span class="si">}</span><span class="s2">*,dmarc_forensic_</span><span class="si">{</span><span class="n">index_suffix</span><span class="si">}</span><span class="s2">*"</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"dmarc_failure*,dmarc_forensic*"</span>
|
||||
<span class="k">if</span> <span class="n">index_prefix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">","</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
|
||||
<span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">part</span><span class="p">)</span> <span class="k">for</span> <span class="n">part</span> <span class="ow">in</span> <span class="n">search_index</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">","</span><span class="p">)</span>
|
||||
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">index_prefix</span><span class="si">}{</span><span class="n">part</span><span class="si">}</span><span class="s2">"</span> <span class="k">for</span> <span class="n">part</span> <span class="ow">in</span> <span class="n">search_index</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">","</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="n">search_index</span><span class="p">)</span>
|
||||
<span class="n">q</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">arrival_date</span><span class="o">=</span><span class="n">arrival_date_epoch_milliseconds</span><span class="p">)))</span> <span class="c1"># pyright: ignore[reportArgumentType]</span>
|
||||
@@ -893,8 +1178,8 @@
|
||||
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">existing</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">AlreadySaved</span><span class="p">(</span>
|
||||
<span class="s2">"A failure sample to </span><span class="si">{0}</span><span class="s2"> from </span><span class="si">{1}</span><span class="s2"> "</span>
|
||||
<span class="s2">"with a subject of </span><span class="si">{2}</span><span class="s2"> and arrival date of </span><span class="si">{3}</span><span class="s2"> "</span>
|
||||
<span class="s2">"A failure sample to </span><span class="si">{}</span><span class="s2"> from </span><span class="si">{}</span><span class="s2"> "</span>
|
||||
<span class="s2">"with a subject of </span><span class="si">{}</span><span class="s2"> and arrival date of </span><span class="si">{}</span><span class="s2"> "</span>
|
||||
<span class="s2">"already exists in "</span>
|
||||
<span class="s2">"Elasticsearch"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">to_</span><span class="p">,</span> <span class="n">from_</span><span class="p">,</span> <span class="n">subject</span><span class="p">,</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"arrival_date_utc"</span><span class="p">]</span>
|
||||
@@ -955,14 +1240,14 @@
|
||||
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"dmarc_failure"</span>
|
||||
<span class="k">if</span> <span class="n">index_suffix</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">_</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">index</span><span class="si">}</span><span class="s2">_</span><span class="si">{</span><span class="n">index_suffix</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="k">if</span> <span class="n">index_prefix</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">index</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">index_prefix</span><span class="si">}{</span><span class="n">index</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="k">if</span> <span class="n">monthly_indexes</span><span class="p">:</span>
|
||||
<span class="n">index_date</span> <span class="o">=</span> <span class="n">arrival_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m"</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">index_date</span> <span class="o">=</span> <span class="n">arrival_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">-</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_date</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">index</span><span class="si">}</span><span class="s2">-</span><span class="si">{</span><span class="n">index_date</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span> <span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span>
|
||||
<span class="p">)</span>
|
||||
@@ -971,10 +1256,10 @@
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">failure_doc</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="s2">"Elasticsearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Elasticsearch error: </span><span class="si">{</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="k">except</span> <span class="ne">KeyError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidFailureReport</span><span class="p">(</span>
|
||||
<span class="s2">"Failure report missing required field: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="sa">f</span><span class="s2">"Failure report missing required field: </span><span class="si">{</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="p">)</span></div>
|
||||
|
||||
|
||||
@@ -1024,11 +1309,11 @@
|
||||
<span class="n">end_date_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">date_end</span><span class="o">=</span><span class="n">end_date</span><span class="p">)))</span> <span class="c1"># pyright: ignore[reportArgumentType]</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">index_suffix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"smtp_tls_</span><span class="si">{0}</span><span class="s2">*"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"smtp_tls_</span><span class="si">{</span><span class="n">index_suffix</span><span class="si">}</span><span class="s2">*"</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"smtp_tls*"</span>
|
||||
<span class="k">if</span> <span class="n">index_prefix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">search_index</span><span class="p">)</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">index_prefix</span><span class="si">}{</span><span class="n">search_index</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="n">search_index</span><span class="p">)</span>
|
||||
<span class="n">query</span> <span class="o">=</span> <span class="n">org_name_query</span> <span class="o">&</span> <span class="n">report_id_query</span>
|
||||
<span class="n">query</span> <span class="o">=</span> <span class="n">query</span> <span class="o">&</span> <span class="n">begin_date_query</span> <span class="o">&</span> <span class="n">end_date_query</span>
|
||||
@@ -1038,8 +1323,7 @@
|
||||
<span class="n">existing</span> <span class="o">=</span> <span class="n">search</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error_</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span>
|
||||
<span class="s2">"Elasticsearch's search for existing report </span><span class="se">\</span>
|
||||
<span class="s2"> error: </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error_</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="sa">f</span><span class="s2">"Elasticsearch's search for existing report error: </span><span class="si">{</span><span class="n">error_</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">existing</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||
@@ -1053,10 +1337,10 @@
|
||||
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"smtp_tls"</span>
|
||||
<span class="k">if</span> <span class="n">index_suffix</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">_</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">index</span><span class="si">}</span><span class="s2">_</span><span class="si">{</span><span class="n">index_suffix</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="k">if</span> <span class="n">index_prefix</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">index</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">-</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_date</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">index_prefix</span><span class="si">}{</span><span class="n">index</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">index</span><span class="si">}</span><span class="s2">-</span><span class="si">{</span><span class="n">index_date</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span> <span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span>
|
||||
<span class="p">)</span>
|
||||
@@ -1077,6 +1361,16 @@
|
||||
<span class="n">policy_strings</span> <span class="o">=</span> <span class="n">policy</span><span class="p">[</span><span class="s2">"policy_strings"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="s2">"mx_host_patterns"</span> <span class="ow">in</span> <span class="n">policy</span><span class="p">:</span>
|
||||
<span class="n">mx_host_patterns</span> <span class="o">=</span> <span class="n">policy</span><span class="p">[</span><span class="s2">"mx_host_patterns"</span><span class="p">]</span>
|
||||
<span class="c1"># policies_combined/failure_details_combined: see the field</span>
|
||||
<span class="c1"># declarations on _SMTPTLSReportDoc and issue #169. policies and</span>
|
||||
<span class="c1"># their failure_details are object arrays with the same</span>
|
||||
<span class="c1"># cross-product problem as dkim_results/spf_results, so dashboards</span>
|
||||
<span class="c1"># aggregate these composed strings instead of the raw subfields.</span>
|
||||
<span class="n">policy_domain_combined</span> <span class="o">=</span> <span class="n">policy</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"policy_domain"</span><span class="p">)</span> <span class="ow">or</span> <span class="s2">"none"</span>
|
||||
<span class="n">policy_type_combined</span> <span class="o">=</span> <span class="n">policy</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"policy_type"</span><span class="p">)</span> <span class="ow">or</span> <span class="s2">"none"</span>
|
||||
<span class="n">smtp_tls_doc</span><span class="o">.</span><span class="n">policies_combined</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">policy_domain_combined</span><span class="si">}</span><span class="s2"> / </span><span class="si">{</span><span class="n">policy_type_combined</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">policy_doc</span> <span class="o">=</span> <span class="n">_SMTPTLSPolicyDoc</span><span class="p">(</span>
|
||||
<span class="n">policy_domain</span><span class="o">=</span><span class="n">policy</span><span class="p">[</span><span class="s2">"policy_domain"</span><span class="p">],</span>
|
||||
<span class="n">policy_type</span><span class="o">=</span><span class="n">policy</span><span class="p">[</span><span class="s2">"policy_type"</span><span class="p">],</span>
|
||||
@@ -1097,7 +1391,12 @@
|
||||
|
||||
<span class="k">if</span> <span class="s2">"receiving_mx_hostname"</span> <span class="ow">in</span> <span class="n">failure_detail</span><span class="p">:</span>
|
||||
<span class="n">receiving_mx_hostname</span> <span class="o">=</span> <span class="n">failure_detail</span><span class="p">[</span><span class="s2">"receiving_mx_hostname"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="s2">"additional_information_uri"</span> <span class="ow">in</span> <span class="n">failure_detail</span><span class="p">:</span>
|
||||
<span class="c1"># The parser's key is additional_info_uri (see</span>
|
||||
<span class="c1"># SMTPTLSFailureDetailsOptional in types.py); accept the</span>
|
||||
<span class="c1"># long-form key too for dicts built by other callers.</span>
|
||||
<span class="k">if</span> <span class="s2">"additional_info_uri"</span> <span class="ow">in</span> <span class="n">failure_detail</span><span class="p">:</span>
|
||||
<span class="n">additional_information_uri</span> <span class="o">=</span> <span class="n">failure_detail</span><span class="p">[</span><span class="s2">"additional_info_uri"</span><span class="p">]</span>
|
||||
<span class="k">elif</span> <span class="s2">"additional_information_uri"</span> <span class="ow">in</span> <span class="n">failure_detail</span><span class="p">:</span>
|
||||
<span class="n">additional_information_uri</span> <span class="o">=</span> <span class="n">failure_detail</span><span class="p">[</span>
|
||||
<span class="s2">"additional_information_uri"</span>
|
||||
<span class="p">]</span>
|
||||
@@ -1122,6 +1421,16 @@
|
||||
<span class="n">additional_information_uri</span><span class="o">=</span><span class="n">additional_information_uri</span><span class="p">,</span>
|
||||
<span class="n">failure_reason_code</span><span class="o">=</span><span class="n">failure_reason_code</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">smtp_tls_doc</span><span class="o">.</span><span class="n">failure_details_combined</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="s2">"</span><span class="si">{}</span><span class="s2"> / </span><span class="si">{}</span><span class="s2"> / </span><span class="si">{}</span><span class="s2"> / </span><span class="si">{}</span><span class="s2"> / </span><span class="si">{}</span><span class="s2"> / </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">policy_domain_combined</span><span class="p">,</span>
|
||||
<span class="n">policy_type_combined</span><span class="p">,</span>
|
||||
<span class="n">failure_detail</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"result_type"</span><span class="p">)</span> <span class="ow">or</span> <span class="s2">"none"</span><span class="p">,</span>
|
||||
<span class="n">sending_mta_ip</span> <span class="ow">or</span> <span class="s2">"none"</span><span class="p">,</span>
|
||||
<span class="n">receiving_ip</span> <span class="ow">or</span> <span class="s2">"none"</span><span class="p">,</span>
|
||||
<span class="n">receiving_mx_hostname</span> <span class="ow">or</span> <span class="s2">"none"</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">smtp_tls_doc</span><span class="o">.</span><span class="n">policies</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">policy_doc</span><span class="p">)</span>
|
||||
|
||||
<span class="n">create_indexes</span><span class="p">([</span><span class="n">index</span><span class="p">],</span> <span class="n">index_settings</span><span class="p">)</span>
|
||||
@@ -1130,7 +1439,7 @@
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">smtp_tls_doc</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="s2">"Elasticsearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Elasticsearch error: </span><span class="si">{</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -5,14 +5,14 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc.opensearch — parsedmarc 10.2.4 documentation</title>
|
||||
<title>parsedmarc.opensearch — parsedmarc 10.3.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=9edc463e" />
|
||||
|
||||
|
||||
<script src="../../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../../_static/documentation_options.js?v=2c8206f1"></script>
|
||||
<script src="../../_static/documentation_options.js?v=a126560f"></script>
|
||||
<script src="../../_static/doctools.js?v=fd6eb6e6"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=6ffebe34"></script>
|
||||
<script src="../../_static/js/theme.js"></script>
|
||||
@@ -119,6 +119,182 @@
|
||||
|
||||
|
||||
|
||||
<span class="c1"># Guard query for the dkim_results_combined/spf_results_combined backfill</span>
|
||||
<span class="c1"># (see ``migrate_indexes``). Matches only documents that have at least one</span>
|
||||
<span class="c1"># DKIM or SPF auth result and are missing the corresponding combined field.</span>
|
||||
<span class="c1"># Empty arrays are invisible to ``exists``, so documents with zero</span>
|
||||
<span class="c1"># DKIM/SPF results are correctly skipped (verified against real data;</span>
|
||||
<span class="c1"># this also makes the query idempotent — a backfilled document no longer</span>
|
||||
<span class="c1"># matches). Each result is matched on an OR of its ``domain``/``result``</span>
|
||||
<span class="c1"># subfields as defense in depth: the parsers we audited never store a</span>
|
||||
<span class="c1"># result without both, but an empty string indexes no text tokens and is</span>
|
||||
<span class="c1"># invisible to ``exists``, and the storage shape of every historical</span>
|
||||
<span class="c1"># parsedmarc version can't be audited — matching either subfield costs</span>
|
||||
<span class="c1"># nothing and cannot skip a document that has something to backfill.</span>
|
||||
<span class="n">_COMBINED_BACKFILL_QUERY</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s2">"bool"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"minimum_should_match"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
|
||||
<span class="s2">"should"</span><span class="p">:</span> <span class="p">[</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s2">"bool"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"must"</span><span class="p">:</span> <span class="p">[</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s2">"bool"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"minimum_should_match"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
|
||||
<span class="s2">"should"</span><span class="p">:</span> <span class="p">[</span>
|
||||
<span class="p">{</span><span class="s2">"exists"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"field"</span><span class="p">:</span> <span class="s2">"dkim_results.domain"</span><span class="p">}},</span>
|
||||
<span class="p">{</span><span class="s2">"exists"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"field"</span><span class="p">:</span> <span class="s2">"dkim_results.result"</span><span class="p">}},</span>
|
||||
<span class="p">],</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">],</span>
|
||||
<span class="s2">"must_not"</span><span class="p">:</span> <span class="p">[{</span><span class="s2">"exists"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"field"</span><span class="p">:</span> <span class="s2">"dkim_results_combined"</span><span class="p">}}],</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">},</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s2">"bool"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"must"</span><span class="p">:</span> <span class="p">[</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s2">"bool"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"minimum_should_match"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
|
||||
<span class="s2">"should"</span><span class="p">:</span> <span class="p">[</span>
|
||||
<span class="p">{</span><span class="s2">"exists"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"field"</span><span class="p">:</span> <span class="s2">"spf_results.domain"</span><span class="p">}},</span>
|
||||
<span class="p">{</span><span class="s2">"exists"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"field"</span><span class="p">:</span> <span class="s2">"spf_results.result"</span><span class="p">}},</span>
|
||||
<span class="p">],</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">],</span>
|
||||
<span class="s2">"must_not"</span><span class="p">:</span> <span class="p">[{</span><span class="s2">"exists"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"field"</span><span class="p">:</span> <span class="s2">"spf_results_combined"</span><span class="p">}}],</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">},</span>
|
||||
<span class="p">],</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">}</span>
|
||||
|
||||
<span class="c1"># Painless script that (re)derives dkim_results_combined/spf_results_combined</span>
|
||||
<span class="c1"># from dkim_results/spf_results, matching the format written by</span>
|
||||
<span class="c1"># save_aggregate_report_to_opensearch(): "{selector} / {domain} / {result}"</span>
|
||||
<span class="c1"># per DKIM result and "{scope} / {domain} / {result}" per SPF result.</span>
|
||||
<span class="n">_COMBINED_BACKFILL_SCRIPT</span> <span class="o">=</span> <span class="p">(</span>
|
||||
<span class="s2">"List dk = new ArrayList(); "</span>
|
||||
<span class="s2">"def dr = ctx._source.dkim_results; "</span>
|
||||
<span class="s2">"if (dr != null) { "</span>
|
||||
<span class="s2">"if (!(dr instanceof List)) { dr = [dr]; } "</span>
|
||||
<span class="s2">"for (e in dr) { "</span>
|
||||
<span class="s2">"if (e == null) { continue; } "</span>
|
||||
<span class="s1">'def sel = e.selector != null ? e.selector : "none"; '</span>
|
||||
<span class="s1">'def dom = e.domain != null ? e.domain : "none"; '</span>
|
||||
<span class="s1">'def res = e.result != null ? e.result : "none"; '</span>
|
||||
<span class="s1">'dk.add(sel + " / " + dom + " / " + res); '</span>
|
||||
<span class="s2">"} } "</span>
|
||||
<span class="s2">"ctx._source.dkim_results_combined = dk; "</span>
|
||||
<span class="s2">"List sp = new ArrayList(); "</span>
|
||||
<span class="s2">"def sr = ctx._source.spf_results; "</span>
|
||||
<span class="s2">"if (sr != null) { "</span>
|
||||
<span class="s2">"if (!(sr instanceof List)) { sr = [sr]; } "</span>
|
||||
<span class="s2">"for (e in sr) { "</span>
|
||||
<span class="s2">"if (e == null) { continue; } "</span>
|
||||
<span class="s1">'def sc = e.scope != null ? e.scope : "mfrom"; '</span>
|
||||
<span class="s1">'def dom = e.domain != null ? e.domain : "none"; '</span>
|
||||
<span class="s1">'def res = e.result != null ? e.result : (e.results != null ? e.results : "none"); '</span>
|
||||
<span class="s1">'sp.add(sc + " / " + dom + " / " + res); '</span>
|
||||
<span class="s2">"} } "</span>
|
||||
<span class="s2">"ctx._source.spf_results_combined = sp;"</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="c1"># Guard query for the policies_combined/failure_details_combined backfill</span>
|
||||
<span class="c1"># (see ``migrate_indexes``). Matches only SMTP TLS documents that have at</span>
|
||||
<span class="c1"># least one policy or failure detail and are missing the corresponding</span>
|
||||
<span class="c1"># combined field. Empty arrays are invisible to ``exists``, so documents</span>
|
||||
<span class="c1"># with zero policies/failure details are correctly skipped (this also</span>
|
||||
<span class="c1"># makes the query idempotent — a backfilled document no longer matches).</span>
|
||||
<span class="c1"># Each result is matched on an OR of its relevant subfields as defense in</span>
|
||||
<span class="c1"># depth: the parsers we audited never store a policy/failure detail</span>
|
||||
<span class="c1"># without these fields, but an empty string indexes no text tokens and is</span>
|
||||
<span class="c1"># invisible to ``exists``, and the storage shape of every historical</span>
|
||||
<span class="c1"># parsedmarc version can't be audited — matching either subfield costs</span>
|
||||
<span class="c1"># nothing and cannot skip a document that has something to backfill.</span>
|
||||
<span class="n">_SMTP_TLS_COMBINED_BACKFILL_QUERY</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s2">"bool"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"minimum_should_match"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
|
||||
<span class="s2">"should"</span><span class="p">:</span> <span class="p">[</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s2">"bool"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"must"</span><span class="p">:</span> <span class="p">[</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s2">"bool"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"minimum_should_match"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
|
||||
<span class="s2">"should"</span><span class="p">:</span> <span class="p">[</span>
|
||||
<span class="p">{</span><span class="s2">"exists"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"field"</span><span class="p">:</span> <span class="s2">"policies.policy_domain"</span><span class="p">}},</span>
|
||||
<span class="p">{</span><span class="s2">"exists"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"field"</span><span class="p">:</span> <span class="s2">"policies.policy_type"</span><span class="p">}},</span>
|
||||
<span class="p">],</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">],</span>
|
||||
<span class="s2">"must_not"</span><span class="p">:</span> <span class="p">[{</span><span class="s2">"exists"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"field"</span><span class="p">:</span> <span class="s2">"policies_combined"</span><span class="p">}}],</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">},</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s2">"bool"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"must"</span><span class="p">:</span> <span class="p">[</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s2">"bool"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"minimum_should_match"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
|
||||
<span class="s2">"should"</span><span class="p">:</span> <span class="p">[</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s2">"exists"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"field"</span><span class="p">:</span> <span class="s2">"policies.failure_details.result_type"</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">},</span>
|
||||
<span class="p">{</span>
|
||||
<span class="s2">"exists"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"field"</span><span class="p">:</span> <span class="s2">"policies.failure_details.sending_mta_ip"</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">},</span>
|
||||
<span class="p">],</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">],</span>
|
||||
<span class="s2">"must_not"</span><span class="p">:</span> <span class="p">[{</span><span class="s2">"exists"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"field"</span><span class="p">:</span> <span class="s2">"failure_details_combined"</span><span class="p">}}],</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">},</span>
|
||||
<span class="p">],</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">}</span>
|
||||
|
||||
<span class="c1"># Painless script that (re)derives policies_combined/failure_details_combined</span>
|
||||
<span class="c1"># from policies/policies.failure_details, matching the format written by</span>
|
||||
<span class="c1"># save_smtp_tls_report_to_opensearch(): "{policy_domain} / {policy_type}"</span>
|
||||
<span class="c1"># per policy and "{policy_domain} / {policy_type} / {result_type} /</span>
|
||||
<span class="c1"># {sending_mta_ip} / {receiving_ip} / {receiving_mx_hostname}" per failure</span>
|
||||
<span class="c1"># detail.</span>
|
||||
<span class="n">_SMTP_TLS_COMBINED_BACKFILL_SCRIPT</span> <span class="o">=</span> <span class="p">(</span>
|
||||
<span class="s2">"List pols = new ArrayList(); "</span>
|
||||
<span class="s2">"List dets = new ArrayList(); "</span>
|
||||
<span class="s2">"def ps = ctx._source.policies; "</span>
|
||||
<span class="s2">"if (ps != null) { "</span>
|
||||
<span class="s2">"if (!(ps instanceof List)) { ps = [ps]; } "</span>
|
||||
<span class="s2">"for (p in ps) { "</span>
|
||||
<span class="s2">"if (p == null) { continue; } "</span>
|
||||
<span class="s1">'def dom = p.policy_domain != null ? p.policy_domain : "none"; '</span>
|
||||
<span class="s1">'def typ = p.policy_type != null ? p.policy_type : "none"; '</span>
|
||||
<span class="s1">'pols.add(dom + " / " + typ); '</span>
|
||||
<span class="s2">"def fds = p.failure_details; "</span>
|
||||
<span class="s2">"if (fds != null) { "</span>
|
||||
<span class="s2">"if (!(fds instanceof List)) { fds = [fds]; } "</span>
|
||||
<span class="s2">"for (f in fds) { "</span>
|
||||
<span class="s2">"if (f == null) { continue; } "</span>
|
||||
<span class="s1">'def rt = f.result_type != null ? f.result_type : "none"; '</span>
|
||||
<span class="s1">'def smi = f.sending_mta_ip != null ? f.sending_mta_ip : "none"; '</span>
|
||||
<span class="s1">'def ri = f.receiving_ip != null ? f.receiving_ip : "none"; '</span>
|
||||
<span class="s1">'def rmh = f.receiving_mx_hostname != null ? f.receiving_mx_hostname : "none"; '</span>
|
||||
<span class="s1">'dets.add(dom + " / " + typ + " / " + rt + " / " + smi + " / " + ri + " / " + rmh); '</span>
|
||||
<span class="s2">"} } } } "</span>
|
||||
<span class="s2">"ctx._source.policies_combined = pols; "</span>
|
||||
<span class="s2">"ctx._source.failure_details_combined = dets;"</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_PolicyOverride</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="nb">type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">comment</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
@@ -186,6 +362,11 @@
|
||||
<span class="n">header_from</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">envelope_from</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">envelope_to</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="c1"># Nested(...) on the two auth-result fields below is only the DSL's</span>
|
||||
<span class="c1"># in-memory document shape; it is never installed as a mapping.</span>
|
||||
<span class="c1"># create_indexes() deliberately skips Index.document() registration so</span>
|
||||
<span class="c1"># these fields stay dynamic-mapped as plain `object` in the cluster</span>
|
||||
<span class="c1"># (see the comment there and issue #169).</span>
|
||||
<span class="n">dkim_results</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_DKIMResult</span><span class="p">)</span>
|
||||
<span class="n">spf_results</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SPFResult</span><span class="p">)</span>
|
||||
<span class="c1"># One "{selector} / {domain} / {result}" (DKIM) or "{scope} / {domain} /</span>
|
||||
@@ -223,9 +404,7 @@
|
||||
<span class="n">human_result</span><span class="o">=</span><span class="n">human_result</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">dkim_results_combined</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="s2">"</span><span class="si">{0}</span><span class="s2"> / </span><span class="si">{1}</span><span class="s2"> / </span><span class="si">{2}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">selector</span><span class="p">,</span> <span class="n">domain</span><span class="p">,</span> <span class="n">result</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">dkim_results_combined</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">selector</span><span class="si">}</span><span class="s2"> / </span><span class="si">{</span><span class="n">domain</span><span class="si">}</span><span class="s2"> / </span><span class="si">{</span><span class="n">result</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_spf_result</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="p">,</span>
|
||||
@@ -242,9 +421,7 @@
|
||||
<span class="n">human_result</span><span class="o">=</span><span class="n">human_result</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">spf_results_combined</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="s2">"</span><span class="si">{0}</span><span class="s2"> / </span><span class="si">{1}</span><span class="s2"> / </span><span class="si">{2}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">scope</span><span class="p">,</span> <span class="n">domain</span><span class="p">,</span> <span class="n">result</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">spf_results_combined</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">scope</span><span class="si">}</span><span class="s2"> / </span><span class="si">{</span><span class="n">domain</span><span class="si">}</span><span class="s2"> / </span><span class="si">{</span><span class="n">result</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">save</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span> <span class="c1"># pyright: ignore[reportIncompatibleMethodOverride]</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">passed_dmarc</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
@@ -331,6 +508,7 @@
|
||||
<span class="n">result_type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">sending_mta_ip</span> <span class="o">=</span> <span class="n">Ip</span><span class="p">()</span>
|
||||
<span class="n">receiving_mx_helo</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">receiving_mx_hostname</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">receiving_ip</span> <span class="o">=</span> <span class="n">Ip</span><span class="p">()</span>
|
||||
<span class="n">failed_session_count</span> <span class="o">=</span> <span class="n">Integer</span><span class="p">()</span>
|
||||
<span class="n">additional_information_uri</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
@@ -366,7 +544,7 @@
|
||||
<span class="n">receiving_mx_helo</span><span class="o">=</span><span class="n">receiving_mx_helo</span><span class="p">,</span>
|
||||
<span class="n">receiving_ip</span><span class="o">=</span><span class="n">receiving_ip</span><span class="p">,</span>
|
||||
<span class="n">failed_session_count</span><span class="o">=</span><span class="n">failed_session_count</span><span class="p">,</span>
|
||||
<span class="n">additional_information</span><span class="o">=</span><span class="n">additional_information_uri</span><span class="p">,</span>
|
||||
<span class="n">additional_information_uri</span><span class="o">=</span><span class="n">additional_information_uri</span><span class="p">,</span>
|
||||
<span class="n">failure_reason_code</span><span class="o">=</span><span class="n">failure_reason_code</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">failure_details</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_details</span><span class="p">)</span>
|
||||
@@ -383,6 +561,19 @@
|
||||
<span class="n">contact_info</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">report_id</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">policies</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SMTPTLSPolicyDoc</span><span class="p">)</span>
|
||||
<span class="c1"># One "{policy_domain} / {policy_type}" string per policy. Kibana/</span>
|
||||
<span class="c1"># Grafana tables cannot terms-aggregate the subfields of an object</span>
|
||||
<span class="c1"># array without producing a cross-product of values (issue #169), so</span>
|
||||
<span class="c1"># dashboards aggregate these composed keywords instead. Declared to</span>
|
||||
<span class="c1"># match what dynamic mapping produces for a string array (text +</span>
|
||||
<span class="c1"># .keyword).</span>
|
||||
<span class="n">policies_combined</span> <span class="o">=</span> <span class="n">Text</span><span class="p">(</span><span class="n">multi</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">fields</span><span class="o">=</span><span class="p">{</span><span class="s2">"keyword"</span><span class="p">:</span> <span class="n">Keyword</span><span class="p">(</span><span class="n">ignore_above</span><span class="o">=</span><span class="mi">256</span><span class="p">)})</span>
|
||||
<span class="c1"># One "{policy_domain} / {policy_type} / {result_type} /</span>
|
||||
<span class="c1"># {sending_mta_ip} / {receiving_ip} / {receiving_mx_hostname}" string</span>
|
||||
<span class="c1"># per failure detail, across all policies.</span>
|
||||
<span class="n">failure_details_combined</span> <span class="o">=</span> <span class="n">Text</span><span class="p">(</span>
|
||||
<span class="n">multi</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">fields</span><span class="o">=</span><span class="p">{</span><span class="s2">"keyword"</span><span class="p">:</span> <span class="n">Keyword</span><span class="p">(</span><span class="n">ignore_above</span><span class="o">=</span><span class="mi">256</span><span class="p">)}</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="AlreadySaved">
|
||||
@@ -478,20 +669,27 @@
|
||||
<span class="k">for</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">names</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="n">Index</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="c1"># Deliberately no Index.document() registration: Kibana/OpenSearch</span>
|
||||
<span class="c1"># Dashboards/Grafana cannot terms-aggregate fields inside a</span>
|
||||
<span class="c1"># `nested` mapping, so the dynamic `object` mapping produced by a</span>
|
||||
<span class="c1"># bare create is load-bearing for the shipped dashboards (issue</span>
|
||||
<span class="c1"># #169; see the *_combined fields on _AggregateReportDoc).</span>
|
||||
<span class="c1"># Deliberately no Index.document() registration: the shipped</span>
|
||||
<span class="c1"># dashboards cannot rebuild their detail tables on a `nested`</span>
|
||||
<span class="c1"># mapping — Kibana/OSD visual editors do not support nested</span>
|
||||
<span class="c1"># fields, Vega can run nested aggregations but does not render</span>
|
||||
<span class="c1"># tables, and Grafana's nested bucket aggregation (9.4+) lacks</span>
|
||||
<span class="c1"># reverse_nested for parent-level metrics like message_count —</span>
|
||||
<span class="c1"># so the dynamic `object` mapping produced by a bare create is</span>
|
||||
<span class="c1"># load-bearing for the shipped dashboards. _AggregateReportDoc</span>
|
||||
<span class="c1"># still declares dkim_results/spf_results with Nested(...), but</span>
|
||||
<span class="c1"># that is only the DSL's in-memory shape for building documents</span>
|
||||
<span class="c1"># — it is never installed as a mapping. See issue #169 and the</span>
|
||||
<span class="c1"># *_combined fields on _AggregateReportDoc.</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">index</span><span class="o">.</span><span class="n">exists</span><span class="p">():</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Creating OpenSearch index: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">name</span><span class="p">))</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Creating OpenSearch index: </span><span class="si">{</span><span class="n">name</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">settings</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">index</span><span class="o">.</span><span class="n">settings</span><span class="p">(</span><span class="n">number_of_shards</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">number_of_replicas</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">index</span><span class="o">.</span><span class="n">settings</span><span class="p">(</span><span class="o">**</span><span class="n">settings</span><span class="p">)</span>
|
||||
<span class="n">index</span><span class="o">.</span><span class="n">create</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span><span class="s2">"OpenSearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"OpenSearch error: </span><span class="si">{</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
|
||||
@@ -500,47 +698,175 @@
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">migrate_indexes</span><span class="p">(</span>
|
||||
<span class="n">aggregate_indexes</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">failure_indexes</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">smtp_tls_indexes</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Updates index mappings</span>
|
||||
<span class="sd"> Runs index migrations and backfills.</span>
|
||||
|
||||
<span class="sd"> First, the legacy ``published_policy.fo`` migration: indexes where that</span>
|
||||
<span class="sd"> field was mapped as ``long`` (data indexed by very old parsedmarc</span>
|
||||
<span class="sd"> releases under the Elasticsearch 6-era ``doc`` mapping type) are rebuilt</span>
|
||||
<span class="sd"> as a ``-v2`` index with the text/keyword shape.</span>
|
||||
|
||||
<span class="sd"> Second, the ``dkim_results_combined``/``spf_results_combined`` backfill</span>
|
||||
<span class="sd"> (added for issue #169) for aggregate report documents that were saved</span>
|
||||
<span class="sd"> before those fields existed. For each name in ``aggregate_indexes``,</span>
|
||||
<span class="sd"> this submits an ``update_by_query`` against the ``f"{name}*"`` index</span>
|
||||
<span class="sd"> pattern (the real indexes are date-suffixed) as a non-blocking</span>
|
||||
<span class="sd"> background task (``wait_for_completion=False``), so it never delays</span>
|
||||
<span class="sd"> parsedmarc startup. Submission is guarded by a cheap ``count`` query</span>
|
||||
<span class="sd"> that only matches documents with DKIM/SPF results but no combined</span>
|
||||
<span class="sd"> field, so once an index is fully backfilled, later calls are a fast</span>
|
||||
<span class="sd"> no-op. Any error talking to the cluster (e.g. no indexes yet on a</span>
|
||||
<span class="sd"> fresh install, or a transient connection issue) is caught and logged</span>
|
||||
<span class="sd"> as a warning rather than raised; the backfill is simply retried on the</span>
|
||||
<span class="sd"> next startup, and the manual ``_update_by_query`` command documented</span>
|
||||
<span class="sd"> in ``docs/source/elasticsearch.md`` remains available in the meantime.</span>
|
||||
|
||||
<span class="sd"> Third, the same treatment for the ``policies_combined``/</span>
|
||||
<span class="sd"> ``failure_details_combined`` fields (same issue #169) on SMTP TLS</span>
|
||||
<span class="sd"> report documents, for each name in ``smtp_tls_indexes``.</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> aggregate_indexes (list): A list of aggregate index names</span>
|
||||
<span class="sd"> failure_indexes (list): A list of failure index names</span>
|
||||
<span class="sd"> (accepted for API compatibility; no migrations are</span>
|
||||
<span class="sd"> currently needed for failure indexes)</span>
|
||||
<span class="sd"> smtp_tls_indexes (list): A list of SMTP TLS index names</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">version</span> <span class="o">=</span> <span class="mi">2</span>
|
||||
<span class="k">if</span> <span class="n">aggregate_indexes</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">aggregate_indexes</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="k">for</span> <span class="n">aggregate_index_name</span> <span class="ow">in</span> <span class="n">aggregate_indexes</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">Index</span><span class="p">(</span><span class="n">aggregate_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">exists</span><span class="p">():</span>
|
||||
<span class="k">continue</span>
|
||||
<span class="n">aggregate_index</span> <span class="o">=</span> <span class="n">Index</span><span class="p">(</span><span class="n">aggregate_index_name</span><span class="p">)</span>
|
||||
<span class="n">doc</span> <span class="o">=</span> <span class="s2">"doc"</span>
|
||||
<span class="n">fo_field</span> <span class="o">=</span> <span class="s2">"published_policy.fo"</span>
|
||||
<span class="n">fo</span> <span class="o">=</span> <span class="s2">"fo"</span>
|
||||
<span class="n">fo_mapping</span> <span class="o">=</span> <span class="n">aggregate_index</span><span class="o">.</span><span class="n">get_field_mapping</span><span class="p">(</span><span class="n">fields</span><span class="o">=</span><span class="p">[</span><span class="n">fo_field</span><span class="p">])</span>
|
||||
<span class="n">fo_mapping</span> <span class="o">=</span> <span class="n">fo_mapping</span><span class="p">[</span><span class="nb">list</span><span class="p">(</span><span class="n">fo_mapping</span><span class="o">.</span><span class="n">keys</span><span class="p">())[</span><span class="mi">0</span><span class="p">]][</span><span class="s2">"mappings"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="n">doc</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">fo_mapping</span><span class="p">:</span>
|
||||
<span class="k">continue</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">aggregate_indexes</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">smtp_tls_indexes</span><span class="p">:</span>
|
||||
<span class="k">return</span>
|
||||
|
||||
<span class="n">fo_mapping</span> <span class="o">=</span> <span class="n">fo_mapping</span><span class="p">[</span><span class="n">doc</span><span class="p">][</span><span class="n">fo_field</span><span class="p">][</span><span class="s2">"mapping"</span><span class="p">][</span><span class="n">fo</span><span class="p">]</span>
|
||||
<span class="n">fo_type</span> <span class="o">=</span> <span class="n">fo_mapping</span><span class="p">[</span><span class="s2">"type"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="n">fo_type</span> <span class="o">==</span> <span class="s2">"long"</span><span class="p">:</span>
|
||||
<span class="n">new_index_name</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">-v</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">aggregate_index_name</span><span class="p">,</span> <span class="n">version</span><span class="p">)</span>
|
||||
<span class="n">body</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s2">"properties"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"published_policy.fo"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"type"</span><span class="p">:</span> <span class="s2">"text"</span><span class="p">,</span>
|
||||
<span class="s2">"fields"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"keyword"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"type"</span><span class="p">:</span> <span class="s2">"keyword"</span><span class="p">,</span> <span class="s2">"ignore_above"</span><span class="p">:</span> <span class="mi">256</span><span class="p">}},</span>
|
||||
<span class="n">version</span> <span class="o">=</span> <span class="mi">2</span>
|
||||
<span class="k">for</span> <span class="n">aggregate_index_name</span> <span class="ow">in</span> <span class="n">aggregate_indexes</span> <span class="ow">or</span> <span class="p">[]:</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">Index</span><span class="p">(</span><span class="n">aggregate_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">exists</span><span class="p">():</span>
|
||||
<span class="k">continue</span>
|
||||
<span class="n">aggregate_index</span> <span class="o">=</span> <span class="n">Index</span><span class="p">(</span><span class="n">aggregate_index_name</span><span class="p">)</span>
|
||||
<span class="n">doc</span> <span class="o">=</span> <span class="s2">"doc"</span>
|
||||
<span class="n">fo_field</span> <span class="o">=</span> <span class="s2">"published_policy.fo"</span>
|
||||
<span class="n">fo</span> <span class="o">=</span> <span class="s2">"fo"</span>
|
||||
<span class="n">fo_mapping</span> <span class="o">=</span> <span class="n">aggregate_index</span><span class="o">.</span><span class="n">get_field_mapping</span><span class="p">(</span><span class="n">fields</span><span class="o">=</span><span class="p">[</span><span class="n">fo_field</span><span class="p">])</span>
|
||||
<span class="n">fo_mapping</span> <span class="o">=</span> <span class="n">fo_mapping</span><span class="p">[</span><span class="nb">list</span><span class="p">(</span><span class="n">fo_mapping</span><span class="o">.</span><span class="n">keys</span><span class="p">())[</span><span class="mi">0</span><span class="p">]][</span><span class="s2">"mappings"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="n">doc</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">fo_mapping</span><span class="p">:</span>
|
||||
<span class="k">continue</span>
|
||||
|
||||
<span class="n">fo_mapping</span> <span class="o">=</span> <span class="n">fo_mapping</span><span class="p">[</span><span class="n">doc</span><span class="p">][</span><span class="n">fo_field</span><span class="p">][</span><span class="s2">"mapping"</span><span class="p">][</span><span class="n">fo</span><span class="p">]</span>
|
||||
<span class="n">fo_type</span> <span class="o">=</span> <span class="n">fo_mapping</span><span class="p">[</span><span class="s2">"type"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="n">fo_type</span> <span class="o">==</span> <span class="s2">"long"</span><span class="p">:</span>
|
||||
<span class="n">new_index_name</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">aggregate_index_name</span><span class="si">}</span><span class="s2">-v</span><span class="si">{</span><span class="n">version</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="n">body</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s2">"properties"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"published_policy.fo"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"type"</span><span class="p">:</span> <span class="s2">"text"</span><span class="p">,</span>
|
||||
<span class="s2">"fields"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"keyword"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"type"</span><span class="p">:</span> <span class="s2">"keyword"</span><span class="p">,</span> <span class="s2">"ignore_above"</span><span class="p">:</span> <span class="mi">256</span><span class="p">}</span>
|
||||
<span class="p">},</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">}</span>
|
||||
<span class="n">Index</span><span class="p">(</span><span class="n">new_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">create</span><span class="p">()</span>
|
||||
<span class="n">Index</span><span class="p">(</span><span class="n">new_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">put_mapping</span><span class="p">(</span><span class="n">doc_type</span><span class="o">=</span><span class="n">doc</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="n">body</span><span class="p">)</span>
|
||||
<span class="n">reindex</span><span class="p">(</span><span class="n">connections</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(),</span> <span class="n">aggregate_index_name</span><span class="p">,</span> <span class="n">new_index_name</span><span class="p">)</span>
|
||||
<span class="n">Index</span><span class="p">(</span><span class="n">aggregate_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">delete</span><span class="p">()</span></div>
|
||||
<span class="n">Index</span><span class="p">(</span><span class="n">new_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">create</span><span class="p">()</span>
|
||||
<span class="n">Index</span><span class="p">(</span><span class="n">new_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">put_mapping</span><span class="p">(</span><span class="n">doc_type</span><span class="o">=</span><span class="n">doc</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="n">body</span><span class="p">)</span>
|
||||
<span class="n">reindex</span><span class="p">(</span>
|
||||
<span class="n">connections</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(),</span> <span class="n">aggregate_index_name</span><span class="p">,</span> <span class="n">new_index_name</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">Index</span><span class="p">(</span><span class="n">aggregate_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">delete</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
||||
<span class="s2">"Failed the legacy published_policy.fo migration for "</span>
|
||||
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">aggregate_index_name</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">. This will be retried at the "</span>
|
||||
<span class="s2">"next startup."</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">client</span> <span class="o">=</span> <span class="n">connections</span><span class="o">.</span><span class="n">get_connection</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
||||
<span class="s2">"Skipping the dkim_results_combined/spf_results_combined/"</span>
|
||||
<span class="s2">"policies_combined/failure_details_combined backfill: could "</span>
|
||||
<span class="sa">f</span><span class="s2">"not get an OpenSearch connection: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">. This will be retried "</span>
|
||||
<span class="s2">"at the next startup."</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">return</span>
|
||||
<span class="k">for</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">aggregate_indexes</span> <span class="ow">or</span> <span class="p">[]:</span>
|
||||
<span class="n">pattern</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">name</span><span class="si">}</span><span class="s2">*"</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">count_response</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">count</span><span class="p">(</span>
|
||||
<span class="n">index</span><span class="o">=</span><span class="n">pattern</span><span class="p">,</span>
|
||||
<span class="n">body</span><span class="o">=</span><span class="p">{</span><span class="s2">"query"</span><span class="p">:</span> <span class="n">_COMBINED_BACKFILL_QUERY</span><span class="p">},</span>
|
||||
<span class="n">ignore_unavailable</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="n">allow_no_indices</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">count</span> <span class="o">=</span> <span class="n">count_response</span><span class="p">[</span><span class="s2">"count"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">count</span><span class="p">:</span>
|
||||
<span class="k">continue</span>
|
||||
<span class="n">update_response</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">update_by_query</span><span class="p">(</span>
|
||||
<span class="n">index</span><span class="o">=</span><span class="n">pattern</span><span class="p">,</span>
|
||||
<span class="n">body</span><span class="o">=</span><span class="p">{</span>
|
||||
<span class="s2">"query"</span><span class="p">:</span> <span class="n">_COMBINED_BACKFILL_QUERY</span><span class="p">,</span>
|
||||
<span class="s2">"script"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"source"</span><span class="p">:</span> <span class="n">_COMBINED_BACKFILL_SCRIPT</span><span class="p">,</span>
|
||||
<span class="s2">"lang"</span><span class="p">:</span> <span class="s2">"painless"</span><span class="p">,</span>
|
||||
<span class="p">},</span>
|
||||
<span class="p">},</span>
|
||||
<span class="n">conflicts</span><span class="o">=</span><span class="s2">"proceed"</span><span class="p">,</span>
|
||||
<span class="n">wait_for_completion</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">ignore_unavailable</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="n">allow_no_indices</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">task_id</span> <span class="o">=</span> <span class="n">update_response</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"task"</span><span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
|
||||
<span class="s2">"Backfilling dkim_results_combined/spf_results_combined on "</span>
|
||||
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">count</span><span class="si">}</span><span class="s2"> existing documents in </span><span class="si">{</span><span class="n">pattern</span><span class="si">}</span><span class="s2"> (task </span><span class="si">{</span><span class="n">task_id</span><span class="si">}</span><span class="s2">)"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
||||
<span class="s2">"Failed to check/submit the dkim_results_combined/"</span>
|
||||
<span class="sa">f</span><span class="s2">"spf_results_combined backfill for </span><span class="si">{</span><span class="n">pattern</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">. This "</span>
|
||||
<span class="s2">"will be retried at the next startup; the manual "</span>
|
||||
<span class="s2">"_update_by_query command in the documentation remains "</span>
|
||||
<span class="s2">"available in the meantime."</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">smtp_tls_indexes</span> <span class="ow">or</span> <span class="p">[]:</span>
|
||||
<span class="n">pattern</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">name</span><span class="si">}</span><span class="s2">*"</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">count_response</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">count</span><span class="p">(</span>
|
||||
<span class="n">index</span><span class="o">=</span><span class="n">pattern</span><span class="p">,</span>
|
||||
<span class="n">body</span><span class="o">=</span><span class="p">{</span><span class="s2">"query"</span><span class="p">:</span> <span class="n">_SMTP_TLS_COMBINED_BACKFILL_QUERY</span><span class="p">},</span>
|
||||
<span class="n">ignore_unavailable</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="n">allow_no_indices</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">count</span> <span class="o">=</span> <span class="n">count_response</span><span class="p">[</span><span class="s2">"count"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">count</span><span class="p">:</span>
|
||||
<span class="k">continue</span>
|
||||
<span class="n">update_response</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">update_by_query</span><span class="p">(</span>
|
||||
<span class="n">index</span><span class="o">=</span><span class="n">pattern</span><span class="p">,</span>
|
||||
<span class="n">body</span><span class="o">=</span><span class="p">{</span>
|
||||
<span class="s2">"query"</span><span class="p">:</span> <span class="n">_SMTP_TLS_COMBINED_BACKFILL_QUERY</span><span class="p">,</span>
|
||||
<span class="s2">"script"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"source"</span><span class="p">:</span> <span class="n">_SMTP_TLS_COMBINED_BACKFILL_SCRIPT</span><span class="p">,</span>
|
||||
<span class="s2">"lang"</span><span class="p">:</span> <span class="s2">"painless"</span><span class="p">,</span>
|
||||
<span class="p">},</span>
|
||||
<span class="p">},</span>
|
||||
<span class="n">conflicts</span><span class="o">=</span><span class="s2">"proceed"</span><span class="p">,</span>
|
||||
<span class="n">wait_for_completion</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">ignore_unavailable</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="n">allow_no_indices</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">task_id</span> <span class="o">=</span> <span class="n">update_response</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"task"</span><span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
|
||||
<span class="s2">"Backfilling policies_combined/failure_details_combined on "</span>
|
||||
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">count</span><span class="si">}</span><span class="s2"> existing documents in </span><span class="si">{</span><span class="n">pattern</span><span class="si">}</span><span class="s2"> (task </span><span class="si">{</span><span class="n">task_id</span><span class="si">}</span><span class="s2">)"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
||||
<span class="s2">"Failed to check/submit the policies_combined/"</span>
|
||||
<span class="sa">f</span><span class="s2">"failure_details_combined backfill for </span><span class="si">{</span><span class="n">pattern</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">. "</span>
|
||||
<span class="s2">"This will be retried at the next startup; the manual "</span>
|
||||
<span class="s2">"_update_by_query command in the documentation remains "</span>
|
||||
<span class="s2">"available in the meantime."</span>
|
||||
<span class="p">)</span></div>
|
||||
|
||||
|
||||
|
||||
@@ -589,11 +915,11 @@
|
||||
<span class="n">end_date_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="nb">range</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">date_end</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">lte</span><span class="o">=</span><span class="n">end_date</span><span class="p">))))</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">index_suffix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"dmarc_aggregate_</span><span class="si">{0}</span><span class="s2">*"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"dmarc_aggregate_</span><span class="si">{</span><span class="n">index_suffix</span><span class="si">}</span><span class="s2">*"</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"dmarc_aggregate*"</span>
|
||||
<span class="k">if</span> <span class="n">index_prefix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">search_index</span><span class="p">)</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">index_prefix</span><span class="si">}{</span><span class="n">search_index</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="n">search_index</span><span class="p">)</span>
|
||||
<span class="n">query</span> <span class="o">=</span> <span class="n">org_name_query</span> <span class="o">&</span> <span class="n">report_id_query</span> <span class="o">&</span> <span class="n">domain_query</span>
|
||||
<span class="n">query</span> <span class="o">=</span> <span class="n">query</span> <span class="o">&</span> <span class="n">begin_date_query</span> <span class="o">&</span> <span class="n">end_date_query</span>
|
||||
@@ -605,18 +931,15 @@
|
||||
<span class="n">existing</span> <span class="o">=</span> <span class="n">search</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error_</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span>
|
||||
<span class="s2">"OpenSearch's search for existing report </span><span class="se">\</span>
|
||||
<span class="s2"> error: </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error_</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="sa">f</span><span class="s2">"OpenSearch's search for existing report error: </span><span class="si">{</span><span class="n">error_</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">existing</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">AlreadySaved</span><span class="p">(</span>
|
||||
<span class="s2">"An aggregate report ID </span><span class="si">{0}</span><span class="s2"> from </span><span class="si">{1}</span><span class="s2"> about </span><span class="si">{2}</span><span class="s2"> "</span>
|
||||
<span class="s2">"with a date range of </span><span class="si">{3}</span><span class="s2"> UTC to </span><span class="si">{4}</span><span class="s2"> UTC already "</span>
|
||||
<span class="sa">f</span><span class="s2">"An aggregate report ID </span><span class="si">{</span><span class="n">report_id</span><span class="si">}</span><span class="s2"> from </span><span class="si">{</span><span class="n">org_name</span><span class="si">}</span><span class="s2"> about </span><span class="si">{</span><span class="n">domain</span><span class="si">}</span><span class="s2"> "</span>
|
||||
<span class="sa">f</span><span class="s2">"with a date range of </span><span class="si">{</span><span class="n">begin_date_human</span><span class="si">}</span><span class="s2"> UTC to </span><span class="si">{</span><span class="n">end_date_human</span><span class="si">}</span><span class="s2"> UTC already "</span>
|
||||
<span class="s2">"exists in "</span>
|
||||
<span class="s2">"OpenSearch"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">report_id</span><span class="p">,</span> <span class="n">org_name</span><span class="p">,</span> <span class="n">domain</span><span class="p">,</span> <span class="n">begin_date_human</span><span class="p">,</span> <span class="n">end_date_human</span>
|
||||
<span class="p">)</span>
|
||||
<span class="s2">"OpenSearch"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">published_policy</span> <span class="o">=</span> <span class="n">_PublishedPolicy</span><span class="p">(</span>
|
||||
<span class="n">domain</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"domain"</span><span class="p">],</span>
|
||||
@@ -709,11 +1032,11 @@
|
||||
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"dmarc_aggregate"</span>
|
||||
<span class="k">if</span> <span class="n">index_suffix</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">_</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">index</span><span class="si">}</span><span class="s2">_</span><span class="si">{</span><span class="n">index_suffix</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="k">if</span> <span class="n">index_prefix</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">index</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">index_prefix</span><span class="si">}{</span><span class="n">index</span><span class="si">}</span><span class="s2">"</span>
|
||||
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">-</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_date</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">index</span><span class="si">}</span><span class="s2">-</span><span class="si">{</span><span class="n">index_date</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span> <span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span>
|
||||
<span class="p">)</span>
|
||||
@@ -723,7 +1046,7 @@
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">agg_doc</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span><span class="s2">"OpenSearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"OpenSearch error: </span><span class="si">{</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
|
||||
@@ -774,12 +1097,12 @@
|
||||
<span class="n">arrival_date_epoch_milliseconds</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">arrival_date</span><span class="o">.</span><span class="n">timestamp</span><span class="p">()</span> <span class="o">*</span> <span class="mi">1000</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">index_suffix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"dmarc_failure_</span><span class="si">{0}</span><span class="s2">*,dmarc_forensic_</span><span class="si">{0}</span><span class="s2">*"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"dmarc_failure_</span><span class="si">{</span><span class="n">index_suffix</span><span class="si">}</span><span class="s2">*,dmarc_forensic_</span><span class="si">{</span><span class="n">index_suffix</span><span class="si">}</span><span class="s2">*"</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"dmarc_failure*,dmarc_forensic*"</span>
|
||||
<span class="k">if</span> <span class="n">index_prefix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">","</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
|
||||
<span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">part</span><span class="p">)</span> <span class="k">for</span> <span class="n">part</span> <span class="ow">in</span> <span class="n">search_index</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">","</span><span class="p">)</span>
|
||||
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">index_prefix</span><span class="si">}{</span><span class="n">part</span><span class="si">}</span><span class="s2">"</span> <span class="k">for</span> <span class="n">part</span> <span class="ow">in</span> <span class="n">search_index</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">","</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="n">search_index</span><span class="p">)</span>
|
||||
<span class="n">q</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">arrival_date</span><span class="o">=</span><span class="n">arrival_date_epoch_milliseconds</span><span class="p">)))</span>
|
||||
@@ -831,8 +1154,8 @@
|
||||
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">existing</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">AlreadySaved</span><span class="p">(</span>
|
||||
<span class="s2">"A failure sample to </span><span class="si">{0}</span><span class="s2"> from </span><span class="si">{1}</span><span class="s2"> "</span>
|
||||
<span class="s2">"with a subject of </span><span class="si">{2}</span><span class="s2"> and arrival date of </span><span class="si">{3}</span><span class="s2"> "</span>
|
||||
<span class="s2">"A failure sample to </span><span class="si">{}</span><span class="s2"> from </span><span class="si">{}</span><span class="s2"> "</span>
|
||||
<span class="s2">"with a subject of </span><span class="si">{}</span><span class="s2"> and arrival date of </span><span class="si">{}</span><span class="s2"> "</span>
|
||||
<span class="s2">"already exists in "</span>
|
||||
<span class="s2">"OpenSearch"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">to_</span><span class="p">,</span> <span class="n">from_</span><span class="p">,</span> <span class="n">subject</span><span class="p">,</span> <span class="n">failure_report</span><span class="p">[</span><span class="s2">"arrival_date_utc"</span><span class="p">])</span>
|
||||
<span class="p">)</span>
|
||||
@@ -891,14 +1214,14 @@
|
||||
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"dmarc_failure"</span>
|
||||
<span class="k">if</span> <span class="n">index_suffix</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">_</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">index</span><span class="si">}</span><span class="s2">_</span><span class="si">{</span><span class="n">index_suffix</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="k">if</span> <span class="n">index_prefix</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">index</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">index_prefix</span><span class="si">}{</span><span class="n">index</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="k">if</span> <span class="n">monthly_indexes</span><span class="p">:</span>
|
||||
<span class="n">index_date</span> <span class="o">=</span> <span class="n">arrival_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m"</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">index_date</span> <span class="o">=</span> <span class="n">arrival_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">-</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_date</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">index</span><span class="si">}</span><span class="s2">-</span><span class="si">{</span><span class="n">index_date</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span> <span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span>
|
||||
<span class="p">)</span>
|
||||
@@ -907,10 +1230,10 @@
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">failure_doc</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span><span class="s2">"OpenSearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"OpenSearch error: </span><span class="si">{</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="k">except</span> <span class="ne">KeyError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidFailureReport</span><span class="p">(</span>
|
||||
<span class="s2">"Failure report missing required field: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="sa">f</span><span class="s2">"Failure report missing required field: </span><span class="si">{</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="p">)</span></div>
|
||||
|
||||
|
||||
@@ -960,11 +1283,11 @@
|
||||
<span class="n">end_date_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">date_end</span><span class="o">=</span><span class="n">end_date</span><span class="p">)))</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">index_suffix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"smtp_tls_</span><span class="si">{0}</span><span class="s2">*"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"smtp_tls_</span><span class="si">{</span><span class="n">index_suffix</span><span class="si">}</span><span class="s2">*"</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"smtp_tls*"</span>
|
||||
<span class="k">if</span> <span class="n">index_prefix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">search_index</span><span class="p">)</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">index_prefix</span><span class="si">}{</span><span class="n">search_index</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="n">search_index</span><span class="p">)</span>
|
||||
<span class="n">query</span> <span class="o">=</span> <span class="n">org_name_query</span> <span class="o">&</span> <span class="n">report_id_query</span>
|
||||
<span class="n">query</span> <span class="o">=</span> <span class="n">query</span> <span class="o">&</span> <span class="n">begin_date_query</span> <span class="o">&</span> <span class="n">end_date_query</span>
|
||||
@@ -974,8 +1297,7 @@
|
||||
<span class="n">existing</span> <span class="o">=</span> <span class="n">search</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error_</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span>
|
||||
<span class="s2">"OpenSearch's search for existing report </span><span class="se">\</span>
|
||||
<span class="s2"> error: </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error_</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="sa">f</span><span class="s2">"OpenSearch's search for existing report error: </span><span class="si">{</span><span class="n">error_</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">existing</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||
@@ -989,10 +1311,10 @@
|
||||
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"smtp_tls"</span>
|
||||
<span class="k">if</span> <span class="n">index_suffix</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">_</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">index</span><span class="si">}</span><span class="s2">_</span><span class="si">{</span><span class="n">index_suffix</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="k">if</span> <span class="n">index_prefix</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">index</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">-</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_date</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">index_prefix</span><span class="si">}{</span><span class="n">index</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">index</span><span class="si">}</span><span class="s2">-</span><span class="si">{</span><span class="n">index_date</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span> <span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span>
|
||||
<span class="p">)</span>
|
||||
@@ -1013,6 +1335,16 @@
|
||||
<span class="n">policy_strings</span> <span class="o">=</span> <span class="n">policy</span><span class="p">[</span><span class="s2">"policy_strings"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="s2">"mx_host_patterns"</span> <span class="ow">in</span> <span class="n">policy</span><span class="p">:</span>
|
||||
<span class="n">mx_host_patterns</span> <span class="o">=</span> <span class="n">policy</span><span class="p">[</span><span class="s2">"mx_host_patterns"</span><span class="p">]</span>
|
||||
<span class="c1"># policies_combined/failure_details_combined: see the field</span>
|
||||
<span class="c1"># declarations on _SMTPTLSReportDoc and issue #169. policies and</span>
|
||||
<span class="c1"># their failure_details are object arrays with the same</span>
|
||||
<span class="c1"># cross-product problem as dkim_results/spf_results, so dashboards</span>
|
||||
<span class="c1"># aggregate these composed strings instead of the raw subfields.</span>
|
||||
<span class="n">policy_domain_combined</span> <span class="o">=</span> <span class="n">policy</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"policy_domain"</span><span class="p">)</span> <span class="ow">or</span> <span class="s2">"none"</span>
|
||||
<span class="n">policy_type_combined</span> <span class="o">=</span> <span class="n">policy</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"policy_type"</span><span class="p">)</span> <span class="ow">or</span> <span class="s2">"none"</span>
|
||||
<span class="n">smtp_tls_doc</span><span class="o">.</span><span class="n">policies_combined</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">policy_domain_combined</span><span class="si">}</span><span class="s2"> / </span><span class="si">{</span><span class="n">policy_type_combined</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">policy_doc</span> <span class="o">=</span> <span class="n">_SMTPTLSPolicyDoc</span><span class="p">(</span>
|
||||
<span class="n">policy_domain</span><span class="o">=</span><span class="n">policy</span><span class="p">[</span><span class="s2">"policy_domain"</span><span class="p">],</span>
|
||||
<span class="n">policy_type</span><span class="o">=</span><span class="n">policy</span><span class="p">[</span><span class="s2">"policy_type"</span><span class="p">],</span>
|
||||
@@ -1033,7 +1365,12 @@
|
||||
|
||||
<span class="k">if</span> <span class="s2">"receiving_mx_hostname"</span> <span class="ow">in</span> <span class="n">failure_detail</span><span class="p">:</span>
|
||||
<span class="n">receiving_mx_hostname</span> <span class="o">=</span> <span class="n">failure_detail</span><span class="p">[</span><span class="s2">"receiving_mx_hostname"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="s2">"additional_information_uri"</span> <span class="ow">in</span> <span class="n">failure_detail</span><span class="p">:</span>
|
||||
<span class="c1"># The parser's key is additional_info_uri (see</span>
|
||||
<span class="c1"># SMTPTLSFailureDetailsOptional in types.py); accept the</span>
|
||||
<span class="c1"># long-form key too for dicts built by other callers.</span>
|
||||
<span class="k">if</span> <span class="s2">"additional_info_uri"</span> <span class="ow">in</span> <span class="n">failure_detail</span><span class="p">:</span>
|
||||
<span class="n">additional_information_uri</span> <span class="o">=</span> <span class="n">failure_detail</span><span class="p">[</span><span class="s2">"additional_info_uri"</span><span class="p">]</span>
|
||||
<span class="k">elif</span> <span class="s2">"additional_information_uri"</span> <span class="ow">in</span> <span class="n">failure_detail</span><span class="p">:</span>
|
||||
<span class="n">additional_information_uri</span> <span class="o">=</span> <span class="n">failure_detail</span><span class="p">[</span>
|
||||
<span class="s2">"additional_information_uri"</span>
|
||||
<span class="p">]</span>
|
||||
@@ -1058,6 +1395,16 @@
|
||||
<span class="n">additional_information_uri</span><span class="o">=</span><span class="n">additional_information_uri</span><span class="p">,</span>
|
||||
<span class="n">failure_reason_code</span><span class="o">=</span><span class="n">failure_reason_code</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">smtp_tls_doc</span><span class="o">.</span><span class="n">failure_details_combined</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="s2">"</span><span class="si">{}</span><span class="s2"> / </span><span class="si">{}</span><span class="s2"> / </span><span class="si">{}</span><span class="s2"> / </span><span class="si">{}</span><span class="s2"> / </span><span class="si">{}</span><span class="s2"> / </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">policy_domain_combined</span><span class="p">,</span>
|
||||
<span class="n">policy_type_combined</span><span class="p">,</span>
|
||||
<span class="n">failure_detail</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"result_type"</span><span class="p">)</span> <span class="ow">or</span> <span class="s2">"none"</span><span class="p">,</span>
|
||||
<span class="n">sending_mta_ip</span> <span class="ow">or</span> <span class="s2">"none"</span><span class="p">,</span>
|
||||
<span class="n">receiving_ip</span> <span class="ow">or</span> <span class="s2">"none"</span><span class="p">,</span>
|
||||
<span class="n">receiving_mx_hostname</span> <span class="ow">or</span> <span class="s2">"none"</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">smtp_tls_doc</span><span class="o">.</span><span class="n">policies</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">policy_doc</span><span class="p">)</span>
|
||||
|
||||
<span class="n">create_indexes</span><span class="p">([</span><span class="n">index</span><span class="p">],</span> <span class="n">index_settings</span><span class="p">)</span>
|
||||
@@ -1066,7 +1413,7 @@
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">smtp_tls_doc</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span><span class="s2">"OpenSearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"OpenSearch error: </span><span class="si">{</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -5,14 +5,14 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc.splunk — parsedmarc 10.2.4 documentation</title>
|
||||
<title>parsedmarc.splunk — parsedmarc 10.3.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=9edc463e" />
|
||||
|
||||
|
||||
<script src="../../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../../_static/documentation_options.js?v=2c8206f1"></script>
|
||||
<script src="../../_static/documentation_options.js?v=a126560f"></script>
|
||||
<script src="../../_static/doctools.js?v=fd6eb6e6"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=6ffebe34"></script>
|
||||
<script src="../../_static/js/theme.js"></script>
|
||||
@@ -133,8 +133,8 @@
|
||||
<span class="sd"> data before giving up</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">parsed_url</span> <span class="o">=</span> <span class="n">urlparse</span><span class="p">(</span><span class="n">url</span><span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">url</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">://</span><span class="si">{1}</span><span class="s2">/services/collector/event/1.0"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">parsed_url</span><span class="o">.</span><span class="n">scheme</span><span class="p">,</span> <span class="n">parsed_url</span><span class="o">.</span><span class="n">netloc</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">url</span> <span class="o">=</span> <span class="p">(</span>
|
||||
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">parsed_url</span><span class="o">.</span><span class="n">scheme</span><span class="si">}</span><span class="s2">://</span><span class="si">{</span><span class="n">parsed_url</span><span class="o">.</span><span class="n">netloc</span><span class="si">}</span><span class="s2">/services/collector/event/1.0"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">access_token</span> <span class="o">=</span> <span class="n">access_token</span><span class="o">.</span><span class="n">lstrip</span><span class="p">(</span><span class="s2">"Splunk "</span><span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">index</span>
|
||||
@@ -149,7 +149,7 @@
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">session</span> <span class="o">=</span> <span class="n">httpx</span><span class="o">.</span><span class="n">Client</span><span class="p">(</span>
|
||||
<span class="n">headers</span><span class="o">=</span><span class="p">{</span>
|
||||
<span class="s2">"User-Agent"</span><span class="p">:</span> <span class="n">USER_AGENT</span><span class="p">,</span>
|
||||
<span class="s2">"Authorization"</span><span class="p">:</span> <span class="s2">"Splunk </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">access_token</span><span class="p">),</span>
|
||||
<span class="s2">"Authorization"</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"Splunk </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">access_token</span><span class="si">}</span><span class="s2">"</span><span class="p">,</span>
|
||||
<span class="p">},</span>
|
||||
<span class="n">verify</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">verify</span><span class="p">,</span>
|
||||
<span class="n">follow_redirects</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
@@ -177,7 +177,7 @@
|
||||
<span class="k">return</span>
|
||||
|
||||
<span class="n">data</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_common_data</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
<span class="n">json_str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||
<span class="n">json_lines</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="k">for</span> <span class="n">report</span> <span class="ow">in</span> <span class="n">aggregate_reports</span><span class="p">:</span>
|
||||
<span class="k">for</span> <span class="n">record</span> <span class="ow">in</span> <span class="n">report</span><span class="p">[</span><span class="s2">"records"</span><span class="p">]:</span>
|
||||
<span class="n">new_report</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span> <span class="o">|</span> <span class="nb">int</span> <span class="o">|</span> <span class="nb">float</span> <span class="o">|</span> <span class="nb">dict</span><span class="p">]</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
|
||||
@@ -216,13 +216,13 @@
|
||||
<span class="p">)</span>
|
||||
<span class="n">data</span><span class="p">[</span><span class="s2">"time"</span><span class="p">]</span> <span class="o">=</span> <span class="n">timestamp</span>
|
||||
<span class="n">data</span><span class="p">[</span><span class="s2">"event"</span><span class="p">]</span> <span class="o">=</span> <span class="n">new_report</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
<span class="n">json_str</span> <span class="o">+=</span> <span class="s2">"</span><span class="si">{0}</span><span class="se">\n</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">data</span><span class="p">))</span>
|
||||
<span class="n">json_lines</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">data</span><span class="p">)</span><span class="si">}</span><span class="se">\n</span><span class="s2">"</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">verify</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Skipping certificate verification for Splunk HEC"</span><span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">post</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">url</span><span class="p">,</span> <span class="n">content</span><span class="o">=</span><span class="n">json_str</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">timeout</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">url</span><span class="p">,</span> <span class="n">content</span><span class="o">=</span><span class="s2">""</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">json_lines</span><span class="p">),</span> <span class="n">timeout</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">timeout</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">response</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
@@ -251,7 +251,7 @@
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">failure_reports</span><span class="p">)</span> <span class="o"><</span> <span class="mi">1</span><span class="p">:</span>
|
||||
<span class="k">return</span>
|
||||
|
||||
<span class="n">json_str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||
<span class="n">json_lines</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="k">for</span> <span class="n">report</span> <span class="ow">in</span> <span class="n">failure_reports</span><span class="p">:</span>
|
||||
<span class="n">data</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_common_data</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
<span class="n">data</span><span class="p">[</span><span class="s2">"sourcetype"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"dmarc:failure"</span>
|
||||
@@ -262,13 +262,13 @@
|
||||
<span class="p">)</span>
|
||||
<span class="n">data</span><span class="p">[</span><span class="s2">"time"</span><span class="p">]</span> <span class="o">=</span> <span class="n">timestamp</span>
|
||||
<span class="n">data</span><span class="p">[</span><span class="s2">"event"</span><span class="p">]</span> <span class="o">=</span> <span class="n">report</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
<span class="n">json_str</span> <span class="o">+=</span> <span class="s2">"</span><span class="si">{0}</span><span class="se">\n</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">data</span><span class="p">))</span>
|
||||
<span class="n">json_lines</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">data</span><span class="p">)</span><span class="si">}</span><span class="se">\n</span><span class="s2">"</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">verify</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Skipping certificate verification for Splunk HEC"</span><span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">post</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">url</span><span class="p">,</span> <span class="n">content</span><span class="o">=</span><span class="n">json_str</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">timeout</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">url</span><span class="p">,</span> <span class="n">content</span><span class="o">=</span><span class="s2">""</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">json_lines</span><span class="p">),</span> <span class="n">timeout</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">timeout</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">response</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
@@ -298,19 +298,19 @@
|
||||
<span class="k">return</span>
|
||||
|
||||
<span class="n">data</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_common_data</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
<span class="n">json_str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||
<span class="n">json_lines</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="k">for</span> <span class="n">report</span> <span class="ow">in</span> <span class="n">reports</span><span class="p">:</span>
|
||||
<span class="n">data</span><span class="p">[</span><span class="s2">"sourcetype"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"smtp:tls"</span>
|
||||
<span class="n">timestamp</span> <span class="o">=</span> <span class="n">human_timestamp_to_unix_timestamp</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">])</span>
|
||||
<span class="n">data</span><span class="p">[</span><span class="s2">"time"</span><span class="p">]</span> <span class="o">=</span> <span class="n">timestamp</span>
|
||||
<span class="n">data</span><span class="p">[</span><span class="s2">"event"</span><span class="p">]</span> <span class="o">=</span> <span class="n">report</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
<span class="n">json_str</span> <span class="o">+=</span> <span class="s2">"</span><span class="si">{0}</span><span class="se">\n</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">data</span><span class="p">))</span>
|
||||
<span class="n">json_lines</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">data</span><span class="p">)</span><span class="si">}</span><span class="se">\n</span><span class="s2">"</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">verify</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Skipping certificate verification for Splunk HEC"</span><span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">post</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">url</span><span class="p">,</span> <span class="n">content</span><span class="o">=</span><span class="n">json_str</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">timeout</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">url</span><span class="p">,</span> <span class="n">content</span><span class="o">=</span><span class="s2">""</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">json_lines</span><span class="p">),</span> <span class="n">timeout</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">timeout</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">response</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
|
||||
@@ -5,14 +5,14 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc.types — parsedmarc 10.2.4 documentation</title>
|
||||
<title>parsedmarc.types — parsedmarc 10.3.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=9edc463e" />
|
||||
|
||||
|
||||
<script src="../../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../../_static/documentation_options.js?v=2c8206f1"></script>
|
||||
<script src="../../_static/documentation_options.js?v=a126560f"></script>
|
||||
<script src="../../_static/doctools.js?v=fd6eb6e6"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=6ffebe34"></script>
|
||||
<script src="../../_static/js/theme.js"></script>
|
||||
|
||||
@@ -5,14 +5,14 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc.utils — parsedmarc 10.2.4 documentation</title>
|
||||
<title>parsedmarc.utils — parsedmarc 10.3.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=9edc463e" />
|
||||
|
||||
|
||||
<script src="../../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../../_static/documentation_options.js?v=2c8206f1"></script>
|
||||
<script src="../../_static/documentation_options.js?v=a126560f"></script>
|
||||
<script src="../../_static/doctools.js?v=fd6eb6e6"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=6ffebe34"></script>
|
||||
<script src="../../_static/js/theme.js"></script>
|
||||
@@ -340,7 +340,7 @@
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">domain</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">domain</span><span class="p">)</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
|
||||
<span class="n">record_type</span> <span class="o">=</span> <span class="n">record_type</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span>
|
||||
<span class="n">cache_key</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">_</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">domain</span><span class="p">,</span> <span class="n">record_type</span><span class="p">)</span>
|
||||
<span class="n">cache_key</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">domain</span><span class="si">}</span><span class="s2">_</span><span class="si">{</span><span class="n">record_type</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="k">if</span> <span class="n">cache</span><span class="p">:</span>
|
||||
<span class="n">cached_records</span> <span class="o">=</span> <span class="n">cache</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">cache_key</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">cached_records</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
|
||||
@@ -1227,7 +1227,7 @@
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">mbox</span><span class="o">.</span><span class="n">keys</span><span class="p">())</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="n">_is_mbox</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Error checking for MBOX file: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Error checking for MBOX file: </span><span class="si">{</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">_is_mbox</span></div>
|
||||
|
||||
@@ -1388,7 +1388,7 @@
|
||||
<span class="n">payload</span> <span class="o">=</span> <span class="nb">str</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="n">payload</span><span class="p">)</span>
|
||||
<span class="n">attachment</span><span class="p">[</span><span class="s2">"sha256"</span><span class="p">]</span> <span class="o">=</span> <span class="n">hashlib</span><span class="o">.</span><span class="n">sha256</span><span class="p">(</span><span class="n">payload</span><span class="p">)</span><span class="o">.</span><span class="n">hexdigest</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Unable to decode attachment: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Unable to decode attachment: </span><span class="si">{</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">strip_attachment_payloads</span><span class="p">:</span>
|
||||
<span class="k">for</span> <span class="n">attachment</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">"attachments"</span><span class="p">]:</span>
|
||||
<span class="k">if</span> <span class="s2">"payload"</span> <span class="ow">in</span> <span class="n">attachment</span><span class="p">:</span>
|
||||
|
||||
Reference in New Issue
Block a user