mirror of
https://github.com/domainaware/parsedmarc.git
synced 2026-09-29 09:00:30 +00:00
Update docs
This commit is contained in:
Sean Whalen
+2
-2
@@ -3,7 +3,7 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Overview: module code — parsedmarc 8.12.0 documentation</title>
|
||||
<title>Overview: module code — parsedmarc 8.13.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../_static/css/theme.css" />
|
||||
|
||||
@@ -35,7 +35,7 @@
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.12.0
|
||||
8.13.0
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
|
||||
|
||||
+16
-11
@@ -3,7 +3,7 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc — parsedmarc 8.12.0 documentation</title>
|
||||
<title>parsedmarc — parsedmarc 8.13.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../_static/css/theme.css" />
|
||||
|
||||
@@ -35,7 +35,7 @@
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.12.0
|
||||
8.13.0
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
|
||||
@@ -121,7 +121,7 @@
|
||||
<span class="kn">from</span> <span class="nn">parsedmarc.utils</span> <span class="kn">import</span> <span class="n">parse_email</span>
|
||||
<span class="kn">from</span> <span class="nn">parsedmarc.utils</span> <span class="kn">import</span> <span class="n">timestamp_to_human</span><span class="p">,</span> <span class="n">human_timestamp_to_datetime</span>
|
||||
|
||||
<span class="n">__version__</span> <span class="o">=</span> <span class="s2">"8.12.0"</span>
|
||||
<span class="n">__version__</span> <span class="o">=</span> <span class="s2">"8.13.0"</span>
|
||||
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"parsedmarc v</span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">__version__</span><span class="p">))</span>
|
||||
|
||||
@@ -678,14 +678,19 @@
|
||||
<span class="sd"> str: The extracted text</span>
|
||||
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">def</span> <span class="nf">is_base64</span><span class="p">(</span><span class="n">s</span><span class="p">):</span>
|
||||
<span class="n">base64_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s1">'^[A-Za-z0-9+/=]+\Z'</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="nb">bool</span><span class="p">(</span><span class="n">base64_regex</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="n">s</span><span class="p">))</span>
|
||||
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">file_object</span> <span class="o">=</span> <span class="n">BytesIO</span><span class="p">()</span>
|
||||
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">input_</span><span class="p">)</span> <span class="ow">is</span> <span class="nb">str</span><span class="p">:</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">file_object</span> <span class="o">=</span> <span class="n">BytesIO</span><span class="p">(</span><span class="n">b64decode</span><span class="p">(</span><span class="n">input_</span><span class="p">))</span>
|
||||
<span class="k">except</span> <span class="n">binascii</span><span class="o">.</span><span class="n">Error</span><span class="p">:</span>
|
||||
<span class="k">pass</span>
|
||||
<span class="k">if</span> <span class="n">file_object</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">file_object</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">input_</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
|
||||
<span class="k">if</span> <span class="n">is_base64</span><span class="p">(</span><span class="n">input_</span><span class="p">):</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">file_object</span> <span class="o">=</span> <span class="n">BytesIO</span><span class="p">(</span><span class="n">b64decode</span><span class="p">(</span><span class="n">input_</span><span class="p">))</span>
|
||||
<span class="k">except</span> <span class="n">binascii</span><span class="o">.</span><span class="n">Error</span><span class="p">:</span>
|
||||
<span class="k">pass</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">file_object</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="n">input_</span><span class="p">,</span> <span class="s2">"rb"</span><span class="p">)</span>
|
||||
<span class="k">elif</span> <span class="nb">type</span><span class="p">(</span><span class="n">input_</span><span class="p">)</span> <span class="ow">is</span> <span class="nb">bytes</span><span class="p">:</span>
|
||||
<span class="n">file_object</span> <span class="o">=</span> <span class="n">BytesIO</span><span class="p">(</span><span class="n">input_</span><span class="p">)</span>
|
||||
@@ -700,7 +705,7 @@
|
||||
<span class="n">errors</span><span class="o">=</span><span class="s1">'ignore'</span><span class="p">)</span>
|
||||
<span class="k">elif</span> <span class="n">header</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="n">MAGIC_GZIP</span><span class="p">):</span>
|
||||
<span class="n">report</span> <span class="o">=</span> <span class="n">zlib</span><span class="o">.</span><span class="n">decompress</span><span class="p">(</span>
|
||||
<span class="n">file_object</span><span class="o">.</span><span class="n">getvalue</span><span class="p">(),</span>
|
||||
<span class="n">file_object</span><span class="o">.</span><span class="n">read</span><span class="p">(),</span>
|
||||
<span class="n">zlib</span><span class="o">.</span><span class="n">MAX_WBITS</span> <span class="o">|</span> <span class="mi">16</span><span class="p">)</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="n">errors</span><span class="o">=</span><span class="s1">'ignore'</span><span class="p">)</span>
|
||||
<span class="k">elif</span> <span class="n">header</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="n">MAGIC_XML</span><span class="p">)</span> <span class="ow">or</span> <span class="n">header</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="n">MAGIC_JSON</span><span class="p">):</span>
|
||||
<span class="n">report</span> <span class="o">=</span> <span class="n">file_object</span><span class="o">.</span><span class="n">read</span><span class="p">()</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="n">errors</span><span class="o">=</span><span class="s1">'ignore'</span><span class="p">)</span>
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc.elastic — parsedmarc 8.12.0 documentation</title>
|
||||
<title>parsedmarc.elastic — parsedmarc 8.13.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css" />
|
||||
|
||||
@@ -35,7 +35,7 @@
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.12.0
|
||||
8.13.0
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
|
||||
@@ -438,6 +438,7 @@
|
||||
|
||||
<div class="viewcode-block" id="save_aggregate_report_to_elasticsearch"><a class="viewcode-back" href="../../api.html#parsedmarc.elastic.save_aggregate_report_to_elasticsearch">[docs]</a><span class="k">def</span> <span class="nf">save_aggregate_report_to_elasticsearch</span><span class="p">(</span><span class="n">aggregate_report</span><span class="p">,</span>
|
||||
<span class="n">index_suffix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">index_prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">monthly_indexes</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="mi">0</span><span class="p">):</span>
|
||||
@@ -447,6 +448,7 @@
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> aggregate_report (OrderedDict): A parsed forensic report</span>
|
||||
<span class="sd"> index_suffix (str): The suffix of the name of the index to save to</span>
|
||||
<span class="sd"> index_prefix (str): The prefix of the name of the index to save to</span>
|
||||
<span class="sd"> monthly_indexes (bool): Use monthly indexes instead of daily indexes</span>
|
||||
<span class="sd"> number_of_shards (int): The number of shards to use in the index</span>
|
||||
<span class="sd"> number_of_replicas (int): The number of replicas to use in the index</span>
|
||||
@@ -482,9 +484,12 @@
|
||||
<span class="n">end_date_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">date_end</span><span class="o">=</span><span class="n">end_date</span><span class="p">)))</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">index_suffix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="s2">"dmarc_aggregate_</span><span class="si">{0}</span><span class="s2">*"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_suffix</span><span class="p">))</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"dmarc_aggregate_</span><span class="si">{0}</span><span class="s2">*"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="s2">"dmarc_aggregate*"</span><span class="p">)</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"dmarc_aggregate*"</span>
|
||||
<span class="k">if</span> <span class="n">index_prefix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">search_index</span><span class="p">)</span>
|
||||
<span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="n">search_index</span><span class="p">)</span>
|
||||
<span class="n">query</span> <span class="o">=</span> <span class="n">org_name_query</span> <span class="o">&</span> <span class="n">report_id_query</span> <span class="o">&</span> <span class="n">domain_query</span>
|
||||
<span class="n">query</span> <span class="o">=</span> <span class="n">query</span> <span class="o">&</span> <span class="n">begin_date_query</span> <span class="o">&</span> <span class="n">end_date_query</span>
|
||||
<span class="n">search</span><span class="o">.</span><span class="n">query</span> <span class="o">=</span> <span class="n">query</span>
|
||||
@@ -560,6 +565,9 @@
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"dmarc_aggregate"</span>
|
||||
<span class="k">if</span> <span class="n">index_suffix</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">_</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">index_prefix</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">index</span><span class="p">)</span>
|
||||
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">-</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_date</span><span class="p">)</span>
|
||||
<span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span><span class="p">)</span>
|
||||
@@ -575,6 +583,7 @@
|
||||
|
||||
<div class="viewcode-block" id="save_forensic_report_to_elasticsearch"><a class="viewcode-back" href="../../api.html#parsedmarc.elastic.save_forensic_report_to_elasticsearch">[docs]</a><span class="k">def</span> <span class="nf">save_forensic_report_to_elasticsearch</span><span class="p">(</span><span class="n">forensic_report</span><span class="p">,</span>
|
||||
<span class="n">index_suffix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">index_prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">monthly_indexes</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="mi">0</span><span class="p">):</span>
|
||||
@@ -584,6 +593,7 @@
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> forensic_report (OrderedDict): A parsed forensic report</span>
|
||||
<span class="sd"> index_suffix (str): The suffix of the name of the index to save to</span>
|
||||
<span class="sd"> index_prefix (str): The prefix of the name of the index to save to</span>
|
||||
<span class="sd"> monthly_indexes (bool): Use monthly indexes instead of daily</span>
|
||||
<span class="sd"> indexes</span>
|
||||
<span class="sd"> number_of_shards (int): The number of shards to use in the index</span>
|
||||
@@ -609,9 +619,12 @@
|
||||
<span class="n">arrival_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">arrival_date_human</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">index_suffix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="s2">"dmarc_forensic_</span><span class="si">{0}</span><span class="s2">*"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_suffix</span><span class="p">))</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"dmarc_forensic_</span><span class="si">{0}</span><span class="s2">*"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="s2">"dmarc_forensic*"</span><span class="p">)</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"dmarc_forensic*"</span>
|
||||
<span class="k">if</span> <span class="n">index_prefix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">search_index</span><span class="p">)</span>
|
||||
<span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="n">search_index</span><span class="p">)</span>
|
||||
<span class="n">arrival_query</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"match"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"arrival_date"</span><span class="p">:</span> <span class="n">arrival_date</span><span class="p">}}</span>
|
||||
<span class="n">q</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="n">arrival_query</span><span class="p">)</span>
|
||||
|
||||
@@ -697,6 +710,8 @@
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"dmarc_forensic"</span>
|
||||
<span class="k">if</span> <span class="n">index_suffix</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">_</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">index_prefix</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">index</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">monthly_indexes</span><span class="p">:</span>
|
||||
<span class="n">index_date</span> <span class="o">=</span> <span class="n">arrival_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m"</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
@@ -718,6 +733,7 @@
|
||||
|
||||
<div class="viewcode-block" id="save_smtp_tls_report_to_elasticsearch"><a class="viewcode-back" href="../../api.html#parsedmarc.elastic.save_smtp_tls_report_to_elasticsearch">[docs]</a><span class="k">def</span> <span class="nf">save_smtp_tls_report_to_elasticsearch</span><span class="p">(</span><span class="n">report</span><span class="p">,</span>
|
||||
<span class="n">index_suffix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">index_prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">monthly_indexes</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="mi">0</span><span class="p">):</span>
|
||||
@@ -727,6 +743,7 @@
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> report (OrderedDict): A parsed SMTP TLS report</span>
|
||||
<span class="sd"> index_suffix (str): The suffix of the name of the index to save to</span>
|
||||
<span class="sd"> index_prefix (str): The prefix of the name of the index to save to</span>
|
||||
<span class="sd"> monthly_indexes (bool): Use monthly indexes instead of daily indexes</span>
|
||||
<span class="sd"> number_of_shards (int): The number of shards to use in the index</span>
|
||||
<span class="sd"> number_of_replicas (int): The number of replicas to use in the index</span>
|
||||
@@ -756,9 +773,12 @@
|
||||
<span class="n">end_date_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">date_end</span><span class="o">=</span><span class="n">end_date</span><span class="p">)))</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">index_suffix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="s2">"smtp_tls_</span><span class="si">{0}</span><span class="s2">*"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_suffix</span><span class="p">))</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"smtp_tls_</span><span class="si">{0}</span><span class="s2">*"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="s2">"smtp_tls*"</span><span class="p">)</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"smtp_tls*"</span>
|
||||
<span class="k">if</span> <span class="n">index_prefix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">search_index</span><span class="p">)</span>
|
||||
<span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="n">search_index</span><span class="p">)</span>
|
||||
<span class="n">query</span> <span class="o">=</span> <span class="n">org_name_query</span> <span class="o">&</span> <span class="n">report_id_query</span>
|
||||
<span class="n">query</span> <span class="o">=</span> <span class="n">query</span> <span class="o">&</span> <span class="n">begin_date_query</span> <span class="o">&</span> <span class="n">end_date_query</span>
|
||||
<span class="n">search</span><span class="o">.</span><span class="n">query</span> <span class="o">=</span> <span class="n">query</span>
|
||||
@@ -779,6 +799,8 @@
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"smtp_tls"</span>
|
||||
<span class="k">if</span> <span class="n">index_suffix</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">_</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">index_prefix</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">index</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">-</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_date</span><span class="p">)</span>
|
||||
<span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span><span class="p">)</span>
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc.opensearch — parsedmarc 8.12.0 documentation</title>
|
||||
<title>parsedmarc.opensearch — parsedmarc 8.13.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css" />
|
||||
|
||||
@@ -35,7 +35,7 @@
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.12.0
|
||||
8.13.0
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
|
||||
@@ -434,6 +434,7 @@
|
||||
|
||||
<div class="viewcode-block" id="save_aggregate_report_to_opensearch"><a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.save_aggregate_report_to_opensearch">[docs]</a><span class="k">def</span> <span class="nf">save_aggregate_report_to_opensearch</span><span class="p">(</span><span class="n">aggregate_report</span><span class="p">,</span>
|
||||
<span class="n">index_suffix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">index_prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">monthly_indexes</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="mi">0</span><span class="p">):</span>
|
||||
@@ -443,6 +444,7 @@
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> aggregate_report (OrderedDict): A parsed forensic report</span>
|
||||
<span class="sd"> index_suffix (str): The suffix of the name of the index to save to</span>
|
||||
<span class="sd"> index_prefix (str): The prefix of the name of the index to save to</span>
|
||||
<span class="sd"> monthly_indexes (bool): Use monthly indexes instead of daily indexes</span>
|
||||
<span class="sd"> number_of_shards (int): The number of shards to use in the index</span>
|
||||
<span class="sd"> number_of_replicas (int): The number of replicas to use in the index</span>
|
||||
@@ -478,9 +480,12 @@
|
||||
<span class="n">end_date_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">date_end</span><span class="o">=</span><span class="n">end_date</span><span class="p">)))</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">index_suffix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="s2">"dmarc_aggregate_</span><span class="si">{0}</span><span class="s2">*"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_suffix</span><span class="p">))</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"dmarc_aggregate_</span><span class="si">{0}</span><span class="s2">*"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="s2">"dmarc_aggregate*"</span><span class="p">)</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"dmarc_aggregate*"</span>
|
||||
<span class="k">if</span> <span class="n">index_prefix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">search_index</span><span class="p">)</span>
|
||||
<span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="n">search_index</span><span class="p">)</span>
|
||||
<span class="n">query</span> <span class="o">=</span> <span class="n">org_name_query</span> <span class="o">&</span> <span class="n">report_id_query</span> <span class="o">&</span> <span class="n">domain_query</span>
|
||||
<span class="n">query</span> <span class="o">=</span> <span class="n">query</span> <span class="o">&</span> <span class="n">begin_date_query</span> <span class="o">&</span> <span class="n">end_date_query</span>
|
||||
<span class="n">search</span><span class="o">.</span><span class="n">query</span> <span class="o">=</span> <span class="n">query</span>
|
||||
@@ -556,6 +561,8 @@
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"dmarc_aggregate"</span>
|
||||
<span class="k">if</span> <span class="n">index_suffix</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">_</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">index_prefix</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">index</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">-</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_date</span><span class="p">)</span>
|
||||
<span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span><span class="p">)</span>
|
||||
@@ -571,6 +578,7 @@
|
||||
|
||||
<div class="viewcode-block" id="save_forensic_report_to_opensearch"><a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.save_forensic_report_to_opensearch">[docs]</a><span class="k">def</span> <span class="nf">save_forensic_report_to_opensearch</span><span class="p">(</span><span class="n">forensic_report</span><span class="p">,</span>
|
||||
<span class="n">index_suffix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">index_prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">monthly_indexes</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="mi">0</span><span class="p">):</span>
|
||||
@@ -580,6 +588,7 @@
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> forensic_report (OrderedDict): A parsed forensic report</span>
|
||||
<span class="sd"> index_suffix (str): The suffix of the name of the index to save to</span>
|
||||
<span class="sd"> index_prefix (str): The prefix of the name of the index to save to</span>
|
||||
<span class="sd"> monthly_indexes (bool): Use monthly indexes instead of daily</span>
|
||||
<span class="sd"> indexes</span>
|
||||
<span class="sd"> number_of_shards (int): The number of shards to use in the index</span>
|
||||
@@ -605,9 +614,12 @@
|
||||
<span class="n">arrival_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">arrival_date_human</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">index_suffix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="s2">"dmarc_forensic_</span><span class="si">{0}</span><span class="s2">*"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_suffix</span><span class="p">))</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"dmarc_forensic_</span><span class="si">{0}</span><span class="s2">*"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="s2">"dmarc_forensic*"</span><span class="p">)</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"dmarc_forensic*"</span>
|
||||
<span class="k">if</span> <span class="n">index_prefix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">search_index</span><span class="p">)</span>
|
||||
<span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="n">search_index</span><span class="p">)</span>
|
||||
<span class="n">arrival_query</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"match"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"arrival_date"</span><span class="p">:</span> <span class="n">arrival_date</span><span class="p">}}</span>
|
||||
<span class="n">q</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="n">arrival_query</span><span class="p">)</span>
|
||||
|
||||
@@ -691,6 +703,8 @@
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"dmarc_forensic"</span>
|
||||
<span class="k">if</span> <span class="n">index_suffix</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">_</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">index_prefix</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">index</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">monthly_indexes</span><span class="p">:</span>
|
||||
<span class="n">index_date</span> <span class="o">=</span> <span class="n">arrival_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m"</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
@@ -712,6 +726,7 @@
|
||||
|
||||
<div class="viewcode-block" id="save_smtp_tls_report_to_opensearch"><a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.save_smtp_tls_report_to_opensearch">[docs]</a><span class="k">def</span> <span class="nf">save_smtp_tls_report_to_opensearch</span><span class="p">(</span><span class="n">report</span><span class="p">,</span>
|
||||
<span class="n">index_suffix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">index_prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">monthly_indexes</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="mi">0</span><span class="p">):</span>
|
||||
@@ -721,6 +736,7 @@
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> report (OrderedDict): A parsed SMTP TLS report</span>
|
||||
<span class="sd"> index_suffix (str): The suffix of the name of the index to save to</span>
|
||||
<span class="sd"> index_prefix (str): The prefix of the name of the index to save to</span>
|
||||
<span class="sd"> monthly_indexes (bool): Use monthly indexes instead of daily indexes</span>
|
||||
<span class="sd"> number_of_shards (int): The number of shards to use in the index</span>
|
||||
<span class="sd"> number_of_replicas (int): The number of replicas to use in the index</span>
|
||||
@@ -750,9 +766,12 @@
|
||||
<span class="n">end_date_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">date_end</span><span class="o">=</span><span class="n">end_date</span><span class="p">)))</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">index_suffix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="s2">"smtp_tls_</span><span class="si">{0}</span><span class="s2">*"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_suffix</span><span class="p">))</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"smtp_tls_</span><span class="si">{0}</span><span class="s2">*"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="s2">"smtp_tls"</span><span class="p">)</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"smtp_tls*"</span>
|
||||
<span class="k">if</span> <span class="n">index_prefix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">search_index</span><span class="p">)</span>
|
||||
<span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="n">search_index</span><span class="p">)</span>
|
||||
<span class="n">query</span> <span class="o">=</span> <span class="n">org_name_query</span> <span class="o">&</span> <span class="n">report_id_query</span>
|
||||
<span class="n">query</span> <span class="o">=</span> <span class="n">query</span> <span class="o">&</span> <span class="n">begin_date_query</span> <span class="o">&</span> <span class="n">end_date_query</span>
|
||||
<span class="n">search</span><span class="o">.</span><span class="n">query</span> <span class="o">=</span> <span class="n">query</span>
|
||||
@@ -773,6 +792,8 @@
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"smtp_tls"</span>
|
||||
<span class="k">if</span> <span class="n">index_suffix</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">_</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">index_prefix</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">index</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">-</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_date</span><span class="p">)</span>
|
||||
<span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span><span class="p">)</span>
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc.splunk — parsedmarc 8.12.0 documentation</title>
|
||||
<title>parsedmarc.splunk — parsedmarc 8.13.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css" />
|
||||
|
||||
@@ -35,7 +35,7 @@
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.12.0
|
||||
8.13.0
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc.utils — parsedmarc 8.12.0 documentation</title>
|
||||
<title>parsedmarc.utils — parsedmarc 8.13.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css" />
|
||||
|
||||
@@ -35,7 +35,7 @@
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.12.0
|
||||
8.13.0
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
|
||||
|
||||
Reference in New Issue
Block a user