This commit is contained in:
Sean Whalen committed 2018-03-27 11:04:57 -04:00
1 parent 5c8b24118b
commit 0e1e0142b1
14 files changed
+1396 -134

No files matched your search

+579
View File
@@ -0,0 +1,579 @@
<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>elasticsearch_dsl.field &mdash; parsedmarc 3.3.0 documentation</title>
<link rel="stylesheet" href="../../_static/css/theme.css" type="text/css" />
<link rel="index" title="Index"
href="../../genindex.html"/>
<link rel="search" title="Search" href="../../search.html"/>
<link rel="top" title="parsedmarc 3.3.0 documentation" href="../../index.html"/>
<link rel="up" title="Module code" href="../index.html"/>
<script src="../../_static/js/modernizr.min.js"></script>
</head>
<body class="wy-body-for-nav" role="document">
<div class="wy-grid-for-nav">
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
<div class="wy-side-scroll">
<div class="wy-side-nav-search">
<a href="../../index.html" class="icon icon-home"> parsedmarc
</a>
<div class="version">
3.3.0
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
<input type="text" name="q" placeholder="Search docs" />
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
</form>
</div>
</div>
<div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
<!-- Local TOC -->
<div class="local-toc"></div>
</div>
</div>
</nav>
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
<nav class="wy-nav-top" role="navigation" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../index.html">parsedmarc</a>
</nav>
<div class="wy-nav-content">
<div class="rst-content">
<div role="navigation" aria-label="breadcrumbs navigation">
<ul class="wy-breadcrumbs">
<li><a href="../../index.html">Docs</a> &raquo;</li>
<li><a href="../index.html">Module code</a> &raquo;</li>
<li>elasticsearch_dsl.field</li>
<li class="wy-breadcrumbs-aside">
</li>
</ul>
<hr/>
</div>
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
<div itemprop="articleBody">
<h1>Source code for elasticsearch_dsl.field</h1><div class="highlight"><pre>
<span></span><span class="kn">import</span> <span class="nn">base64</span>
<span class="kn">import</span> <span class="nn">ipaddress</span>
<span class="kn">import</span> <span class="nn">collections</span>
<span class="kn">from</span> <span class="nn">datetime</span> <span class="k">import</span> <span class="n">date</span><span class="p">,</span> <span class="n">datetime</span>
<span class="kn">from</span> <span class="nn">dateutil</span> <span class="k">import</span> <span class="n">parser</span><span class="p">,</span> <span class="n">tz</span>
<span class="kn">from</span> <span class="nn">six</span> <span class="k">import</span> <span class="n">itervalues</span><span class="p">,</span> <span class="n">string_types</span><span class="p">,</span> <span class="n">iteritems</span>
<span class="kn">from</span> <span class="nn">six.moves</span> <span class="k">import</span> <span class="nb">map</span>
<span class="kn">from</span> <span class="nn">.utils</span> <span class="k">import</span> <span class="n">DslBase</span><span class="p">,</span> <span class="n">ObjectBase</span><span class="p">,</span> <span class="n">AttrDict</span><span class="p">,</span> <span class="n">AttrList</span>
<span class="kn">from</span> <span class="nn">.exceptions</span> <span class="k">import</span> <span class="n">ValidationException</span>
<span class="n">unicode</span> <span class="o">=</span> <span class="nb">type</span><span class="p">(</span><span class="sa">u</span><span class="s1">&#39;&#39;</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">construct_field</span><span class="p">(</span><span class="n">name_or_field</span><span class="p">,</span> <span class="o">**</span><span class="n">params</span><span class="p">):</span>
<span class="c1"># {&quot;type&quot;: &quot;text&quot;, &quot;analyzer&quot;: &quot;snowball&quot;}</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">name_or_field</span><span class="p">,</span> <span class="n">collections</span><span class="o">.</span><span class="n">Mapping</span><span class="p">):</span>
<span class="k">if</span> <span class="n">params</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;construct_field() cannot accept parameters when passing in a dict.&#39;</span><span class="p">)</span>
<span class="n">params</span> <span class="o">=</span> <span class="n">name_or_field</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="k">if</span> <span class="s1">&#39;type&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">params</span><span class="p">:</span>
<span class="c1"># inner object can be implicitly defined</span>
<span class="k">if</span> <span class="s1">&#39;properties&#39;</span> <span class="ow">in</span> <span class="n">params</span><span class="p">:</span>
<span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;object&#39;</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;construct_field() needs to have a &quot;type&quot; key.&#39;</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">name</span> <span class="o">=</span> <span class="n">params</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="s1">&#39;type&#39;</span><span class="p">)</span>
<span class="k">return</span> <span class="n">Field</span><span class="o">.</span><span class="n">get_dsl_class</span><span class="p">(</span><span class="n">name</span><span class="p">)(</span><span class="o">**</span><span class="n">params</span><span class="p">)</span>
<span class="c1"># Text()</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">name_or_field</span><span class="p">,</span> <span class="n">Field</span><span class="p">):</span>
<span class="k">if</span> <span class="n">params</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;construct_field() cannot accept parameters when passing in a construct_field object.&#39;</span><span class="p">)</span>
<span class="k">return</span> <span class="n">name_or_field</span>
<span class="c1"># &quot;text&quot;, analyzer=&quot;snowball&quot;</span>
<span class="k">return</span> <span class="n">Field</span><span class="o">.</span><span class="n">get_dsl_class</span><span class="p">(</span><span class="n">name_or_field</span><span class="p">)(</span><span class="o">**</span><span class="n">params</span><span class="p">)</span>
<span class="k">class</span> <span class="nc">Field</span><span class="p">(</span><span class="n">DslBase</span><span class="p">):</span>
<span class="n">_type_name</span> <span class="o">=</span> <span class="s1">&#39;field&#39;</span>
<span class="n">_type_shortcut</span> <span class="o">=</span> <span class="nb">staticmethod</span><span class="p">(</span><span class="n">construct_field</span><span class="p">)</span>
<span class="c1"># all fields can be multifields</span>
<span class="n">_param_defs</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;fields&#39;</span><span class="p">:</span> <span class="p">{</span><span class="s1">&#39;type&#39;</span><span class="p">:</span> <span class="s1">&#39;field&#39;</span><span class="p">,</span> <span class="s1">&#39;hash&#39;</span><span class="p">:</span> <span class="kc">True</span><span class="p">}}</span>
<span class="n">name</span> <span class="o">=</span> <span class="kc">None</span>
<span class="n">_coerce</span> <span class="o">=</span> <span class="kc">False</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_multi</span> <span class="o">=</span> <span class="n">kwargs</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="s1">&#39;multi&#39;</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_required</span> <span class="o">=</span> <span class="n">kwargs</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="s1">&#39;required&#39;</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span>
<span class="nb">super</span><span class="p">(</span><span class="n">Field</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">__getitem__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">subfield</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_params</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;fields&#39;</span><span class="p">,</span> <span class="p">{})[</span><span class="n">subfield</span><span class="p">]</span>
<span class="k">def</span> <span class="nf">_serialize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data</span><span class="p">):</span>
<span class="k">return</span> <span class="n">data</span>
<span class="k">def</span> <span class="nf">_deserialize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data</span><span class="p">):</span>
<span class="k">return</span> <span class="n">data</span>
<span class="k">def</span> <span class="nf">_empty</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="k">def</span> <span class="nf">empty</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_multi</span><span class="p">:</span>
<span class="k">return</span> <span class="n">AttrList</span><span class="p">([])</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_empty</span><span class="p">()</span>
<span class="k">def</span> <span class="nf">serialize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data</span><span class="p">):</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="p">(</span><span class="nb">list</span><span class="p">,</span> <span class="n">AttrList</span><span class="p">)):</span>
<span class="k">return</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_serialize</span><span class="p">,</span> <span class="n">data</span><span class="p">))</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_serialize</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">deserialize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data</span><span class="p">):</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="p">(</span><span class="nb">list</span><span class="p">,</span> <span class="n">AttrList</span><span class="p">)):</span>
<span class="n">data</span><span class="p">[:]</span> <span class="o">=</span> <span class="p">[</span>
<span class="kc">None</span> <span class="k">if</span> <span class="n">d</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">_deserialize</span><span class="p">(</span><span class="n">d</span><span class="p">)</span>
<span class="k">for</span> <span class="n">d</span> <span class="ow">in</span> <span class="n">data</span>
<span class="p">]</span>
<span class="k">return</span> <span class="n">data</span>
<span class="k">if</span> <span class="n">data</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_deserialize</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">clean</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data</span><span class="p">):</span>
<span class="k">if</span> <span class="n">data</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">data</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">deserialize</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
<span class="k">if</span> <span class="n">data</span> <span class="ow">in</span> <span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="p">[],</span> <span class="p">{})</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_required</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">ValidationException</span><span class="p">(</span><span class="s2">&quot;Value required for this field.&quot;</span><span class="p">)</span>
<span class="k">return</span> <span class="n">data</span>
<span class="k">def</span> <span class="nf">to_dict</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="n">d</span> <span class="o">=</span> <span class="nb">super</span><span class="p">(</span><span class="n">Field</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">to_dict</span><span class="p">()</span>
<span class="n">name</span><span class="p">,</span> <span class="n">value</span> <span class="o">=</span> <span class="n">d</span><span class="o">.</span><span class="n">popitem</span><span class="p">()</span>
<span class="n">value</span><span class="p">[</span><span class="s1">&#39;type&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">name</span>
<span class="k">return</span> <span class="n">value</span>
<span class="k">class</span> <span class="nc">CustomField</span><span class="p">(</span><span class="n">Field</span><span class="p">):</span>
<span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;custom&#39;</span>
<span class="n">_coerce</span> <span class="o">=</span> <span class="kc">True</span>
<span class="k">def</span> <span class="nf">to_dict</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">builtin_type</span><span class="p">,</span> <span class="n">Field</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">builtin_type</span><span class="o">.</span><span class="n">to_dict</span><span class="p">()</span>
<span class="n">d</span> <span class="o">=</span> <span class="nb">super</span><span class="p">(</span><span class="n">CustomField</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">to_dict</span><span class="p">()</span>
<span class="n">d</span><span class="p">[</span><span class="s1">&#39;type&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">builtin_type</span>
<span class="k">return</span> <span class="n">d</span>
<span class="k">class</span> <span class="nc">Object</span><span class="p">(</span><span class="n">Field</span><span class="p">):</span>
<span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;object&#39;</span>
<span class="n">_coerce</span> <span class="o">=</span> <span class="kc">True</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">doc_class</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_doc_class</span> <span class="o">=</span> <span class="n">doc_class</span>
<span class="k">if</span> <span class="n">doc_class</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="c1"># FIXME import</span>
<span class="kn">from</span> <span class="nn">.document</span> <span class="k">import</span> <span class="n">InnerDoc</span>
<span class="c1"># no InnerDoc subclass, creating one instead...</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_doc_class</span> <span class="o">=</span> <span class="nb">type</span><span class="p">(</span><span class="s1">&#39;InnerDoc&#39;</span><span class="p">,</span> <span class="p">(</span><span class="n">InnerDoc</span><span class="p">,</span> <span class="p">),</span> <span class="p">{})</span>
<span class="k">for</span> <span class="n">name</span><span class="p">,</span> <span class="n">field</span> <span class="ow">in</span> <span class="n">iteritems</span><span class="p">(</span><span class="n">kwargs</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="s1">&#39;properties&#39;</span><span class="p">,</span> <span class="p">{})):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_doc_class</span><span class="o">.</span><span class="n">_doc_type</span><span class="o">.</span><span class="n">mapping</span><span class="o">.</span><span class="n">field</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">field</span><span class="p">)</span>
<span class="k">if</span> <span class="s1">&#39;dynamic&#39;</span> <span class="ow">in</span> <span class="n">kwargs</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_doc_class</span><span class="o">.</span><span class="n">_doc_type</span><span class="o">.</span><span class="n">mapping</span><span class="o">.</span><span class="n">meta</span><span class="p">(</span><span class="s1">&#39;dynamic&#39;</span><span class="p">,</span> <span class="n">kwargs</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="s1">&#39;dynamic&#39;</span><span class="p">))</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_mapping</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_doc_class</span><span class="o">.</span><span class="n">_doc_type</span><span class="o">.</span><span class="n">mapping</span>
<span class="nb">super</span><span class="p">(</span><span class="n">Object</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">__getitem__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_mapping</span><span class="p">[</span><span class="n">name</span><span class="p">]</span>
<span class="k">def</span> <span class="nf">__contains__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">):</span>
<span class="k">return</span> <span class="n">name</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_mapping</span>
<span class="k">def</span> <span class="nf">_empty</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_wrap</span><span class="p">({})</span>
<span class="k">def</span> <span class="nf">_wrap</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_doc_class</span><span class="o">.</span><span class="n">from_es</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">empty</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_multi</span><span class="p">:</span>
<span class="k">return</span> <span class="n">AttrList</span><span class="p">([],</span> <span class="bp">self</span><span class="o">.</span><span class="n">_wrap</span><span class="p">)</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_empty</span><span class="p">()</span>
<span class="k">def</span> <span class="nf">to_dict</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="n">d</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_mapping</span><span class="o">.</span><span class="n">to_dict</span><span class="p">()</span>
<span class="n">_</span><span class="p">,</span> <span class="n">d</span> <span class="o">=</span> <span class="n">d</span><span class="o">.</span><span class="n">popitem</span><span class="p">()</span>
<span class="n">d</span><span class="p">[</span><span class="s2">&quot;type&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">name</span>
<span class="k">return</span> <span class="n">d</span>
<span class="k">def</span> <span class="nf">_collect_fields</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_mapping</span><span class="o">.</span><span class="n">properties</span><span class="o">.</span><span class="n">_collect_fields</span><span class="p">()</span>
<span class="k">def</span> <span class="nf">_deserialize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data</span><span class="p">):</span>
<span class="c1"># don&#39;t wrap already wrapped data</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_doc_class</span><span class="p">):</span>
<span class="k">return</span> <span class="n">data</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">AttrDict</span><span class="p">):</span>
<span class="n">data</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">_d_</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_wrap</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">_serialize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data</span><span class="p">):</span>
<span class="k">if</span> <span class="n">data</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="c1"># somebody assigned raw dict to the field, we should tolerate that</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">collections</span><span class="o">.</span><span class="n">Mapping</span><span class="p">):</span>
<span class="k">return</span> <span class="n">data</span>
<span class="k">return</span> <span class="n">data</span><span class="o">.</span><span class="n">to_dict</span><span class="p">()</span>
<span class="k">def</span> <span class="nf">clean</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data</span><span class="p">):</span>
<span class="n">data</span> <span class="o">=</span> <span class="nb">super</span><span class="p">(</span><span class="n">Object</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">clean</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
<span class="k">if</span> <span class="n">data</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="p">(</span><span class="nb">list</span><span class="p">,</span> <span class="n">AttrList</span><span class="p">)):</span>
<span class="k">for</span> <span class="n">d</span> <span class="ow">in</span> <span class="n">data</span><span class="p">:</span>
<span class="n">d</span><span class="o">.</span><span class="n">full_clean</span><span class="p">()</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">data</span><span class="o">.</span><span class="n">full_clean</span><span class="p">()</span>
<span class="k">return</span> <span class="n">data</span>
<span class="k">def</span> <span class="nf">update</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="n">Object</span><span class="p">):</span>
<span class="c1"># not an inner/nested object, no merge possible</span>
<span class="k">return</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_mapping</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">other</span><span class="o">.</span><span class="n">_mapping</span><span class="p">)</span>
<span class="k">class</span> <span class="nc">Nested</span><span class="p">(</span><span class="n">Object</span><span class="p">):</span>
<span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;nested&#39;</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
<span class="n">kwargs</span><span class="o">.</span><span class="n">setdefault</span><span class="p">(</span><span class="s1">&#39;multi&#39;</span><span class="p">,</span> <span class="kc">True</span><span class="p">)</span>
<span class="nb">super</span><span class="p">(</span><span class="n">Nested</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
<span class="k">class</span> <span class="nc">Date</span><span class="p">(</span><span class="n">Field</span><span class="p">):</span>
<span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;date&#39;</span>
<span class="n">_coerce</span> <span class="o">=</span> <span class="kc">True</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_default_timezone</span> <span class="o">=</span> <span class="n">kwargs</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="s1">&#39;default_timezone&#39;</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_default_timezone</span><span class="p">,</span> <span class="n">string_types</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_default_timezone</span> <span class="o">=</span> <span class="n">tz</span><span class="o">.</span><span class="n">gettz</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_default_timezone</span><span class="p">)</span>
<span class="nb">super</span><span class="p">(</span><span class="n">Date</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">_deserialize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data</span><span class="p">):</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">string_types</span><span class="p">):</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">data</span> <span class="o">=</span> <span class="n">parser</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">ValidationException</span><span class="p">(</span><span class="s1">&#39;Could not parse date from the value (</span><span class="si">%r</span><span class="s1">)&#39;</span> <span class="o">%</span> <span class="n">data</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">datetime</span><span class="p">):</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_default_timezone</span> <span class="ow">and</span> <span class="n">data</span><span class="o">.</span><span class="n">tzinfo</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">data</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="n">tzinfo</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_default_timezone</span><span class="p">)</span>
<span class="k">return</span> <span class="n">data</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">date</span><span class="p">):</span>
<span class="k">return</span> <span class="n">data</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="nb">int</span><span class="p">):</span>
<span class="c1"># Divide by a float to preserve milliseconds on the datetime.</span>
<span class="k">return</span> <span class="n">datetime</span><span class="o">.</span><span class="n">utcfromtimestamp</span><span class="p">(</span><span class="n">data</span> <span class="o">/</span> <span class="mf">1000.0</span><span class="p">)</span>
<span class="k">raise</span> <span class="n">ValidationException</span><span class="p">(</span><span class="s1">&#39;Could not parse date from the value (</span><span class="si">%r</span><span class="s1">)&#39;</span> <span class="o">%</span> <span class="n">data</span><span class="p">)</span>
<span class="k">class</span> <span class="nc">Text</span><span class="p">(</span><span class="n">Field</span><span class="p">):</span>
<span class="n">_param_defs</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">&#39;fields&#39;</span><span class="p">:</span> <span class="p">{</span><span class="s1">&#39;type&#39;</span><span class="p">:</span> <span class="s1">&#39;field&#39;</span><span class="p">,</span> <span class="s1">&#39;hash&#39;</span><span class="p">:</span> <span class="kc">True</span><span class="p">},</span>
<span class="s1">&#39;analyzer&#39;</span><span class="p">:</span> <span class="p">{</span><span class="s1">&#39;type&#39;</span><span class="p">:</span> <span class="s1">&#39;analyzer&#39;</span><span class="p">},</span>
<span class="s1">&#39;search_analyzer&#39;</span><span class="p">:</span> <span class="p">{</span><span class="s1">&#39;type&#39;</span><span class="p">:</span> <span class="s1">&#39;analyzer&#39;</span><span class="p">},</span>
<span class="s1">&#39;search_quote_analyzer&#39;</span><span class="p">:</span> <span class="p">{</span><span class="s1">&#39;type&#39;</span><span class="p">:</span> <span class="s1">&#39;analyzer&#39;</span><span class="p">},</span>
<span class="p">}</span>
<span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;text&#39;</span>
<span class="k">class</span> <span class="nc">Keyword</span><span class="p">(</span><span class="n">Field</span><span class="p">):</span>
<span class="n">_param_defs</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">&#39;fields&#39;</span><span class="p">:</span> <span class="p">{</span><span class="s1">&#39;type&#39;</span><span class="p">:</span> <span class="s1">&#39;field&#39;</span><span class="p">,</span> <span class="s1">&#39;hash&#39;</span><span class="p">:</span> <span class="kc">True</span><span class="p">},</span>
<span class="s1">&#39;search_analyzer&#39;</span><span class="p">:</span> <span class="p">{</span><span class="s1">&#39;type&#39;</span><span class="p">:</span> <span class="s1">&#39;analyzer&#39;</span><span class="p">},</span>
<span class="s1">&#39;normalizer&#39;</span><span class="p">:</span> <span class="p">{</span><span class="s1">&#39;type&#39;</span><span class="p">:</span> <span class="s1">&#39;normalizer&#39;</span><span class="p">}</span>
<span class="p">}</span>
<span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;keyword&#39;</span>
<span class="k">class</span> <span class="nc">Boolean</span><span class="p">(</span><span class="n">Field</span><span class="p">):</span>
<span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;boolean&#39;</span>
<span class="n">_coerce</span> <span class="o">=</span> <span class="kc">True</span>
<span class="k">def</span> <span class="nf">_deserialize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data</span><span class="p">):</span>
<span class="k">if</span> <span class="n">data</span> <span class="o">==</span> <span class="s2">&quot;false&quot;</span><span class="p">:</span>
<span class="k">return</span> <span class="kc">False</span>
<span class="k">return</span> <span class="nb">bool</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">clean</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data</span><span class="p">):</span>
<span class="k">if</span> <span class="n">data</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">data</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">deserialize</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
<span class="k">if</span> <span class="n">data</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_required</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">ValidationException</span><span class="p">(</span><span class="s2">&quot;Value required for this field.&quot;</span><span class="p">)</span>
<span class="k">return</span> <span class="n">data</span>
<span class="k">class</span> <span class="nc">Float</span><span class="p">(</span><span class="n">Field</span><span class="p">):</span>
<span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;float&#39;</span>
<span class="n">_coerce</span> <span class="o">=</span> <span class="kc">True</span>
<span class="k">def</span> <span class="nf">_deserialize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data</span><span class="p">):</span>
<span class="k">return</span> <span class="nb">float</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
<span class="k">class</span> <span class="nc">HalfFloat</span><span class="p">(</span><span class="n">Float</span><span class="p">):</span>
<span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;half_float&#39;</span>
<span class="k">class</span> <span class="nc">ScaledFloat</span><span class="p">(</span><span class="n">Float</span><span class="p">):</span>
<span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;scaled_float&#39;</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">scaling_factor</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
<span class="nb">super</span><span class="p">(</span><span class="n">ScaledFloat</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">scaling_factor</span><span class="o">=</span><span class="n">scaling_factor</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
<span class="k">class</span> <span class="nc">Double</span><span class="p">(</span><span class="n">Float</span><span class="p">):</span>
<span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;double&#39;</span>
<span class="k">class</span> <span class="nc">Integer</span><span class="p">(</span><span class="n">Field</span><span class="p">):</span>
<span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;integer&#39;</span>
<span class="n">_coerce</span> <span class="o">=</span> <span class="kc">True</span>
<span class="k">def</span> <span class="nf">_deserialize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data</span><span class="p">):</span>
<span class="k">return</span> <span class="nb">int</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
<span class="k">class</span> <span class="nc">Byte</span><span class="p">(</span><span class="n">Integer</span><span class="p">):</span>
<span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;byte&#39;</span>
<span class="k">class</span> <span class="nc">Short</span><span class="p">(</span><span class="n">Integer</span><span class="p">):</span>
<span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;short&#39;</span>
<span class="k">class</span> <span class="nc">Long</span><span class="p">(</span><span class="n">Integer</span><span class="p">):</span>
<span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;long&#39;</span>
<span class="k">class</span> <span class="nc">Ip</span><span class="p">(</span><span class="n">Field</span><span class="p">):</span>
<span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;ip&#39;</span>
<span class="n">_coerce</span> <span class="o">=</span> <span class="kc">True</span>
<span class="k">def</span> <span class="nf">_deserialize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data</span><span class="p">):</span>
<span class="c1"># the ipaddress library for pypy, python2.5 and 2.6 only accepts unicode.</span>
<span class="k">return</span> <span class="n">ipaddress</span><span class="o">.</span><span class="n">ip_address</span><span class="p">(</span><span class="n">unicode</span><span class="p">(</span><span class="n">data</span><span class="p">))</span>
<span class="k">def</span> <span class="nf">_serialize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data</span><span class="p">):</span>
<span class="k">if</span> <span class="n">data</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="k">return</span> <span class="nb">str</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
<span class="k">class</span> <span class="nc">Binary</span><span class="p">(</span><span class="n">Field</span><span class="p">):</span>
<span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;binary&#39;</span>
<span class="n">_coerce</span> <span class="o">=</span> <span class="kc">True</span>
<span class="k">def</span> <span class="nf">_deserialize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data</span><span class="p">):</span>
<span class="k">return</span> <span class="n">base64</span><span class="o">.</span><span class="n">b64decode</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">_serialize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data</span><span class="p">):</span>
<span class="k">if</span> <span class="n">data</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="k">return</span> <span class="n">base64</span><span class="o">.</span><span class="n">b64encode</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
<span class="k">class</span> <span class="nc">GeoPoint</span><span class="p">(</span><span class="n">Field</span><span class="p">):</span>
<span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;geo_point&#39;</span>
<span class="k">class</span> <span class="nc">GeoShape</span><span class="p">(</span><span class="n">Field</span><span class="p">):</span>
<span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;geo_shape&#39;</span>
<span class="k">class</span> <span class="nc">Completion</span><span class="p">(</span><span class="n">Field</span><span class="p">):</span>
<span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;completion&#39;</span>
<span class="k">class</span> <span class="nc">Percolator</span><span class="p">(</span><span class="n">Field</span><span class="p">):</span>
<span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;percolator&#39;</span>
<span class="k">class</span> <span class="nc">IntegerRange</span><span class="p">(</span><span class="n">Field</span><span class="p">):</span>
<span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;integer_range&#39;</span>
<span class="k">class</span> <span class="nc">FloatRange</span><span class="p">(</span><span class="n">Field</span><span class="p">):</span>
<span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;float_range&#39;</span>
<span class="k">class</span> <span class="nc">LongRange</span><span class="p">(</span><span class="n">Field</span><span class="p">):</span>
<span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;long_range&#39;</span>
<span class="k">class</span> <span class="nc">DoubleRange</span><span class="p">(</span><span class="n">Field</span><span class="p">):</span>
<span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;double_ranged&#39;</span>
<span class="k">class</span> <span class="nc">DateRange</span><span class="p">(</span><span class="n">Field</span><span class="p">):</span>
<span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;date_range&#39;</span>
<span class="k">class</span> <span class="nc">Join</span><span class="p">(</span><span class="n">Field</span><span class="p">):</span>
<span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;join&#39;</span>
<span class="k">class</span> <span class="nc">TokenCount</span><span class="p">(</span><span class="n">Field</span><span class="p">):</span>
<span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;token_count&#39;</span>
<span class="k">class</span> <span class="nc">Murmur3</span><span class="p">(</span><span class="n">Field</span><span class="p">):</span>
<span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;murmur3&#39;</span>
</pre></div>
</div>
<div class="articleComments">
</div>
</div>
<footer>
<hr/>
<div role="contentinfo">
<p>
&copy; Copyright 2018, Sean Whalen.
</p>
</div>
Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
</div>
</div>
</section>
</div>
<script type="text/javascript">
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../',
VERSION:'3.3.0',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
SOURCELINK_SUFFIX: '.txt'
};
</script>
<script type="text/javascript" src="../../_static/jquery.js"></script>
<script type="text/javascript" src="../../_static/underscore.js"></script>
<script type="text/javascript" src="../../_static/doctools.js"></script>
<script type="text/javascript" src="../../_static/js/theme.js"></script>
<script type="text/javascript">
jQuery(function () {
SphinxRtdTheme.StickyNav.enable();
});
</script>
</body>
</html>
+8 -6
View File
@@ -8,7 +8,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Overview: module code &mdash; parsedmarc 3.2.0 documentation</title>
<title>Overview: module code &mdash; parsedmarc 3.3.0 documentation</title>
@@ -35,7 +35,7 @@
<link rel="index" title="Index"
href="../genindex.html"/>
<link rel="search" title="Search" href="../search.html"/>
<link rel="top" title="parsedmarc 3.2.0 documentation" href="../index.html"/>
<link rel="top" title="parsedmarc 3.3.0 documentation" href="../index.html"/>
<script src="../_static/js/modernizr.min.js"></script>
@@ -64,7 +64,7 @@
<div class="version">
3.2.0
3.3.0
</div>
@@ -150,8 +150,10 @@
<div itemprop="articleBody">
<h1>All modules for which code is available</h1>
<ul><li><a href="parsedmarc.html">parsedmarc</a></li>
</ul>
<ul><li><a href="elasticsearch_dsl/field.html">elasticsearch_dsl.field</a></li>
<li><a href="parsedmarc.html">parsedmarc</a></li>
<ul><li><a href="parsedmarc/elastic.html">parsedmarc.elastic</a></li>
</ul></ul>
</div>
<div class="articleComments">
@@ -187,7 +189,7 @@
<script type="text/javascript">
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../',
VERSION:'3.2.0',
VERSION:'3.3.0',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
+50 -49
View File
@@ -8,7 +8,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>parsedmarc &mdash; parsedmarc 3.2.0 documentation</title>
<title>parsedmarc &mdash; parsedmarc 3.3.0 documentation</title>
@@ -35,7 +35,7 @@
<link rel="index" title="Index"
href="../genindex.html"/>
<link rel="search" title="Search" href="../search.html"/>
<link rel="top" title="parsedmarc 3.2.0 documentation" href="../index.html"/>
<link rel="top" title="parsedmarc 3.3.0 documentation" href="../index.html"/>
<link rel="up" title="Module code" href="index.html"/>
@@ -65,7 +65,7 @@
<div class="version">
3.2.0
3.3.0
</div>
@@ -196,7 +196,7 @@
<span class="kn">import</span> <span class="nn">dateparser</span>
<span class="kn">import</span> <span class="nn">mailparser</span>
<span class="n">__version__</span> <span class="o">=</span> <span class="s2">&quot;3.2.0&quot;</span>
<span class="n">__version__</span> <span class="o">=</span> <span class="s2">&quot;3.3.0&quot;</span>
<span class="n">logger</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="vm">__name__</span><span class="p">)</span>
<span class="n">logger</span><span class="o">.</span><span class="n">setLevel</span><span class="p">(</span><span class="n">logging</span><span class="o">.</span><span class="n">INFO</span><span class="p">)</span>
@@ -209,28 +209,28 @@
<div class="viewcode-block" id="ParserError"><a class="viewcode-back" href="../index.html#parsedmarc.ParserError">[docs]</a><span class="k">class</span> <span class="nc">ParserError</span><span class="p">(</span><span class="ne">RuntimeError</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;Raised whenever the parser fails for some reason&quot;&quot;&quot;</span></div>
<span class="sd">&quot;&quot;&quot;Raised whenever the parser fails for some reason&quot;&quot;&quot;</span>
</div>
<div class="viewcode-block" id="IMAPError"><a class="viewcode-back" href="../index.html#parsedmarc.IMAPError">[docs]</a><span class="k">class</span> <span class="nc">IMAPError</span><span class="p">(</span><span class="ne">RuntimeError</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;Raised when an IMAP error occurs&quot;&quot;&quot;</span></div>
<span class="sd">&quot;&quot;&quot;Raised when an IMAP error occurs&quot;&quot;&quot;</span>
</div>
<div class="viewcode-block" id="SMTPError"><a class="viewcode-back" href="../index.html#parsedmarc.SMTPError">[docs]</a><span class="k">class</span> <span class="nc">SMTPError</span><span class="p">(</span><span class="ne">RuntimeError</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;Raised when a SMTP error occurs&quot;&quot;&quot;</span></div>
<span class="sd">&quot;&quot;&quot;Raised when a SMTP error occurs&quot;&quot;&quot;</span>
</div>
<div class="viewcode-block" id="InvalidDMARCReport"><a class="viewcode-back" href="../index.html#parsedmarc.InvalidDMARCReport">[docs]</a><span class="k">class</span> <span class="nc">InvalidDMARCReport</span><span class="p">(</span><span class="n">ParserError</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;Raised when an invalid DMARC report is encountered&quot;&quot;&quot;</span></div>
<span class="sd">&quot;&quot;&quot;Raised when an invalid DMARC report is encountered&quot;&quot;&quot;</span>
</div>
<div class="viewcode-block" id="InvalidAggregateReport"><a class="viewcode-back" href="../index.html#parsedmarc.InvalidAggregateReport">[docs]</a><span class="k">class</span> <span class="nc">InvalidAggregateReport</span><span class="p">(</span><span class="n">InvalidDMARCReport</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;Raised when an invalid DMARC aggregate report is encountered&quot;&quot;&quot;</span></div>
<span class="sd">&quot;&quot;&quot;Raised when an invalid DMARC aggregate report is encountered&quot;&quot;&quot;</span>
</div>
<div class="viewcode-block" id="InvalidForensicReport"><a class="viewcode-back" href="../index.html#parsedmarc.InvalidForensicReport">[docs]</a><span class="k">class</span> <span class="nc">InvalidForensicReport</span><span class="p">(</span><span class="n">InvalidDMARCReport</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;Raised when an invalid DMARC forensic report is encountered&quot;&quot;&quot;</span></div>
<span class="sd">&quot;&quot;&quot;Raised when an invalid DMARC forensic report is encountered&quot;&quot;&quot;</span>
</div>
<span class="k">def</span> <span class="nf">_get_base_domain</span><span class="p">(</span><span class="n">domain</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
@@ -338,7 +338,7 @@
<span class="k">return</span> <span class="n">datetime</span><span class="o">.</span><span class="n">fromtimestamp</span><span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="n">timestamp</span><span class="p">))</span>
<div class="viewcode-block" id="timestamp_to_human"><a class="viewcode-back" href="../index.html#parsedmarc.timestamp_to_human">[docs]</a><span class="k">def</span> <span class="nf">timestamp_to_human</span><span class="p">(</span><span class="n">timestamp</span><span class="p">):</span>
<span class="k">def</span> <span class="nf">_timestamp_to_human</span><span class="p">(</span><span class="n">timestamp</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Converts a UNIX/DMARC timestamp to a human-readable string</span>
@@ -348,7 +348,7 @@
<span class="sd"> Returns:</span>
<span class="sd"> str: The converted timestamp in ``YYYY-MM-DD HH:MM:SS`` format</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">return</span> <span class="n">_timestamp_to_datetime</span><span class="p">(</span><span class="n">timestamp</span><span class="p">)</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%S&quot;</span><span class="p">)</span></div>
<span class="k">return</span> <span class="n">_timestamp_to_datetime</span><span class="p">(</span><span class="n">timestamp</span><span class="p">)</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%S&quot;</span><span class="p">)</span>
<div class="viewcode-block" id="human_timestamp_to_datetime"><a class="viewcode-back" href="../index.html#parsedmarc.human_timestamp_to_datetime">[docs]</a><span class="k">def</span> <span class="nf">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">human_timestamp</span><span class="p">):</span>
@@ -361,8 +361,8 @@
<span class="sd"> Returns:</span>
<span class="sd"> DateTime: The converted timestamp</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">return</span> <span class="n">datetime</span><span class="o">.</span><span class="n">strptime</span><span class="p">(</span><span class="n">human_timestamp</span><span class="p">,</span> <span class="s2">&quot;%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%S&quot;</span><span class="p">)</span></div>
<span class="k">return</span> <span class="n">datetime</span><span class="o">.</span><span class="n">strptime</span><span class="p">(</span><span class="n">human_timestamp</span><span class="p">,</span> <span class="s2">&quot;%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%S&quot;</span><span class="p">)</span>
</div>
<span class="k">def</span> <span class="nf">_get_ip_address_country</span><span class="p">(</span><span class="n">ip_address</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
@@ -584,8 +584,8 @@
<span class="s2">&quot;&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;&gt;&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">&quot;@&quot;</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">&quot;report_id&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">report_id</span>
<span class="n">date_range</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">&quot;report_metadata&quot;</span><span class="p">][</span><span class="s2">&quot;date_range&quot;</span><span class="p">]</span>
<span class="n">date_range</span><span class="p">[</span><span class="s2">&quot;begin&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">timestamp_to_human</span><span class="p">(</span><span class="n">date_range</span><span class="p">[</span><span class="s2">&quot;begin&quot;</span><span class="p">])</span>
<span class="n">date_range</span><span class="p">[</span><span class="s2">&quot;end&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">timestamp_to_human</span><span class="p">(</span><span class="n">date_range</span><span class="p">[</span><span class="s2">&quot;end&quot;</span><span class="p">])</span>
<span class="n">date_range</span><span class="p">[</span><span class="s2">&quot;begin&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">_timestamp_to_human</span><span class="p">(</span><span class="n">date_range</span><span class="p">[</span><span class="s2">&quot;begin&quot;</span><span class="p">])</span>
<span class="n">date_range</span><span class="p">[</span><span class="s2">&quot;end&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">_timestamp_to_human</span><span class="p">(</span><span class="n">date_range</span><span class="p">[</span><span class="s2">&quot;end&quot;</span><span class="p">])</span>
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">&quot;begin_date&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">date_range</span><span class="p">[</span><span class="s2">&quot;begin&quot;</span><span class="p">]</span>
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">&quot;end_date&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">date_range</span><span class="p">[</span><span class="s2">&quot;end&quot;</span><span class="p">]</span>
<span class="n">errors</span> <span class="o">=</span> <span class="p">[]</span>
@@ -643,8 +643,8 @@
<span class="k">except</span> <span class="ne">KeyError</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">InvalidAggregateReport</span><span class="p">(</span><span class="s2">&quot;Missing field: &quot;</span>
<span class="s2">&quot;</span><span class="si">{0}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
<span class="s2">&quot;</span><span class="si">{0}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
</div>
<div class="viewcode-block" id="extract_xml"><a class="viewcode-back" href="../index.html#parsedmarc.extract_xml">[docs]</a><span class="k">def</span> <span class="nf">extract_xml</span><span class="p">(</span><span class="n">input_</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
@@ -684,8 +684,8 @@
<span class="k">raise</span> <span class="n">InvalidAggregateReport</span><span class="p">(</span><span class="s2">&quot;File objects must be opened in binary &quot;</span>
<span class="s2">&quot;(rb) mode&quot;</span><span class="p">)</span>
<span class="k">return</span> <span class="n">xml</span></div>
<span class="k">return</span> <span class="n">xml</span>
</div>
<div class="viewcode-block" id="parse_aggregate_report_file"><a class="viewcode-back" href="../index.html#parsedmarc.parse_aggregate_report_file">[docs]</a><span class="k">def</span> <span class="nf">parse_aggregate_report_file</span><span class="p">(</span><span class="n">_input</span><span class="p">,</span> <span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mf">6.0</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;Parses a file at the given path, a file-like object. or bytes as a</span>
@@ -704,8 +704,8 @@
<span class="k">return</span> <span class="n">parse_aggregate_report_xml</span><span class="p">(</span><span class="n">xml</span><span class="p">,</span>
<span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
<span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">)</span></div>
<span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">)</span>
</div>
<div class="viewcode-block" id="parsed_aggregate_reports_to_csv"><a class="viewcode-back" href="../index.html#parsedmarc.parsed_aggregate_reports_to_csv">[docs]</a><span class="k">def</span> <span class="nf">parsed_aggregate_reports_to_csv</span><span class="p">(</span><span class="n">reports</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
@@ -808,8 +808,8 @@
<span class="n">writer</span><span class="o">.</span><span class="n">writerow</span><span class="p">(</span><span class="n">row</span><span class="p">)</span>
<span class="n">csv_file_object</span><span class="o">.</span><span class="n">flush</span><span class="p">()</span>
<span class="k">return</span> <span class="n">csv_file_object</span><span class="o">.</span><span class="n">getvalue</span><span class="p">()</span></div>
<span class="k">return</span> <span class="n">csv_file_object</span><span class="o">.</span><span class="n">getvalue</span><span class="p">()</span>
</div>
<div class="viewcode-block" id="parse_forensic_report"><a class="viewcode-back" href="../index.html#parsedmarc.parse_forensic_report">[docs]</a><span class="k">def</span> <span class="nf">parse_forensic_report</span><span class="p">(</span><span class="n">feedback_report</span><span class="p">,</span> <span class="n">sample</span><span class="p">,</span> <span class="n">sample_headers_only</span><span class="p">,</span>
<span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mf">6.0</span><span class="p">):</span>
@@ -962,8 +962,8 @@
<span class="k">except</span> <span class="ne">KeyError</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">InvalidForensicReport</span><span class="p">(</span><span class="s2">&quot;Missing value: </span><span class="si">{0}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
<span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
<span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
</div>
<div class="viewcode-block" id="parsed_forensic_reports_to_csv"><a class="viewcode-back" href="../index.html#parsedmarc.parsed_forensic_reports_to_csv">[docs]</a><span class="k">def</span> <span class="nf">parsed_forensic_reports_to_csv</span><span class="p">(</span><span class="n">reports</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
@@ -1005,8 +1005,8 @@
<span class="k">del</span> <span class="n">row</span><span class="p">[</span><span class="s2">&quot;parsed_sample&quot;</span><span class="p">]</span>
<span class="n">csv_writer</span><span class="o">.</span><span class="n">writerow</span><span class="p">(</span><span class="n">row</span><span class="p">)</span>
<span class="k">return</span> <span class="n">csv_file</span><span class="o">.</span><span class="n">getvalue</span><span class="p">()</span></div>
<span class="k">return</span> <span class="n">csv_file</span><span class="o">.</span><span class="n">getvalue</span><span class="p">()</span>
</div>
<div class="viewcode-block" id="parse_report_email"><a class="viewcode-back" href="../index.html#parsedmarc.parse_report_email">[docs]</a><span class="k">def</span> <span class="nf">parse_report_email</span><span class="p">(</span><span class="n">input_</span><span class="p">,</span> <span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mf">6.0</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
@@ -1133,8 +1133,8 @@
<span class="s1">&#39;not a valid DMARC report&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">subject</span><span class="p">)</span>
<span class="k">raise</span> <span class="n">InvalidDMARCReport</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
<span class="k">return</span> <span class="n">result</span></div>
<span class="k">return</span> <span class="n">result</span>
</div>
<div class="viewcode-block" id="parse_report_file"><a class="viewcode-back" href="../index.html#parsedmarc.parse_report_file">[docs]</a><span class="k">def</span> <span class="nf">parse_report_file</span><span class="p">(</span><span class="n">input_</span><span class="p">,</span> <span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mf">6.0</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;Parses a DMARC aggregate or forensic file at the given path, a</span>
@@ -1170,8 +1170,8 @@
<span class="k">except</span> <span class="n">InvalidDMARCReport</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">InvalidDMARCReport</span><span class="p">(</span><span class="s2">&quot;Not a valid aggregate or forensic &quot;</span>
<span class="s2">&quot;report&quot;</span><span class="p">)</span>
<span class="k">return</span> <span class="n">results</span></div>
<span class="k">return</span> <span class="n">results</span>
</div>
<div class="viewcode-block" id="get_dmarc_reports_from_inbox"><a class="viewcode-back" href="../index.html#parsedmarc.get_dmarc_reports_from_inbox">[docs]</a><span class="k">def</span> <span class="nf">get_dmarc_reports_from_inbox</span><span class="p">(</span><span class="n">host</span><span class="p">,</span> <span class="n">user</span><span class="p">,</span> <span class="n">password</span><span class="p">,</span>
<span class="n">reports_folder</span><span class="o">=</span><span class="s2">&quot;INBOX&quot;</span><span class="p">,</span>
@@ -1277,8 +1277,8 @@
<span class="k">except</span> <span class="n">ssl</span><span class="o">.</span><span class="n">SSLError</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">IMAPError</span><span class="p">(</span><span class="s2">&quot;SSL error: </span><span class="si">{0}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
<span class="k">except</span> <span class="n">ssl</span><span class="o">.</span><span class="n">CertificateError</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">IMAPError</span><span class="p">(</span><span class="s2">&quot;Certificate error: </span><span class="si">{0}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
<span class="k">raise</span> <span class="n">IMAPError</span><span class="p">(</span><span class="s2">&quot;Certificate error: </span><span class="si">{0}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
</div>
<div class="viewcode-block" id="save_output"><a class="viewcode-back" href="../index.html#parsedmarc.save_output">[docs]</a><span class="k">def</span> <span class="nf">save_output</span><span class="p">(</span><span class="n">results</span><span class="p">,</span> <span class="n">output_directory</span><span class="o">=</span><span class="s2">&quot;output&quot;</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
@@ -1339,8 +1339,8 @@
<span class="n">filename</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">{0}</span><span class="s2">.eml&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">filename</span><span class="p">)</span>
<span class="n">path</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">samples_directory</span><span class="p">,</span> <span class="n">filename</span><span class="p">)</span>
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="s2">&quot;w&quot;</span><span class="p">,</span> <span class="n">newline</span><span class="o">=</span><span class="s2">&quot;</span><span class="se">\n</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s2">&quot;utf-8&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">sample_file</span><span class="p">:</span>
<span class="n">sample_file</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">sample</span><span class="p">)</span></div>
<span class="n">sample_file</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">sample</span><span class="p">)</span>
</div>
<div class="viewcode-block" id="get_report_zip"><a class="viewcode-back" href="../index.html#parsedmarc.get_report_zip">[docs]</a><span class="k">def</span> <span class="nf">get_report_zip</span><span class="p">(</span><span class="n">results</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
@@ -1384,8 +1384,8 @@
<span class="k">finally</span><span class="p">:</span>
<span class="n">shutil</span><span class="o">.</span><span class="n">rmtree</span><span class="p">(</span><span class="n">tmp_dir</span><span class="p">)</span>
<span class="k">return</span> <span class="n">storage</span><span class="o">.</span><span class="n">getvalue</span><span class="p">()</span></div>
<span class="k">return</span> <span class="n">storage</span><span class="o">.</span><span class="n">getvalue</span><span class="p">()</span>
</div>
<div class="viewcode-block" id="email_results"><a class="viewcode-back" href="../index.html#parsedmarc.email_results">[docs]</a><span class="k">def</span> <span class="nf">email_results</span><span class="p">(</span><span class="n">results</span><span class="p">,</span> <span class="n">host</span><span class="p">,</span> <span class="n">mail_from</span><span class="p">,</span> <span class="n">mail_to</span><span class="p">,</span> <span class="n">port</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">starttls</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
<span class="n">use_ssl</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">user</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">password</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">subject</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
@@ -1464,8 +1464,8 @@
<span class="k">except</span> <span class="n">ssl</span><span class="o">.</span><span class="n">SSLError</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">SMTPError</span><span class="p">(</span><span class="s2">&quot;SSL error: </span><span class="si">{0}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
<span class="k">except</span> <span class="n">ssl</span><span class="o">.</span><span class="n">CertificateError</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">SMTPError</span><span class="p">(</span><span class="s2">&quot;Certificate error: </span><span class="si">{0}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
<span class="k">raise</span> <span class="n">SMTPError</span><span class="p">(</span><span class="s2">&quot;Certificate error: </span><span class="si">{0}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
</div>
<div class="viewcode-block" id="watch_inbox"><a class="viewcode-back" href="../index.html#parsedmarc.watch_inbox">[docs]</a><span class="k">def</span> <span class="nf">watch_inbox</span><span class="p">(</span><span class="n">host</span><span class="p">,</span> <span class="n">username</span><span class="p">,</span> <span class="n">password</span><span class="p">,</span> <span class="n">callback</span><span class="p">,</span> <span class="n">reports_folder</span><span class="o">=</span><span class="s2">&quot;INBOX&quot;</span><span class="p">,</span>
<span class="n">archive_folder</span><span class="o">=</span><span class="s2">&quot;Archive&quot;</span><span class="p">,</span> <span class="n">delete</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">test</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">wait</span><span class="o">=</span><span class="mi">30</span><span class="p">,</span>
@@ -1516,6 +1516,7 @@
<span class="n">nameservers</span><span class="o">=</span><span class="n">ns</span><span class="p">,</span>
<span class="n">dns_timeout</span><span class="o">=</span><span class="n">dt</span><span class="p">)</span>
<span class="n">callback</span><span class="p">(</span><span class="n">res</span><span class="p">)</span>
<span class="k">break</span>
<span class="k">except</span> <span class="n">imapclient</span><span class="o">.</span><span class="n">exceptions</span><span class="o">.</span><span class="n">IMAPClientError</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
<span class="n">error</span> <span class="o">=</span> <span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="o">.</span><span class="n">lstrip</span><span class="p">(</span><span class="s2">&quot;b&#39;&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s2">&quot;&#39;&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s2">&quot;.&quot;</span><span class="p">)</span>
@@ -1539,8 +1540,8 @@
<span class="n">server</span><span class="o">.</span><span class="n">idle_done</span><span class="p">()</span>
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;</span><span class="se">\n</span><span class="s2">IDLE mode done&quot;</span><span class="p">)</span>
<span class="n">server</span><span class="o">.</span><span class="n">logout</span><span class="p">()</span></div>
</pre></div>
<span class="n">server</span><span class="o">.</span><span class="n">logout</span><span class="p">()</span>
</pre></div></div>
</div>
<div class="articleComments">
@@ -1576,7 +1577,7 @@
<script type="text/javascript">
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../',
VERSION:'3.2.0',
VERSION:'3.3.0',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
+584
View File
@@ -0,0 +1,584 @@
<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>parsedmarc.elastic &mdash; parsedmarc 3.3.0 documentation</title>
<link rel="stylesheet" href="../../_static/css/theme.css" type="text/css" />
<link rel="index" title="Index"
href="../../genindex.html"/>
<link rel="search" title="Search" href="../../search.html"/>
<link rel="top" title="parsedmarc 3.3.0 documentation" href="../../index.html"/>
<link rel="up" title="parsedmarc" href="../parsedmarc.html"/>
<script src="../../_static/js/modernizr.min.js"></script>
</head>
<body class="wy-body-for-nav" role="document">
<div class="wy-grid-for-nav">
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
<div class="wy-side-scroll">
<div class="wy-side-nav-search">
<a href="../../index.html" class="icon icon-home"> parsedmarc
</a>
<div class="version">
3.3.0
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
<input type="text" name="q" placeholder="Search docs" />
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
</form>
</div>
</div>
<div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
<!-- Local TOC -->
<div class="local-toc"></div>
</div>
</div>
</nav>
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
<nav class="wy-nav-top" role="navigation" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../index.html">parsedmarc</a>
</nav>
<div class="wy-nav-content">
<div class="rst-content">
<div role="navigation" aria-label="breadcrumbs navigation">
<ul class="wy-breadcrumbs">
<li><a href="../../index.html">Docs</a> &raquo;</li>
<li><a href="../index.html">Module code</a> &raquo;</li>
<li><a href="../parsedmarc.html">parsedmarc</a> &raquo;</li>
<li>parsedmarc.elastic</li>
<li class="wy-breadcrumbs-aside">
</li>
</ul>
<hr/>
</div>
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
<div itemprop="articleBody">
<h1>Source code for parsedmarc.elastic</h1><div class="highlight"><pre>
<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
<span class="kn">from</span> <span class="nn">collections</span> <span class="k">import</span> <span class="n">OrderedDict</span>
<span class="kn">import</span> <span class="nn">parsedmarc</span>
<span class="kn">from</span> <span class="nn">elasticsearch_dsl.search</span> <span class="k">import</span> <span class="n">Q</span>
<span class="kn">from</span> <span class="nn">elasticsearch_dsl</span> <span class="k">import</span> <span class="n">connections</span><span class="p">,</span> <span class="n">Object</span><span class="p">,</span> <span class="n">DocType</span><span class="p">,</span> <span class="n">Index</span><span class="p">,</span> <span class="n">Nested</span><span class="p">,</span> \
<span class="n">InnerDoc</span><span class="p">,</span> <span class="n">Integer</span><span class="p">,</span> <span class="n">Text</span><span class="p">,</span> <span class="n">Boolean</span><span class="p">,</span> <span class="n">DateRange</span><span class="p">,</span> <span class="n">Ip</span><span class="p">,</span> <span class="n">Date</span>
<span class="n">aggregate_index</span> <span class="o">=</span> <span class="n">Index</span><span class="p">(</span><span class="s2">&quot;dmarc_aggregate&quot;</span><span class="p">)</span>
<span class="n">forensic_index</span> <span class="o">=</span> <span class="n">Index</span><span class="p">(</span><span class="s2">&quot;dmarc_forensic&quot;</span><span class="p">)</span>
<span class="k">class</span> <span class="nc">_PolicyOverride</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
<span class="nb">type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">comment</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="k">class</span> <span class="nc">_PublishedPolicy</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
<span class="n">adkim</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">aspf</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">p</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">sp</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">pct</span> <span class="o">=</span> <span class="n">Integer</span><span class="p">()</span>
<span class="n">fo</span> <span class="o">=</span> <span class="n">Integer</span><span class="p">()</span>
<span class="k">class</span> <span class="nc">_DKIMResult</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
<span class="n">domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">selector</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">result</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="k">class</span> <span class="nc">_SPFResult</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
<span class="n">domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">scope</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">results</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="k">class</span> <span class="nc">_AggregateReportDoc</span><span class="p">(</span><span class="n">DocType</span><span class="p">):</span>
<span class="k">class</span> <span class="nc">Meta</span><span class="p">:</span>
<span class="n">index</span> <span class="o">=</span> <span class="s2">&quot;dmarc_aggregate&quot;</span>
<span class="n">xml_schema</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">org_name</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">org_email</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">org_extra_contact_info</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">report_id</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">date_range</span> <span class="o">=</span> <span class="n">DateRange</span><span class="p">()</span>
<span class="n">errors</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">published_policy</span> <span class="o">=</span> <span class="n">Object</span><span class="p">(</span><span class="n">_PublishedPolicy</span><span class="p">)</span>
<span class="n">source_ip_address</span> <span class="o">=</span> <span class="n">Ip</span><span class="p">()</span>
<span class="n">source_country</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">source_reverse_dns</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">source_Base_domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">message_count</span> <span class="o">=</span> <span class="n">Integer</span>
<span class="n">disposition</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">dkim_aligned</span> <span class="o">=</span> <span class="n">Boolean</span><span class="p">()</span>
<span class="n">spf_aligned</span> <span class="o">=</span> <span class="n">Boolean</span><span class="p">()</span>
<span class="n">passed_dmarc</span> <span class="o">=</span> <span class="n">Boolean</span><span class="p">()</span>
<span class="n">policy_overrides</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_PolicyOverride</span><span class="p">)</span>
<span class="n">header_from</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">envelope_from</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">envelope_to</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">dkim_results</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_DKIMResult</span><span class="p">)</span>
<span class="n">spf_results</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SPFResult</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">add_policy_override</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">type_</span><span class="p">,</span> <span class="n">comment</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">policy_overrides</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_PolicyOverride</span><span class="p">(</span><span class="nb">type</span><span class="o">=</span><span class="n">type_</span><span class="p">,</span>
<span class="n">comment</span><span class="o">=</span><span class="n">comment</span><span class="p">))</span>
<span class="k">def</span> <span class="nf">add_dkim_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">domain</span><span class="p">,</span> <span class="n">selector</span><span class="p">,</span> <span class="n">result</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">dkim_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_DKIMResult</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span>
<span class="n">selector</span><span class="o">=</span><span class="n">selector</span><span class="p">,</span>
<span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">))</span>
<span class="k">def</span> <span class="nf">add_spf_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">domain</span><span class="p">,</span> <span class="n">scope</span><span class="p">,</span> <span class="n">result</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">spf_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_SPFResult</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span>
<span class="n">scope</span><span class="o">=</span><span class="n">scope</span><span class="p">,</span>
<span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">))</span>
<span class="k">def</span> <span class="nf">save</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">**</span> <span class="n">kwargs</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">passed_dmarc</span> <span class="o">=</span> <span class="kc">False</span>
<span class="bp">self</span><span class="o">.</span><span class="n">passed_dmarc</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">spf_aligned</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">dkim_aligned</span>
<span class="k">return</span> <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="o">**</span> <span class="n">kwargs</span><span class="p">)</span>
<span class="k">class</span> <span class="nc">_EmailAddressDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
<span class="n">display_name</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">address</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="k">class</span> <span class="nc">_EmailAttachmentDoc</span><span class="p">(</span><span class="n">DocType</span><span class="p">):</span>
<span class="n">filename</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">content_type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="k">class</span> <span class="nc">_ForensicSampleDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
<span class="n">raw</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">headers</span> <span class="o">=</span> <span class="n">Object</span><span class="p">()</span>
<span class="n">headers_only</span> <span class="o">=</span> <span class="n">Boolean</span><span class="p">()</span>
<span class="n">to</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">)</span>
<span class="n">subject</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">filename_safe_subject</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">_from</span> <span class="o">=</span> <span class="n">Object</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">)</span>
<span class="n">date</span> <span class="o">=</span> <span class="n">Date</span><span class="p">()</span>
<span class="n">reply_to</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">)</span>
<span class="n">cc</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">)</span>
<span class="n">bcc</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">)</span>
<span class="n">body</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">attachments</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_EmailAttachmentDoc</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">add_to</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">to</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span>
<span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
<span class="k">def</span> <span class="nf">add_reply_to</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">reply_to</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span>
<span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
<span class="k">def</span> <span class="nf">add_cc</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">cc</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span>
<span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
<span class="k">def</span> <span class="nf">add_bcc</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">bcc</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span>
<span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
<span class="k">def</span> <span class="nf">add_attachment</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">filename</span><span class="p">,</span> <span class="n">content_type</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">attachments</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">filename</span><span class="o">=</span><span class="n">filename</span><span class="p">,</span>
<span class="n">content_type</span><span class="o">=</span><span class="n">content_type</span><span class="p">)</span>
<span class="k">class</span> <span class="nc">_ForensicReportDoc</span><span class="p">(</span><span class="n">DocType</span><span class="p">):</span>
<span class="k">class</span> <span class="nc">Meta</span><span class="p">:</span>
<span class="n">index</span> <span class="o">=</span> <span class="s2">&quot;dmarc_forensic&quot;</span>
<span class="n">feedback_type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">user_agent</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">version</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">original_mail_from</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">arrival_date</span> <span class="o">=</span> <span class="n">Date</span><span class="p">()</span>
<span class="n">domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">original_envelope_id</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">authentication_results</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">delivery_results</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">source_ip_address</span> <span class="o">=</span> <span class="n">Ip</span><span class="p">()</span>
<span class="n">source_country</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">source_reverse_dns</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">source_authentication_mechanisms</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">source_auth_failures</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">dkim_domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">original_rcpt_to</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">sample</span> <span class="o">=</span> <span class="n">Object</span><span class="p">(</span><span class="n">_ForensicSampleDoc</span><span class="p">)</span>
<div class="viewcode-block" id="AlreadySaved"><a class="viewcode-back" href="../../index.html#parsedmarc.elastic.AlreadySaved">[docs]</a><span class="k">class</span> <span class="nc">AlreadySaved</span><span class="p">(</span><span class="ne">ValueError</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;Raised when a report to be saved matches an existing report&quot;&quot;&quot;</span></div>
<div class="viewcode-block" id="set_hosts"><a class="viewcode-back" href="../../index.html#parsedmarc.elastic.set_hosts">[docs]</a><span class="k">def</span> <span class="nf">set_hosts</span><span class="p">(</span><span class="n">hosts</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Sets the Elasticsearch hosts to use</span>
<span class="sd"> Args:</span>
<span class="sd"> hosts: A single hostname or URL, or list of hostnames or URLs</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">hosts</span><span class="p">)</span> <span class="o">!=</span> <span class="nb">list</span><span class="p">:</span>
<span class="n">hosts</span> <span class="o">=</span> <span class="p">[</span><span class="n">hosts</span><span class="p">]</span>
<span class="n">connections</span><span class="o">.</span><span class="n">create_connection</span><span class="p">(</span><span class="n">hosts</span><span class="o">=</span><span class="n">hosts</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mi">20</span><span class="p">)</span></div>
<div class="viewcode-block" id="create_indexes"><a class="viewcode-back" href="../../index.html#parsedmarc.elastic.create_indexes">[docs]</a><span class="k">def</span> <span class="nf">create_indexes</span><span class="p">():</span>
<span class="sd">&quot;&quot;&quot;Creates the required indexes&quot;&quot;&quot;</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">aggregate_index</span><span class="o">.</span><span class="n">exists</span><span class="p">():</span>
<span class="n">aggregate_index</span><span class="o">.</span><span class="n">create</span><span class="p">()</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">forensic_index</span><span class="o">.</span><span class="n">exists</span><span class="p">():</span>
<span class="n">forensic_index</span><span class="o">.</span><span class="n">create</span><span class="p">()</span></div>
<div class="viewcode-block" id="save_aggregate_report_to_elasticsearch"><a class="viewcode-back" href="../../index.html#parsedmarc.elastic.save_aggregate_report_to_elasticsearch">[docs]</a><span class="k">def</span> <span class="nf">save_aggregate_report_to_elasticsearch</span><span class="p">(</span><span class="n">aggregate_report</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Saves a parsed DMARC aggregate report to ElasticSearch</span>
<span class="sd"> Args:</span>
<span class="sd"> aggregate_report (OrderedDict): A parsed forensic report</span>
<span class="sd"> Raises:</span>
<span class="sd"> AlreadySaved</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">aggregate_report</span> <span class="o">=</span> <span class="n">aggregate_report</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="n">metadata</span> <span class="o">=</span> <span class="n">aggregate_report</span><span class="p">[</span><span class="s2">&quot;report_metadata&quot;</span><span class="p">]</span>
<span class="n">org_name</span> <span class="o">=</span> <span class="n">metadata</span><span class="p">[</span><span class="s2">&quot;org_name&quot;</span><span class="p">]</span>
<span class="n">domain</span> <span class="o">=</span> <span class="n">aggregate_report</span><span class="p">[</span><span class="s2">&quot;policy_published&quot;</span><span class="p">][</span><span class="s2">&quot;domain&quot;</span><span class="p">]</span>
<span class="n">begin_date</span> <span class="o">=</span> <span class="n">parsedmarc</span><span class="o">.</span><span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">metadata</span><span class="p">[</span><span class="s2">&quot;begin_date&quot;</span><span class="p">])</span>
<span class="n">end_date</span> <span class="o">=</span> <span class="n">parsedmarc</span><span class="o">.</span><span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">metadata</span><span class="p">[</span><span class="s2">&quot;end_date&quot;</span><span class="p">])</span>
<span class="n">begin_date_human</span> <span class="o">=</span> <span class="n">begin_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%S&quot;</span><span class="p">)</span>
<span class="n">end_date_human</span> <span class="o">=</span> <span class="n">end_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%S&quot;</span><span class="p">)</span>
<span class="n">aggregate_report</span><span class="p">[</span><span class="s2">&quot;begin_date&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">begin_date</span>
<span class="n">aggregate_report</span><span class="p">[</span><span class="s2">&quot;end_date&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">end_date</span>
<span class="n">date_range</span> <span class="o">=</span> <span class="p">(</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">&quot;begin_date&quot;</span><span class="p">],</span>
<span class="n">aggregate_report</span><span class="p">[</span><span class="s2">&quot;end_date&quot;</span><span class="p">])</span>
<span class="n">org_name_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">org_name</span><span class="o">=</span><span class="n">org_name</span><span class="p">)))</span>
<span class="n">domain_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">)))</span>
<span class="n">begin_date_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">date_range</span><span class="o">=</span><span class="n">begin_date</span><span class="p">)))</span>
<span class="n">end_date_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">date_range</span><span class="o">=</span><span class="n">end_date</span><span class="p">)))</span>
<span class="n">search</span> <span class="o">=</span> <span class="n">aggregate_index</span><span class="o">.</span><span class="n">search</span><span class="p">()</span>
<span class="n">search</span><span class="o">.</span><span class="n">query</span> <span class="o">=</span> <span class="n">org_name_query</span> <span class="o">&amp;</span> <span class="n">domain_query</span> <span class="o">&amp;</span> <span class="n">begin_date_query</span> <span class="o">&amp;</span> \
<span class="n">end_date_query</span>
<span class="n">existing</span> <span class="o">=</span> <span class="n">search</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">existing</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">AlreadySaved</span><span class="p">(</span><span class="s2">&quot;An aggregate report from </span><span class="si">{0}</span><span class="s2"> about </span><span class="si">{1}</span><span class="s2"> with a &quot;</span>
<span class="s2">&quot;date range of </span><span class="si">{2}</span><span class="s2"> UTC to </span><span class="si">{3}</span><span class="s2"> UTC already exists &quot;</span>
<span class="s2">&quot;in Elasticsearch&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">org_name</span><span class="p">,</span>
<span class="n">domain</span><span class="p">,</span>
<span class="n">begin_date_human</span><span class="p">,</span>
<span class="n">end_date_human</span><span class="p">))</span>
<span class="n">published_policy</span> <span class="o">=</span> <span class="n">_PublishedPolicy</span><span class="p">(</span>
<span class="n">adkim</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">&quot;policy_published&quot;</span><span class="p">][</span><span class="s2">&quot;adkim&quot;</span><span class="p">],</span>
<span class="n">aspf</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">&quot;policy_published&quot;</span><span class="p">][</span><span class="s2">&quot;aspf&quot;</span><span class="p">],</span>
<span class="n">p</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">&quot;policy_published&quot;</span><span class="p">][</span><span class="s2">&quot;p&quot;</span><span class="p">],</span>
<span class="n">sp</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">&quot;policy_published&quot;</span><span class="p">][</span><span class="s2">&quot;sp&quot;</span><span class="p">],</span>
<span class="n">pct</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">&quot;policy_published&quot;</span><span class="p">][</span><span class="s2">&quot;pct&quot;</span><span class="p">],</span>
<span class="n">fo</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">&quot;policy_published&quot;</span><span class="p">][</span><span class="s2">&quot;fo&quot;</span><span class="p">]</span>
<span class="p">)</span>
<span class="k">for</span> <span class="n">record</span> <span class="ow">in</span> <span class="n">aggregate_report</span><span class="p">[</span><span class="s2">&quot;records&quot;</span><span class="p">]:</span>
<span class="n">agg_doc</span> <span class="o">=</span> <span class="n">_AggregateReportDoc</span><span class="p">(</span>
<span class="n">xml_schemea</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">&quot;xml_schema&quot;</span><span class="p">],</span>
<span class="n">org_name</span><span class="o">=</span><span class="n">metadata</span><span class="p">[</span><span class="s2">&quot;org_name&quot;</span><span class="p">],</span>
<span class="n">org_email</span><span class="o">=</span><span class="n">metadata</span><span class="p">[</span><span class="s2">&quot;org_email&quot;</span><span class="p">],</span>
<span class="n">org_extra_contact_info</span><span class="o">=</span><span class="n">metadata</span><span class="p">[</span><span class="s2">&quot;org_extra_contact_info&quot;</span><span class="p">],</span>
<span class="n">report_id</span><span class="o">=</span><span class="n">metadata</span><span class="p">[</span><span class="s2">&quot;report_id&quot;</span><span class="p">],</span>
<span class="n">date_range</span><span class="o">=</span><span class="n">date_range</span><span class="p">,</span>
<span class="n">errors</span><span class="o">=</span><span class="n">metadata</span><span class="p">[</span><span class="s2">&quot;errors&quot;</span><span class="p">],</span>
<span class="n">domain</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">&quot;policy_published&quot;</span><span class="p">][</span><span class="s2">&quot;domain&quot;</span><span class="p">],</span>
<span class="n">published_policy</span><span class="o">=</span><span class="n">published_policy</span><span class="p">,</span>
<span class="n">source_ip_address</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">&quot;source&quot;</span><span class="p">][</span><span class="s2">&quot;ip_address&quot;</span><span class="p">],</span>
<span class="n">source_country</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">&quot;source&quot;</span><span class="p">][</span><span class="s2">&quot;country&quot;</span><span class="p">],</span>
<span class="n">source_reverse_dns</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">&quot;source&quot;</span><span class="p">][</span><span class="s2">&quot;reverse_dns&quot;</span><span class="p">],</span>
<span class="n">source_base_domain</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">&quot;source&quot;</span><span class="p">][</span><span class="s2">&quot;base_domain&quot;</span><span class="p">],</span>
<span class="n">message_count</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">&quot;count&quot;</span><span class="p">],</span>
<span class="n">disposition</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">&quot;policy_evaluated&quot;</span><span class="p">][</span><span class="s2">&quot;disposition&quot;</span><span class="p">],</span>
<span class="n">dkim_aligned</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">&quot;policy_evaluated&quot;</span><span class="p">][</span><span class="s2">&quot;dkim&quot;</span><span class="p">]</span> <span class="o">==</span> <span class="s2">&quot;pass&quot;</span><span class="p">,</span>
<span class="n">spf_aligned</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">&quot;policy_evaluated&quot;</span><span class="p">][</span><span class="s2">&quot;spf&quot;</span><span class="p">]</span> <span class="o">==</span> <span class="s2">&quot;pass&quot;</span><span class="p">,</span>
<span class="n">header_from</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">&quot;identifiers&quot;</span><span class="p">][</span><span class="s2">&quot;header_from&quot;</span><span class="p">],</span>
<span class="n">envelope_from</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">&quot;identifiers&quot;</span><span class="p">][</span><span class="s2">&quot;envelope_from&quot;</span><span class="p">],</span>
<span class="n">envelope_to</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">&quot;identifiers&quot;</span><span class="p">][</span><span class="s2">&quot;envelope_to&quot;</span><span class="p">]</span>
<span class="p">)</span>
<span class="k">for</span> <span class="n">override</span> <span class="ow">in</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;policy_evaluated&quot;</span><span class="p">][</span><span class="s2">&quot;policy_override_reasons&quot;</span><span class="p">]:</span>
<span class="n">agg_doc</span><span class="o">.</span><span class="n">add_policy_override</span><span class="p">(</span><span class="n">type_</span><span class="o">=</span><span class="n">override</span><span class="p">[</span><span class="s2">&quot;type&quot;</span><span class="p">],</span>
<span class="n">comment</span><span class="o">=</span><span class="n">override</span><span class="p">[</span><span class="s2">&quot;comment&quot;</span><span class="p">])</span>
<span class="k">for</span> <span class="n">dkim_result</span> <span class="ow">in</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;auth_results&quot;</span><span class="p">][</span><span class="s2">&quot;dkim&quot;</span><span class="p">]:</span>
<span class="n">agg_doc</span><span class="o">.</span><span class="n">add_dkim_result</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">&quot;domain&quot;</span><span class="p">],</span>
<span class="n">selector</span><span class="o">=</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">&quot;selector&quot;</span><span class="p">],</span>
<span class="n">result</span><span class="o">=</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">&quot;result&quot;</span><span class="p">])</span>
<span class="k">for</span> <span class="n">spf_result</span> <span class="ow">in</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;auth_results&quot;</span><span class="p">][</span><span class="s2">&quot;spf&quot;</span><span class="p">]:</span>
<span class="n">agg_doc</span><span class="o">.</span><span class="n">add_spf_result</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">&quot;domain&quot;</span><span class="p">],</span>
<span class="n">scope</span><span class="o">=</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">&quot;scope&quot;</span><span class="p">],</span>
<span class="n">result</span><span class="o">=</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">&quot;result&quot;</span><span class="p">])</span>
<span class="n">agg_doc</span><span class="o">.</span><span class="n">save</span><span class="p">()</span></div>
<div class="viewcode-block" id="save_forensic_report_to_elasticsearch"><a class="viewcode-back" href="../../index.html#parsedmarc.elastic.save_forensic_report_to_elasticsearch">[docs]</a><span class="k">def</span> <span class="nf">save_forensic_report_to_elasticsearch</span><span class="p">(</span><span class="n">forensic_report</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Saves a parsed DMARC forensic report to ElasticSearch</span>
<span class="sd"> Args:</span>
<span class="sd"> forensic_report (OrderedDict): A parsed forensic report</span>
<span class="sd"> Raises:</span>
<span class="sd"> AlreadySaved</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">forensic_report</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="n">sample_date</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;parsed_sample&quot;</span><span class="p">][</span><span class="s2">&quot;date&quot;</span><span class="p">]</span>
<span class="n">sample_date</span> <span class="o">=</span> <span class="n">parsedmarc</span><span class="o">.</span><span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">sample_date</span><span class="p">)</span>
<span class="n">original_headers</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;parsed_sample&quot;</span><span class="p">][</span><span class="s2">&quot;headers&quot;</span><span class="p">]</span>
<span class="n">headers</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">()</span>
<span class="k">for</span> <span class="n">original_header</span> <span class="ow">in</span> <span class="n">original_headers</span><span class="p">:</span>
<span class="n">headers</span><span class="p">[</span><span class="n">original_header</span><span class="o">.</span><span class="n">lower</span><span class="p">()]</span> <span class="o">=</span> <span class="n">original_headers</span><span class="p">[</span><span class="n">original_header</span><span class="p">]</span>
<span class="n">arrival_date_human</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;arrival_date_utc&quot;</span><span class="p">]</span>
<span class="n">arrival_date</span> <span class="o">=</span> <span class="n">parsedmarc</span><span class="o">.</span><span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">arrival_date_human</span><span class="p">)</span>
<span class="n">search</span> <span class="o">=</span> <span class="n">forensic_index</span><span class="o">.</span><span class="n">search</span><span class="p">()</span>
<span class="n">to_query</span> <span class="o">=</span> <span class="p">{</span><span class="s2">&quot;match&quot;</span><span class="p">:</span> <span class="p">{</span><span class="s2">&quot;sample.headers.to&quot;</span><span class="p">:</span> <span class="n">headers</span><span class="p">[</span><span class="s2">&quot;to&quot;</span><span class="p">]}}</span>
<span class="n">from_query</span> <span class="o">=</span> <span class="p">{</span><span class="s2">&quot;match&quot;</span><span class="p">:</span> <span class="p">{</span><span class="s2">&quot;sample.headers.from&quot;</span><span class="p">:</span> <span class="n">headers</span><span class="p">[</span><span class="s2">&quot;from&quot;</span><span class="p">]}}</span>
<span class="n">subject_query</span> <span class="o">=</span> <span class="p">{</span><span class="s2">&quot;match&quot;</span><span class="p">:</span> <span class="p">{</span><span class="s2">&quot;sample.headers.subject&quot;</span><span class="p">:</span> <span class="n">headers</span><span class="p">[</span><span class="s2">&quot;subject&quot;</span><span class="p">]}}</span>
<span class="n">arrival_date_query</span> <span class="o">=</span> <span class="p">{</span><span class="s2">&quot;match&quot;</span><span class="p">:</span> <span class="p">{</span><span class="s2">&quot;sample.headers.arrival_date&quot;</span><span class="p">:</span> <span class="n">arrival_date</span>
<span class="p">}}</span>
<span class="n">q</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="n">to_query</span><span class="p">)</span> <span class="o">&amp;</span> <span class="n">Q</span><span class="p">(</span><span class="n">from_query</span><span class="p">)</span> <span class="o">&amp;</span> <span class="n">Q</span><span class="p">(</span><span class="n">subject_query</span><span class="p">)</span> <span class="o">&amp;</span> <span class="n">Q</span><span class="p">(</span><span class="n">arrival_date_query</span><span class="p">)</span>
<span class="n">search</span><span class="o">.</span><span class="n">query</span> <span class="o">=</span> <span class="n">q</span>
<span class="n">existing</span> <span class="o">=</span> <span class="n">search</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">existing</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">AlreadySaved</span><span class="p">(</span><span class="s2">&quot;A forensic sample to </span><span class="si">{0}</span><span class="s2"> from </span><span class="si">{1}</span><span class="s2"> &quot;</span>
<span class="s2">&quot;with a subject of </span><span class="si">{2}</span><span class="s2"> and arrival date of </span><span class="si">{3}</span><span class="s2"> &quot;</span>
<span class="s2">&quot;already exists in &quot;</span>
<span class="s2">&quot;Elasticsearch&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">headers</span><span class="p">[</span><span class="s2">&quot;to&quot;</span><span class="p">],</span>
<span class="n">headers</span><span class="p">[</span><span class="s2">&quot;from&quot;</span><span class="p">],</span>
<span class="n">headers</span><span class="p">[</span><span class="s2">&quot;subject&quot;</span><span class="p">],</span>
<span class="n">arrival_date_human</span>
<span class="p">))</span>
<span class="n">parsed_sample</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;parsed_sample&quot;</span><span class="p">]</span>
<span class="n">sample</span> <span class="o">=</span> <span class="n">_ForensicSampleDoc</span><span class="p">(</span>
<span class="n">raw</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;sample&quot;</span><span class="p">],</span>
<span class="n">headers</span><span class="o">=</span><span class="n">headers</span><span class="p">,</span>
<span class="n">headers_only</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;sample_headers_only&quot;</span><span class="p">],</span>
<span class="n">date</span><span class="o">=</span><span class="n">sample_date</span><span class="p">,</span>
<span class="n">subject</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;parsed_sample&quot;</span><span class="p">][</span><span class="s2">&quot;subject&quot;</span><span class="p">],</span>
<span class="n">filename_safe_subject</span><span class="o">=</span><span class="n">parsed_sample</span><span class="p">[</span><span class="s2">&quot;filename_safe_subject&quot;</span><span class="p">],</span>
<span class="n">body</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;parsed_sample&quot;</span><span class="p">][</span><span class="s2">&quot;body&quot;</span><span class="p">]</span>
<span class="p">)</span>
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;parsed_sample&quot;</span><span class="p">][</span><span class="s2">&quot;to&quot;</span><span class="p">]:</span>
<span class="n">sample</span><span class="o">.</span><span class="n">add_to</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">&quot;display_name&quot;</span><span class="p">],</span>
<span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">&quot;address&quot;</span><span class="p">])</span>
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;parsed_sample&quot;</span><span class="p">][</span><span class="s2">&quot;reply_to&quot;</span><span class="p">]:</span>
<span class="n">sample</span><span class="o">.</span><span class="n">add_reply_to</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">&quot;display_name&quot;</span><span class="p">],</span>
<span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">&quot;address&quot;</span><span class="p">])</span>
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;parsed_sample&quot;</span><span class="p">][</span><span class="s2">&quot;cc&quot;</span><span class="p">]:</span>
<span class="n">sample</span><span class="o">.</span><span class="n">add_cc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">&quot;display_name&quot;</span><span class="p">],</span>
<span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">&quot;address&quot;</span><span class="p">])</span>
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;parsed_sample&quot;</span><span class="p">][</span><span class="s2">&quot;bcc&quot;</span><span class="p">]:</span>
<span class="n">sample</span><span class="o">.</span><span class="n">add_bcc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">&quot;display_name&quot;</span><span class="p">],</span>
<span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">&quot;address&quot;</span><span class="p">])</span>
<span class="k">for</span> <span class="n">attachment</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;parsed_sample&quot;</span><span class="p">][</span><span class="s2">&quot;attachments&quot;</span><span class="p">]:</span>
<span class="n">sample</span><span class="o">.</span><span class="n">add_attachment</span><span class="p">(</span><span class="n">filename</span><span class="o">=</span><span class="n">attachment</span><span class="p">[</span><span class="s2">&quot;filename&quot;</span><span class="p">],</span>
<span class="n">content_type</span><span class="o">=</span><span class="n">attachment</span><span class="p">[</span><span class="s2">&quot;mail_content_type&quot;</span><span class="p">])</span>
<span class="n">forensic_doc</span> <span class="o">=</span> <span class="n">_ForensicReportDoc</span><span class="p">(</span>
<span class="n">feedback_type</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;feedback_type&quot;</span><span class="p">],</span>
<span class="n">user_agent</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;user_agent&quot;</span><span class="p">],</span>
<span class="n">version</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;version&quot;</span><span class="p">],</span>
<span class="n">original_mail_from</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;original_mail_from&quot;</span><span class="p">],</span>
<span class="n">arrival_date</span><span class="o">=</span><span class="n">arrival_date</span><span class="p">,</span>
<span class="n">domain</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;reported_domain&quot;</span><span class="p">],</span>
<span class="n">original_envelope_id</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;original_envelope_id&quot;</span><span class="p">],</span>
<span class="n">authentication_results</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;authentication_results&quot;</span><span class="p">],</span>
<span class="n">delivery_results</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;delivery_result&quot;</span><span class="p">],</span>
<span class="n">source_ip_address</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;source&quot;</span><span class="p">][</span><span class="s2">&quot;ip_address&quot;</span><span class="p">],</span>
<span class="n">source_country</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;source&quot;</span><span class="p">][</span><span class="s2">&quot;country&quot;</span><span class="p">],</span>
<span class="n">source_reverse_dns</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;source&quot;</span><span class="p">][</span><span class="s2">&quot;reverse_dns&quot;</span><span class="p">],</span>
<span class="n">source_base_domain</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;source&quot;</span><span class="p">][</span><span class="s2">&quot;base_domain&quot;</span><span class="p">],</span>
<span class="n">authentication_mechanisms</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;authentication_mechanisms&quot;</span><span class="p">],</span>
<span class="n">auth_failure</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;auth_failure&quot;</span><span class="p">],</span>
<span class="n">dkim_domain</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;dkim_domain&quot;</span><span class="p">],</span>
<span class="n">original_rcpt_to</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;original_rcpt_to&quot;</span><span class="p">],</span>
<span class="n">sample</span><span class="o">=</span><span class="n">sample</span>
<span class="p">)</span>
<span class="n">forensic_doc</span><span class="o">.</span><span class="n">save</span><span class="p">()</span></div>
</pre></div>
</div>
<div class="articleComments">
</div>
</div>
<footer>
<hr/>
<div role="contentinfo">
<p>
&copy; Copyright 2018, Sean Whalen.
</p>
</div>
Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
</div>
</div>
</section>
</div>
<script type="text/javascript">
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../',
VERSION:'3.3.0',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
SOURCELINK_SUFFIX: '.txt'
};
</script>
<script type="text/javascript" src="../../_static/jquery.js"></script>
<script type="text/javascript" src="../../_static/underscore.js"></script>
<script type="text/javascript" src="../../_static/doctools.js"></script>
<script type="text/javascript" src="../../_static/js/theme.js"></script>
<script type="text/javascript">
jQuery(function () {
SphinxRtdTheme.StickyNav.enable();
});
</script>
</body>
</html>
+6
View File
@@ -253,6 +253,12 @@ API
.. automodule:: parsedmarc
:members:
parsedmarc.elastic
------------------
.. automodule:: parsedmarc.elastic
:members:
.. toctree::
:maxdepth: 2
:caption: Contents:
+18 -1
View File
@@ -82,9 +82,21 @@ div.sphinxsidebar input {
}
div.sphinxsidebar #searchbox input[type="text"] {
width: 170px;
float: left;
width: 80%;
padding: 0.25em;
box-sizing: border-box;
}
div.sphinxsidebar #searchbox input[type="submit"] {
float: left;
width: 20%;
border-left: none;
padding: 0.25em;
box-sizing: border-box;
}
img {
border: 0;
max-width: 100%;
@@ -199,6 +211,11 @@ table.modindextable td {
/* -- general body styles --------------------------------------------------- */
div.body {
min-width: 450px;
max-width: 800px;
}
div.body p, div.body dd, div.body li, div.body blockquote {
-moz-hyphens: auto;
-ms-hyphens: auto;
+9
View File
@@ -0,0 +1,9 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT: '',
VERSION: '3.3.0',
LANGUAGE: 'None',
COLLAPSE_INDEX: false,
FILE_SUFFIX: '.html',
HAS_SOURCE: true,
SOURCELINK_SUFFIX: '.txt'
};
+1 -1
View File
@@ -301,7 +301,7 @@
li.hide();
// Determine where in the parents children list to insert this comment.
for(i=0; i < siblings.length; i++) {
for(var i=0; i < siblings.length; i++) {
if (comp(comment, siblings[i]) <= 0) {
$('#cd' + siblings[i].id)
.parent()
+35 -18
View File
@@ -9,7 +9,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Index &mdash; parsedmarc 3.2.0 documentation</title>
<title>Index &mdash; parsedmarc 3.3.0 documentation</title>
@@ -36,7 +36,7 @@
<link rel="index" title="Index"
href="#"/>
<link rel="search" title="Search" href="search.html"/>
<link rel="top" title="parsedmarc 3.2.0 documentation" href="index.html"/>
<link rel="top" title="parsedmarc 3.3.0 documentation" href="index.html"/>
<script src="_static/js/modernizr.min.js"></script>
@@ -65,7 +65,7 @@
<div class="version">
3.2.0
3.3.0
</div>
@@ -154,16 +154,33 @@
<h1 id="index">Index</h1>
<div class="genindex-jumpbox">
<a href="#E"><strong>E</strong></a>
<a href="#A"><strong>A</strong></a>
| <a href="#C"><strong>C</strong></a>
| <a href="#E"><strong>E</strong></a>
| <a href="#G"><strong>G</strong></a>
| <a href="#H"><strong>H</strong></a>
| <a href="#I"><strong>I</strong></a>
| <a href="#P"><strong>P</strong></a>
| <a href="#S"><strong>S</strong></a>
| <a href="#T"><strong>T</strong></a>
| <a href="#W"><strong>W</strong></a>
</div>
<h2 id="A">A</h2>
<table style="width: 100%" class="indextable genindextable"><tr>
<td style="width: 33%; vertical-align: top;"><ul>
<li><a href="index.html#parsedmarc.elastic.AlreadySaved">AlreadySaved</a>
</li>
</ul></td>
</tr></table>
<h2 id="C">C</h2>
<table style="width: 100%" class="indextable genindextable"><tr>
<td style="width: 33%; vertical-align: top;"><ul>
<li><a href="index.html#parsedmarc.elastic.create_indexes">create_indexes() (in module parsedmarc.elastic)</a>
</li>
</ul></td>
</tr></table>
<h2 id="E">E</h2>
<table style="width: 100%" class="indextable genindextable"><tr>
<td style="width: 33%; vertical-align: top;"><ul>
@@ -223,15 +240,17 @@
</li>
<li><a href="index.html#parsedmarc.parse_report_email">parse_report_email() (in module parsedmarc)</a>
</li>
</ul></td>
<td style="width: 33%; vertical-align: top;"><ul>
<li><a href="index.html#parsedmarc.parse_report_file">parse_report_file() (in module parsedmarc)</a>
</li>
</ul></td>
<td style="width: 33%; vertical-align: top;"><ul>
<li><a href="index.html#parsedmarc.parsed_aggregate_reports_to_csv">parsed_aggregate_reports_to_csv() (in module parsedmarc)</a>
</li>
<li><a href="index.html#parsedmarc.parsed_forensic_reports_to_csv">parsed_forensic_reports_to_csv() (in module parsedmarc)</a>
</li>
<li><a href="index.html#module-parsedmarc">parsedmarc (module)</a>
</li>
<li><a href="index.html#module-parsedmarc.elastic">parsedmarc.elastic (module)</a>
</li>
<li><a href="index.html#parsedmarc.ParserError">ParserError</a>
</li>
@@ -240,20 +259,18 @@
<h2 id="S">S</h2>
<table style="width: 100%" class="indextable genindextable"><tr>
<td style="width: 33%; vertical-align: top;"><ul>
<li><a href="index.html#parsedmarc.elastic.save_aggregate_report_to_elasticsearch">save_aggregate_report_to_elasticsearch() (in module parsedmarc.elastic)</a>
</li>
<li><a href="index.html#parsedmarc.elastic.save_forensic_report_to_elasticsearch">save_forensic_report_to_elasticsearch() (in module parsedmarc.elastic)</a>
</li>
</ul></td>
<td style="width: 33%; vertical-align: top;"><ul>
<li><a href="index.html#parsedmarc.save_output">save_output() (in module parsedmarc)</a>
</li>
</ul></td>
<td style="width: 33%; vertical-align: top;"><ul>
<li><a href="index.html#parsedmarc.SMTPError">SMTPError</a>
<li><a href="index.html#parsedmarc.elastic.set_hosts">set_hosts() (in module parsedmarc.elastic)</a>
</li>
</ul></td>
</tr></table>
<h2 id="T">T</h2>
<table style="width: 100%" class="indextable genindextable"><tr>
<td style="width: 33%; vertical-align: top;"><ul>
<li><a href="index.html#parsedmarc.timestamp_to_human">timestamp_to_human() (in module parsedmarc)</a>
<li><a href="index.html#parsedmarc.SMTPError">SMTPError</a>
</li>
</ul></td>
</tr></table>
@@ -302,7 +319,7 @@
<script type="text/javascript">
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'./',
VERSION:'3.2.0',
VERSION:'3.3.0',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
+88 -42
View File
@@ -8,7 +8,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Welcome to parsedmarc’s documentation! &mdash; parsedmarc 3.2.0 documentation</title>
<title>Welcome to parsedmarc’s documentation! &mdash; parsedmarc 3.3.0 documentation</title>
@@ -35,7 +35,7 @@
<link rel="index" title="Index"
href="genindex.html"/>
<link rel="search" title="Search" href="search.html"/>
<link rel="top" title="parsedmarc 3.2.0 documentation" href="#"/>
<link rel="top" title="parsedmarc 3.3.0 documentation" href="#"/>
<script src="_static/js/modernizr.min.js"></script>
@@ -64,7 +64,7 @@
<div class="version">
3.2.0
3.3.0
</div>
@@ -105,7 +105,10 @@
<li><a class="reference internal" href="#optional-dependencies">Optional dependencies</a></li>
</ul>
</li>
<li><a class="reference internal" href="#module-parsedmarc">API</a></li>
<li><a class="reference internal" href="#module-parsedmarc">API</a><ul>
<li><a class="reference internal" href="#module-parsedmarc.elastic">parsedmarc.elastic</a></li>
</ul>
</li>
<li><a class="reference internal" href="#indices-and-tables">Indices and tables</a></li>
</ul>
</li>
@@ -176,7 +179,7 @@
<h1>Welcome to parsedmarc’s documentation!<a class="headerlink" href="#welcome-to-parsedmarc-s-documentation" title="Permalink to this headline"></a></h1>
<p><a class="reference external" href="https://travis-ci.org/domainaware/parsedmarc"><img alt="Build Status" src="https://travis-ci.org/domainaware/parsedmarc.svg?branch=master" /></a></p>
<a class="reference external image-reference" href="_static/screenshots/dmarc-summary-charts.png"><img alt="A screenshot of DMARC summary charts in Kibana" class="align-center" src="_images/dmarc-summary-charts.png" style="width: 643.0px; height: 369.5px;" /></a>
<p><code class="docutils literal"><span class="pre">pasedmarc</span></code> is a Python module and CLI utility for parsing DMARC reports.</p>
<p><code class="docutils literal notranslate"><span class="pre">pasedmarc</span></code> is a Python module and CLI utility for parsing DMARC reports.</p>
<div class="section" id="features">
<h2>Features<a class="headerlink" href="#features" title="Permalink to this headline"></a></h2>
<ul class="simple">
@@ -192,7 +195,7 @@
</div>
<div class="section" id="cli-help">
<h2>CLI help<a class="headerlink" href="#cli-help" title="Permalink to this headline"></a></h2>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="n">usage</span><span class="p">:</span> <span class="n">parsedmarc</span> <span class="p">[</span><span class="o">-</span><span class="n">h</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">o</span> <span class="n">OUTPUT</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">n</span> <span class="n">NAMESERVERS</span> <span class="p">[</span><span class="n">NAMESERVERS</span> <span class="o">...</span><span class="p">]]</span>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">usage</span><span class="p">:</span> <span class="n">parsedmarc</span> <span class="p">[</span><span class="o">-</span><span class="n">h</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">o</span> <span class="n">OUTPUT</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">n</span> <span class="n">NAMESERVERS</span> <span class="p">[</span><span class="n">NAMESERVERS</span> <span class="o">...</span><span class="p">]]</span>
<span class="p">[</span><span class="o">-</span><span class="n">t</span> <span class="n">TIMEOUT</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">H</span> <span class="n">HOST</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">u</span> <span class="n">USER</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">p</span> <span class="n">PASSWORD</span><span class="p">]</span>
<span class="p">[</span><span class="o">-</span><span class="n">r</span> <span class="n">REPORTS_FOLDER</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">a</span> <span class="n">ARCHIVE_FOLDER</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">d</span><span class="p">]</span>
<span class="p">[</span><span class="o">-</span><span class="n">E</span> <span class="p">[</span><span class="n">ELASTICSEARCH_HOST</span> <span class="p">[</span><span class="n">ELASTICSEARCH_HOST</span> <span class="o">...</span><span class="p">]]]</span>
@@ -268,11 +271,11 @@ report from the dmarc.org wiki. It’s actually an older draft of the the 1.0
report schema standardized in
<a class="reference external" href="https://tools.ietf.org/html/rfc7489#appendix-C">RFC 7480 Appendix C</a>.
This draft schema is still in wide use.</p>
<p><code class="docutils literal"><span class="pre">parsedmarc</span></code> produces consistent, normalized output, regardless of the report
<p><code class="docutils literal notranslate"><span class="pre">parsedmarc</span></code> produces consistent, normalized output, regardless of the report
schema.</p>
<div class="section" id="json">
<h3>JSON<a class="headerlink" href="#json" title="Permalink to this headline"></a></h3>
<div class="highlight-json"><div class="highlight"><pre><span></span><span class="p">{</span>
<div class="highlight-json notranslate"><div class="highlight"><pre><span></span><span class="p">{</span>
<span class="nt">&quot;xml_schema&quot;</span><span class="p">:</span> <span class="s2">&quot;draft&quot;</span><span class="p">,</span>
<span class="nt">&quot;report_metadata&quot;</span><span class="p">:</span> <span class="p">{</span>
<span class="nt">&quot;org_name&quot;</span><span class="p">:</span> <span class="s2">&quot;acme.com&quot;</span><span class="p">,</span>
@@ -336,7 +339,7 @@ schema.</p>
</div>
<div class="section" id="csv">
<h3>CSV<a class="headerlink" href="#csv" title="Permalink to this headline"></a></h3>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="n">xml_schema</span><span class="p">,</span><span class="n">org_name</span><span class="p">,</span><span class="n">org_email</span><span class="p">,</span><span class="n">org_extra_contact_info</span><span class="p">,</span><span class="n">report_id</span><span class="p">,</span><span class="n">begin_date</span><span class="p">,</span><span class="n">end_date</span><span class="p">,</span><span class="n">errors</span><span class="p">,</span><span class="n">domain</span><span class="p">,</span><span class="n">adkim</span><span class="p">,</span><span class="n">aspf</span><span class="p">,</span><span class="n">p</span><span class="p">,</span><span class="n">sp</span><span class="p">,</span><span class="n">pct</span><span class="p">,</span><span class="n">fo</span><span class="p">,</span><span class="n">source_ip_address</span><span class="p">,</span><span class="n">source_country</span><span class="p">,</span><span class="n">source_reverse_dns</span><span class="p">,</span><span class="n">source_base_domain</span><span class="p">,</span><span class="n">count</span><span class="p">,</span><span class="n">disposition</span><span class="p">,</span><span class="n">dkim_alignment</span><span class="p">,</span><span class="n">spf_alignment</span><span class="p">,</span><span class="n">policy_override_reasons</span><span class="p">,</span><span class="n">policy_override_comments</span><span class="p">,</span><span class="n">envelope_from</span><span class="p">,</span><span class="n">header_from</span><span class="p">,</span><span class="n">envelope_to</span><span class="p">,</span><span class="n">dkim_domains</span><span class="p">,</span><span class="n">dkim_selectors</span><span class="p">,</span><span class="n">dkim_results</span><span class="p">,</span><span class="n">spf_domains</span><span class="p">,</span><span class="n">spf_scopes</span><span class="p">,</span><span class="n">spf_results</span>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">xml_schema</span><span class="p">,</span><span class="n">org_name</span><span class="p">,</span><span class="n">org_email</span><span class="p">,</span><span class="n">org_extra_contact_info</span><span class="p">,</span><span class="n">report_id</span><span class="p">,</span><span class="n">begin_date</span><span class="p">,</span><span class="n">end_date</span><span class="p">,</span><span class="n">errors</span><span class="p">,</span><span class="n">domain</span><span class="p">,</span><span class="n">adkim</span><span class="p">,</span><span class="n">aspf</span><span class="p">,</span><span class="n">p</span><span class="p">,</span><span class="n">sp</span><span class="p">,</span><span class="n">pct</span><span class="p">,</span><span class="n">fo</span><span class="p">,</span><span class="n">source_ip_address</span><span class="p">,</span><span class="n">source_country</span><span class="p">,</span><span class="n">source_reverse_dns</span><span class="p">,</span><span class="n">source_base_domain</span><span class="p">,</span><span class="n">count</span><span class="p">,</span><span class="n">disposition</span><span class="p">,</span><span class="n">dkim_alignment</span><span class="p">,</span><span class="n">spf_alignment</span><span class="p">,</span><span class="n">policy_override_reasons</span><span class="p">,</span><span class="n">policy_override_comments</span><span class="p">,</span><span class="n">envelope_from</span><span class="p">,</span><span class="n">header_from</span><span class="p">,</span><span class="n">envelope_to</span><span class="p">,</span><span class="n">dkim_domains</span><span class="p">,</span><span class="n">dkim_selectors</span><span class="p">,</span><span class="n">dkim_results</span><span class="p">,</span><span class="n">spf_domains</span><span class="p">,</span><span class="n">spf_scopes</span><span class="p">,</span><span class="n">spf_results</span>
<span class="n">draft</span><span class="p">,</span><span class="n">acme</span><span class="o">.</span><span class="n">com</span><span class="p">,</span><span class="n">noreply</span><span class="o">-</span><span class="n">dmarc</span><span class="o">-</span><span class="n">support</span><span class="nd">@acme</span><span class="o">.</span><span class="n">com</span><span class="p">,</span><span class="n">http</span><span class="p">:</span><span class="o">//</span><span class="n">acme</span><span class="o">.</span><span class="n">com</span><span class="o">/</span><span class="n">dmarc</span><span class="o">/</span><span class="n">support</span><span class="p">,</span><span class="mi">9391651994964116463</span><span class="p">,</span><span class="mi">2012</span><span class="o">-</span><span class="mi">04</span><span class="o">-</span><span class="mi">27</span> <span class="mi">20</span><span class="p">:</span><span class="mi">00</span><span class="p">:</span><span class="mi">00</span><span class="p">,</span><span class="mi">2012</span><span class="o">-</span><span class="mi">04</span><span class="o">-</span><span class="mi">28</span> <span class="mi">19</span><span class="p">:</span><span class="mi">59</span><span class="p">:</span><span class="mi">59</span><span class="p">,,</span><span class="n">example</span><span class="o">.</span><span class="n">com</span><span class="p">,</span><span class="n">r</span><span class="p">,</span><span class="n">r</span><span class="p">,</span><span class="n">none</span><span class="p">,</span><span class="n">none</span><span class="p">,</span><span class="mi">100</span><span class="p">,</span><span class="mi">0</span><span class="p">,</span><span class="mf">72.150</span><span class="o">.</span><span class="mf">241.94</span><span class="p">,</span><span class="n">US</span><span class="p">,</span><span class="n">adsl</span><span class="o">-</span><span class="mi">72</span><span class="o">-</span><span class="mi">150</span><span class="o">-</span><span class="mi">241</span><span class="o">-</span><span class="mf">94.</span><span class="n">shv</span><span class="o">.</span><span class="n">bellsouth</span><span class="o">.</span><span class="n">net</span><span class="p">,</span><span class="n">bellsouth</span><span class="o">.</span><span class="n">net</span><span class="p">,</span><span class="mi">2</span><span class="p">,</span><span class="n">none</span><span class="p">,</span><span class="n">fail</span><span class="p">,</span><span class="k">pass</span><span class="p">,,,</span><span class="n">example</span><span class="o">.</span><span class="n">com</span><span class="p">,</span><span class="n">example</span><span class="o">.</span><span class="n">com</span><span class="p">,,</span><span class="n">example</span><span class="o">.</span><span class="n">com</span><span class="p">,</span><span class="n">none</span><span class="p">,</span><span class="n">fail</span><span class="p">,</span><span class="n">example</span><span class="o">.</span><span class="n">com</span><span class="p">,</span><span class="n">mfrom</span><span class="p">,</span><span class="k">pass</span>
</pre></div>
</div>
@@ -354,34 +357,34 @@ forensic report that you can share publicly, please contact me!</p>
</div>
<div class="section" id="installation">
<h2>Installation<a class="headerlink" href="#installation" title="Permalink to this headline"></a></h2>
<p><code class="docutils literal"><span class="pre">parsedmarc</span></code> works with Python 3 only.</p>
<p><code class="docutils literal notranslate"><span class="pre">parsedmarc</span></code> works with Python 3 only.</p>
<p>On Debian or Ubuntu systems, run:</p>
<div class="highlight-bash"><div class="highlight"><pre><span></span>$ sudo apt-get install python3-pip
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>$ sudo apt-get install python3-pip
</pre></div>
</div>
<p>Python 3 installers for Windows and macOS can be found at
<a class="reference external" href="https://www.python.org/downloads/">https://www.python.org/downloads/</a></p>
<p>To install or upgrade to the latest stable release of <code class="docutils literal"><span class="pre">parsedmarc</span></code> on
<p>To install or upgrade to the latest stable release of <code class="docutils literal notranslate"><span class="pre">parsedmarc</span></code> on
macOS or Linux, run</p>
<div class="highlight-bash"><div class="highlight"><pre><span></span>$ sudo -H pip3 install -U parsedmarc
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>$ sudo -H pip3 install -U parsedmarc
</pre></div>
</div>
<p>Or, install the latest development release directly from GitHub:</p>
<div class="highlight-bash"><div class="highlight"><pre><span></span>$ sudo -H pip3 install -U git+https://github.com/domainaware/parsedmarc.git
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>$ sudo -H pip3 install -U git+https://github.com/domainaware/parsedmarc.git
</pre></div>
</div>
<div class="admonition note">
<p class="first admonition-title">Note</p>
<p class="last">On Windows, <code class="docutils literal"><span class="pre">pip3</span></code> is <code class="docutils literal"><span class="pre">pip</span></code>, even with Python 3. So on Windows, simply
substitute <code class="docutils literal"><span class="pre">pip</span></code> as an administrator in place of <code class="docutils literal"><span class="pre">sudo</span> <span class="pre">pip3</span></code>, in the
<p class="last">On Windows, <code class="docutils literal notranslate"><span class="pre">pip3</span></code> is <code class="docutils literal notranslate"><span class="pre">pip</span></code>, even with Python 3. So on Windows, simply
substitute <code class="docutils literal notranslate"><span class="pre">pip</span></code> as an administrator in place of <code class="docutils literal notranslate"><span class="pre">sudo</span> <span class="pre">pip3</span></code>, in the
above commands.</p>
</div>
<div class="section" id="optional-dependencies">
<h3>Optional dependencies<a class="headerlink" href="#optional-dependencies" title="Permalink to this headline"></a></h3>
<p>If you would like to be able to parse emails saved from Microsoft Outlook
(i.e. OLE .msg files), install <code class="docutils literal"><span class="pre">msgconvert</span></code>:</p>
(i.e. OLE .msg files), install <code class="docutils literal notranslate"><span class="pre">msgconvert</span></code>:</p>
<p>On Debian or Ubuntu systems, run:</p>
<div class="highlight-bash"><div class="highlight"><pre><span></span>$ sudo apt-get install libemail-outlook-message-perl
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>$ sudo apt-get install libemail-outlook-message-perl
</pre></div>
</div>
</div>
@@ -494,7 +497,7 @@ or bytes.</p>
</ul>
</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body"><p class="first">Lists of <code class="docutils literal"><span class="pre">aggregate_reports</span></code> and <code class="docutils literal"><span class="pre">forensic_reports</span></code></p>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body"><p class="first">Lists of <code class="docutils literal notranslate"><span class="pre">aggregate_reports</span></code> and <code class="docutils literal notranslate"><span class="pre">forensic_reports</span></code></p>
</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body"><p class="first last">OrderedDict</p>
@@ -525,7 +528,7 @@ or bytes.</p>
<dl class="function">
<dt id="parsedmarc.human_timestamp_to_datetime">
<code class="descclassname">parsedmarc.</code><code class="descname">human_timestamp_to_datetime</code><span class="sig-paren">(</span><em>human_timestamp</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc.html#human_timestamp_to_datetime"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#parsedmarc.human_timestamp_to_datetime" title="Permalink to this definition"></a></dt>
<dd><p>Converts a human-readable timestamp into a Python <code class="docutils literal"><span class="pre">DateTime</span></code> object</p>
<dd><p>Converts a human-readable timestamp into a Python <code class="docutils literal notranslate"><span class="pre">DateTime</span></code> object</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
@@ -596,7 +599,7 @@ aggregate DMARC report</p>
<dl class="function">
<dt id="parsedmarc.parse_forensic_report">
<code class="descclassname">parsedmarc.</code><code class="descname">parse_forensic_report</code><span class="sig-paren">(</span><em>feedback_report</em>, <em>sample</em>, <em>sample_headers_only</em>, <em>nameservers=None</em>, <em>timeout=6.0</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc.html#parse_forensic_report"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#parsedmarc.parse_forensic_report" title="Permalink to this definition"></a></dt>
<dd><p>Converts a DMARC forensic report and sample to a <code class="docutils literal"><span class="pre">OrderedDict</span></code></p>
<dd><p>Converts a DMARC forensic report and sample to a <code class="docutils literal notranslate"><span class="pre">OrderedDict</span></code></p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
@@ -637,8 +640,8 @@ aggregate DMARC report</p>
</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body"><p class="first"><ul class="simple">
<li><code class="docutils literal"><span class="pre">report_type</span></code>: <code class="docutils literal"><span class="pre">aggregate</span></code> or <code class="docutils literal"><span class="pre">forensic</span></code></li>
<li><code class="docutils literal"><span class="pre">report</span></code>: The parsed report</li>
<li><code class="docutils literal notranslate"><span class="pre">report_type</span></code>: <code class="docutils literal notranslate"><span class="pre">aggregate</span></code> or <code class="docutils literal notranslate"><span class="pre">forensic</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">report</span></code>: The parsed report</li>
</ul>
</p>
</td>
@@ -733,24 +736,6 @@ headers</p>
</table>
</dd></dl>
<dl class="function">
<dt id="parsedmarc.timestamp_to_human">
<code class="descclassname">parsedmarc.</code><code class="descname">timestamp_to_human</code><span class="sig-paren">(</span><em>timestamp</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc.html#timestamp_to_human"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#parsedmarc.timestamp_to_human" title="Permalink to this definition"></a></dt>
<dd><p>Converts a UNIX/DMARC timestamp to a human-readable string</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>timestamp</strong> – The timestamp</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body">The converted timestamp in <code class="docutils literal"><span class="pre">YYYY-MM-DD</span> <span class="pre">HH:MM:SS</span></code> format</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body">str</td>
</tr>
</tbody>
</table>
</dd></dl>
<dl class="function">
<dt id="parsedmarc.watch_inbox">
<code class="descclassname">parsedmarc.</code><code class="descname">watch_inbox</code><span class="sig-paren">(</span><em>host</em>, <em>username</em>, <em>password</em>, <em>callback</em>, <em>reports_folder='INBOX'</em>, <em>archive_folder='Archive'</em>, <em>delete=False</em>, <em>test=False</em>, <em>wait=30</em>, <em>nameservers=None</em>, <em>dns_timeout=6.0</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc.html#watch_inbox"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#parsedmarc.watch_inbox" title="Permalink to this definition"></a></dt>
@@ -780,9 +765,70 @@ to a callback function</p>
</table>
</dd></dl>
<div class="section" id="module-parsedmarc.elastic">
<span id="parsedmarc-elastic"></span><h3>parsedmarc.elastic<a class="headerlink" href="#module-parsedmarc.elastic" title="Permalink to this headline"></a></h3>
<dl class="exception">
<dt id="parsedmarc.elastic.AlreadySaved">
<em class="property">exception </em><code class="descclassname">parsedmarc.elastic.</code><code class="descname">AlreadySaved</code><a class="reference internal" href="_modules/parsedmarc/elastic.html#AlreadySaved"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#parsedmarc.elastic.AlreadySaved" title="Permalink to this definition"></a></dt>
<dd><p>Raised when a report to be saved matches an existing report</p>
</dd></dl>
<dl class="function">
<dt id="parsedmarc.elastic.create_indexes">
<code class="descclassname">parsedmarc.elastic.</code><code class="descname">create_indexes</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc/elastic.html#create_indexes"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#parsedmarc.elastic.create_indexes" title="Permalink to this definition"></a></dt>
<dd><p>Creates the required indexes</p>
</dd></dl>
<dl class="function">
<dt id="parsedmarc.elastic.save_aggregate_report_to_elasticsearch">
<code class="descclassname">parsedmarc.elastic.</code><code class="descname">save_aggregate_report_to_elasticsearch</code><span class="sig-paren">(</span><em>aggregate_report</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc/elastic.html#save_aggregate_report_to_elasticsearch"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#parsedmarc.elastic.save_aggregate_report_to_elasticsearch" title="Permalink to this definition"></a></dt>
<dd><p>Saves a parsed DMARC aggregate report to ElasticSearch</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>aggregate_report</strong> (<em>OrderedDict</em>) – A parsed forensic report</td>
</tr>
<tr class="field-even field"><th class="field-name">Raises:</th><td class="field-body"><a class="reference internal" href="#parsedmarc.elastic.AlreadySaved" title="parsedmarc.elastic.AlreadySaved"><code class="xref py py-exc docutils literal notranslate"><span class="pre">AlreadySaved</span></code></a></td>
</tr>
</tbody>
</table>
</dd></dl>
<dl class="function">
<dt id="parsedmarc.elastic.save_forensic_report_to_elasticsearch">
<code class="descclassname">parsedmarc.elastic.</code><code class="descname">save_forensic_report_to_elasticsearch</code><span class="sig-paren">(</span><em>forensic_report</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc/elastic.html#save_forensic_report_to_elasticsearch"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#parsedmarc.elastic.save_forensic_report_to_elasticsearch" title="Permalink to this definition"></a></dt>
<dd><p>Saves a parsed DMARC forensic report to ElasticSearch</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>forensic_report</strong> (<em>OrderedDict</em>) – A parsed forensic report</td>
</tr>
<tr class="field-even field"><th class="field-name">Raises:</th><td class="field-body"><a class="reference internal" href="#parsedmarc.elastic.AlreadySaved" title="parsedmarc.elastic.AlreadySaved"><code class="xref py py-exc docutils literal notranslate"><span class="pre">AlreadySaved</span></code></a></td>
</tr>
</tbody>
</table>
</dd></dl>
<dl class="function">
<dt id="parsedmarc.elastic.set_hosts">
<code class="descclassname">parsedmarc.elastic.</code><code class="descname">set_hosts</code><span class="sig-paren">(</span><em>hosts</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc/elastic.html#set_hosts"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#parsedmarc.elastic.set_hosts" title="Permalink to this definition"></a></dt>
<dd><p>Sets the Elasticsearch hosts to use</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>hosts</strong> – A single hostname or URL, or list of hostnames or URLs</td>
</tr>
</tbody>
</table>
</dd></dl>
<div class="toctree-wrapper compound">
</div>
</div>
</div>
<div class="section" id="indices-and-tables">
<h2>Indices and tables<a class="headerlink" href="#indices-and-tables" title="Permalink to this headline"></a></h2>
<ul class="simple">
@@ -828,7 +874,7 @@ to a callback function</p>
<script type="text/javascript">
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'./',
VERSION:'3.2.0',
VERSION:'3.3.0',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
+2 -3
View File
@@ -1,6 +1,5 @@
# Sphinx inventory version 2
# Project: parsedmarc
# Version: 3.2.0
# Version: 3.3.0
# The remainder of this file is compressed using zlib.
xÚ¥”KNÃ0†÷9…lƒè¶»Š‡ÔE¤ªEbi¹Î$µä—ìIIYq␍®ÇIˆã†>@".»ÑøÿþñŒG¶Ìy(sœØÝT™²‘@îˆÐ%´·Tò:æò’gö[|;/f‹GçŒ ´,
£Éä˜<Gô–IQÎêÚAÍ–`�Ãdþ¡˜-ï/dŸŒíOÅ!t‰ý®ŠçÔ�bBR¾‘èV5šÿMµèGÚ*9š©iuÕÂ,<­œQTèµi“L"Nß„�mÅ4E¡À#S–¢¡e·!1Ú£)vi¸F%ä¿=RÆ-ªýZíé}Ùþí/CÓ».´íÃ3p¿Mõ9ë=ÙÆ³-PÓ mÆÏíduúm�¾2ä›1k^ƒî3Äc9•l␍’ä2d{™‡0;ÈJÃèHñ’›n­Ñ�Ã%>ß?<é´��,”¿Êº_ö—rv—ѯˆt,ì¡3Ûœ1Å«x¾`5d_¿#$�
xÚ­”KNÃ0†÷=…lSÑmw©‹HU‹ÄÒríibÉŽ-{RRV\ƒëqì$¥¥°ˆ ;kòÿx±e΃ÐÌqb÷smD£€ÜY h§juÝDz’Mì—xº(òåƒsÆE Z¥©Éì”<GêSRäeé d+°Æa2_ä«» ÙGã ö’§âËxt‰õ®‹§Ô�beâ(hš+Lì×l"9å”�vBð‘ß65�û�•²Ãh©ëLÑÐAà!¨«Ël·ÃÔþÏ�VÆcB™šIòûF¥P-:Æ‘¶Z�fÊp·î4TëéÖæ²1m’ÉЬWiGcU£YMQjðÈ´��aš10Ú£;þÜ„­TöHicoq¶9‰ô�¶›ýehzÕâGÙ>Ž�û]ªÏYíÉ6ݯg´Íø¾½0äÕ˜]-¡î"Ä£˜+¶E²9D;YÄãä(†Gщâ7a7Ñ�ã%>ÞÞ= ÚFC�,¦¿š„‡ó—tvŸ>ô~Eÿæö‰‡·åÑÇzñºÿ¾d%L>%;¤ë
+11 -9
View File
@@ -8,7 +8,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Python Module Index &mdash; parsedmarc 3.2.0 documentation</title>
<title>Python Module Index &mdash; parsedmarc 3.3.0 documentation</title>
@@ -35,13 +35,9 @@
<link rel="index" title="Index"
href="genindex.html"/>
<link rel="search" title="Search" href="search.html"/>
<link rel="top" title="parsedmarc 3.2.0 documentation" href="index.html"/>
<link rel="top" title="parsedmarc 3.3.0 documentation" href="index.html"/>
<script type="text/javascript">
DOCUMENTATION_OPTIONS.COLLAPSE_INDEX = true;
</script>
@@ -71,7 +67,7 @@
<div class="version">
3.2.0
3.3.0
</div>
@@ -168,10 +164,16 @@
<tr class="cap" id="cap-p"><td></td><td>
<strong>p</strong></td><td></td></tr>
<tr>
<td></td>
<td><img src="_static/minus.png" class="toggler"
id="toggle-1" style="display: none" alt="-" /></td>
<td>
<a href="index.html#module-parsedmarc"><code class="xref">parsedmarc</code></a></td><td>
<em></em></td></tr>
<tr class="cg-1">
<td></td>
<td>&#160;&#160;&#160;
<a href="index.html#module-parsedmarc.elastic"><code class="xref">parsedmarc.elastic</code></a></td><td>
<em></em></td></tr>
</table>
@@ -209,7 +211,7 @@
<script type="text/javascript">
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'./',
VERSION:'3.2.0',
VERSION:'3.3.0',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
+4 -4
View File
@@ -8,7 +8,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Search &mdash; parsedmarc 3.2.0 documentation</title>
<title>Search &mdash; parsedmarc 3.3.0 documentation</title>
@@ -35,7 +35,7 @@
<link rel="index" title="Index"
href="genindex.html"/>
<link rel="search" title="Search" href="#"/>
<link rel="top" title="parsedmarc 3.2.0 documentation" href="index.html"/>
<link rel="top" title="parsedmarc 3.3.0 documentation" href="index.html"/>
<script src="_static/js/modernizr.min.js"></script>
@@ -64,7 +64,7 @@
<div class="version">
3.2.0
3.3.0
</div>
@@ -195,7 +195,7 @@
<script type="text/javascript">
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'./',
VERSION:'3.2.0',
VERSION:'3.3.0',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
+1 -1
View File
@@ -1 +1 @@
Search.setIndex({docnames:["index"],envversion:53,filenames:["index.rst"],objects:{"":{parsedmarc:[0,0,0,"-"]},parsedmarc:{IMAPError:[0,1,1,""],InvalidAggregateReport:[0,1,1,""],InvalidDMARCReport:[0,1,1,""],InvalidForensicReport:[0,1,1,""],ParserError:[0,1,1,""],SMTPError:[0,1,1,""],email_results:[0,2,1,""],extract_xml:[0,2,1,""],get_dmarc_reports_from_inbox:[0,2,1,""],get_report_zip:[0,2,1,""],human_timestamp_to_datetime:[0,2,1,""],parse_aggregate_report_file:[0,2,1,""],parse_aggregate_report_xml:[0,2,1,""],parse_forensic_report:[0,2,1,""],parse_report_email:[0,2,1,""],parse_report_file:[0,2,1,""],parsed_aggregate_reports_to_csv:[0,2,1,""],parsed_forensic_reports_to_csv:[0,2,1,""],save_output:[0,2,1,""],timestamp_to_human:[0,2,1,""],watch_inbox:[0,2,1,""]}},objnames:{"0":["py","module","Python module"],"1":["py","exception","Python exception"],"2":["py","function","Python function"]},objtypes:{"0":"py:module","1":"py:exception","2":"py:function"},terms:{"byte":0,"default":0,"float":0,"function":0,"int":0,"null":0,"return":0,"true":0,DNS:0,OLE:0,The:0,Use:0,_input:0,abl:0,abov:0,acm:0,actual:0,address:0,adkim:0,administr:0,adsl:0,after:0,aggreg:[],aggregate_report:0,ani:[],answer:0,appendix:0,apt:0,archiv:0,archive_fold:0,argument:0,arriv:0,aspf:0,attach:0,attachment_filenam:0,auth_result:0,base_domain:0,begin_d:0,bellsouth:0,bodi:0,bool:0,callback:0,can:0,check:0,checkdmarc:0,com:0,command:0,compress:0,connect:0,consist:0,contact:0,contain:0,content:[],context:0,convert:0,count:0,countri:0,creat:0,dashboard:0,data:0,datetim:0,debian:0,delet:0,develop:0,directli:0,directori:0,disposit:0,dkim:0,dkim_align:0,dkim_domain:0,dkim_result:0,dkim_selector:0,dmarc:[],dns_timeout:0,domain:0,domainawar:0,don:0,download:0,draft:0,elasticsearch:0,elasticsearch_host:0,email:0,email_result:0,encount:0,end_dat:0,envelope_from:0,envelope_to:0,error:0,even:0,exampl:0,except:0,exit:0,extract:0,extract_xml:0,fail:0,fals:0,feedback:0,feedback_report:0,fetch:0,few:[],file:0,file_path:0,filenam:0,flat:0,folder:0,forensic_report:0,format:0,found:0,from:0,get:0,get_dmarc_reports_from_inbox:0,get_report_zip:0,git:0,github:0,given:0,gzip:0,handl:0,have:0,header:0,header_from:0,here:0,host:0,hostnam:0,http:0,human:0,human_timestamp:0,human_timestamp_to_datetim:0,identifi:0,idl:0,imap:0,imaperror:0,inbox:0,includ:0,incom:0,index:0,input:0,input_:0,invalid:0,invalidaggregatereport:0,invaliddmarcreport:0,invalidforensicreport:0,ip_address:0,issu:0,kibana:0,latest:0,libemail:0,like:0,linux:0,list:0,localhost:0,look:0,maco:0,mail:0,mail_from:0,mail_to:0,messag:0,mfrom:0,microsoft:0,modul:0,more:0,move:0,msg:0,msgconvert:0,nameserv:0,net:0,none:0,norepli:0,normal:0,number:0,object:0,occur:0,often:[],older:0,one:0,onli:0,option:[],ordereddict:0,org:0,org_email:0,org_extra_contact_info:0,org_nam:0,out:0,outgo:0,outgoing_attach:0,outgoing_from:0,outgoing_host:0,outgoing_messag:0,outgoing_password:0,outgoing_subject:0,outgoing_to:0,outgoing_us:0,outlook:0,output_directori:0,over:0,overrid:0,page:0,paramet:0,pars:0,parse_aggregate_report_fil:0,parse_aggregate_report_xml:0,parse_forensic_report:0,parse_report_email:0,parse_report_fil:0,parsed_aggregate_report_to_csv:[],parsed_aggregate_reports_to_csv:0,parsed_dmarc_forensic_reports_to_csv:[],parsed_forensic_reports_to_csv:0,parser:0,parsererror:0,pasedmarc:0,pass:0,password:0,patch:0,path:0,pct:0,perl:0,pip3:0,pip:0,place:0,plain:0,pleas:0,policy_evalu:0,policy_override_com:0,policy_override_reason:0,policy_publish:0,port:0,posit:0,prefer:[],premad:0,print:[],privaci:0,process:0,produc:0,program:0,project:0,provid:[],publicli:0,python3:0,python:0,queri:0,rais:0,rather:[],readabl:0,reason:0,receiv:0,recipi:[],record:[],regardless:0,releas:0,report_id:0,report_metadata:0,report_typ:0,reports_fold:0,requir:0,respons:0,result:0,reverse_dn:0,rfc:0,run:0,sample_headers_onli:0,save:0,save_output:0,schema:0,scope:0,screen:[],search:0,second:0,selector:0,send:0,server:0,set:0,share:0,show:0,shv:0,simpl:0,simpli:0,sister:0,smtp:0,smtperror:0,some:0,sourc:0,source_base_domain:0,source_countri:0,source_ip_address:0,source_reverse_dn:0,specifi:0,spf:[],spf_align:0,spf_domain:0,spf_result:0,spf_scope:0,ssl:0,ssl_context:0,stabl:0,standard:0,start:0,starttl:0,still:0,str:0,string:0,structur:0,subject:0,substitut:0,sudo:0,support:0,system:0,test:0,text:0,than:[],thei:0,them:0,thi:0,those:[],timeout:0,timestamp:0,timestamp_to_human:0,tracker:0,transpar:0,type:0,ubuntu:0,uncompress:[],unix:0,upgrad:0,url:0,usag:0,use:0,use_ssl:0,user:0,usernam:0,using:0,util:0,valu:0,veri:[],version:0,wait:0,watch_inbox:0,when:0,whenev:0,where:0,who:[],wide:0,wiki:0,window:0,work:0,would:0,write:0,www:0,xml:0,xml_schema:0,you:0,yyyi:0,zip:0},titles:["Welcome to parsedmarc\u2019s documentation!"],titleterms:{about:[],aggreg:0,api:0,bug:0,cli:0,csv:0,depend:0,dmarc:0,document:0,featur:0,forens:0,help:0,indic:0,instal:0,json:0,option:0,output:0,parsedmarc:0,record:0,report:0,sampl:0,spf:0,tabl:0,valid:0,welcom:0,what:[]}})
Search.setIndex({docnames:["index"],envversion:53,filenames:["index.rst"],objects:{"":{parsedmarc:[0,0,0,"-"]},"parsedmarc.elastic":{AlreadySaved:[0,1,1,""],create_indexes:[0,2,1,""],save_aggregate_report_to_elasticsearch:[0,2,1,""],save_forensic_report_to_elasticsearch:[0,2,1,""],set_hosts:[0,2,1,""]},parsedmarc:{IMAPError:[0,1,1,""],InvalidAggregateReport:[0,1,1,""],InvalidDMARCReport:[0,1,1,""],InvalidForensicReport:[0,1,1,""],ParserError:[0,1,1,""],SMTPError:[0,1,1,""],elastic:[0,0,0,"-"],email_results:[0,2,1,""],extract_xml:[0,2,1,""],get_dmarc_reports_from_inbox:[0,2,1,""],get_report_zip:[0,2,1,""],human_timestamp_to_datetime:[0,2,1,""],parse_aggregate_report_file:[0,2,1,""],parse_aggregate_report_xml:[0,2,1,""],parse_forensic_report:[0,2,1,""],parse_report_email:[0,2,1,""],parse_report_file:[0,2,1,""],parsed_aggregate_reports_to_csv:[0,2,1,""],parsed_forensic_reports_to_csv:[0,2,1,""],save_output:[0,2,1,""],watch_inbox:[0,2,1,""]}},objnames:{"0":["py","module","Python module"],"1":["py","exception","Python exception"],"2":["py","function","Python function"]},objtypes:{"0":"py:module","1":"py:exception","2":"py:function"},terms:{"byte":0,"class":[],"default":0,"float":0,"function":0,"int":0,"new":[],"null":0,"return":0,"true":0,DNS:0,OLE:0,The:0,Use:0,_input:0,abl:0,abov:0,acm:0,actual:0,addit:[],address:0,adkim:0,administr:0,adsl:0,after:0,aggreg:[],aggregate_report:0,aggregatereportdoc:[],alia:[],alreadysav:0,ani:[],answer:0,appendix:0,apt:0,archiv:0,archive_fold:0,argument:0,arriv:0,aspf:0,associ:[],attach:0,attachment_filenam:0,auth_result:0,base_domain:0,begin_d:0,being:[],bellsouth:0,bodi:0,bool:0,callback:0,can:0,check:0,checkdmarc:0,com:0,command:0,compress:0,connect:0,consist:0,contact:0,contain:0,content:[],context:0,convert:0,count:0,countri:0,creat:0,create_index:0,dashboard:0,data:0,datetim:0,debian:0,delet:0,develop:0,directli:0,directori:0,disposit:0,dkim:0,dkim_align:0,dkim_domain:0,dkim_result:0,dkim_selector:0,dkimresult:[],dmarc:[],dns_timeout:0,doctyp:[],doesn:[],domain:0,domainawar:0,don:0,download:0,draft:0,elasticsearch:0,elasticsearch_dsl:[],elasticsearch_host:0,email:0,email_result:0,emailaddressdoc:[],emailattachmentdoc:[],encount:0,end_dat:0,envelope_from:0,envelope_to:0,error:0,even:0,exampl:0,except:0,exist:0,exit:0,extract:0,extract_xml:0,fail:0,fals:0,feedback:0,feedback_report:0,fetch:0,few:[],field:[],file:0,file_path:0,filenam:0,flat:0,folder:0,forensic_report:0,forensicreportdoc:[],forensicsampledoc:[],format:0,found:0,from:0,get:0,get_dmarc_reports_from_inbox:0,get_report_zip:0,git:0,github:0,given:0,gzip:0,handl:0,have:0,header:0,header_from:0,here:0,host:0,hostnam:0,http:0,human:0,human_timestamp:0,human_timestamp_to_datetim:0,identifi:0,idl:0,imap:0,imaperror:0,inbox:0,includ:0,incom:0,index:0,input:0,input_:0,integ:[],invalid:0,invalidaggregatereport:0,invaliddmarcreport:0,invalidforensicreport:0,ip_address:0,issu:0,keyword:[],kibana:0,kwarg:[],latest:0,libemail:0,like:0,linux:0,list:0,localhost:0,look:0,maco:0,mail:0,mail_from:0,mail_to:0,match:0,messag:0,message_count:[],meta:[],mfrom:0,microsoft:0,modul:0,more:0,move:0,msg:0,msgconvert:0,nameserv:0,net:0,none:0,norepli:0,normal:0,number:0,object:0,occur:0,often:[],older:0,omit:[],one:0,onli:0,oper:[],option:[],ordereddict:0,org:0,org_email:0,org_extra_contact_info:0,org_nam:0,otherwis:[],out:0,outgo:0,outgoing_attach:0,outgoing_from:0,outgoing_host:0,outgoing_messag:0,outgoing_password:0,outgoing_subject:0,outgoing_to:0,outgoing_us:0,outlook:0,output_directori:0,over:0,overrid:0,overwritten:[],page:0,paramet:0,pars:0,parse_aggregate_report_fil:0,parse_aggregate_report_xml:0,parse_forensic_report:0,parse_report_email:0,parse_report_fil:0,parsed_aggregate_report_to_csv:[],parsed_aggregate_reports_to_csv:0,parsed_dmarc_forensic_reports_to_csv:[],parsed_forensic_reports_to_csv:0,parser:0,parsererror:0,pasedmarc:0,pass:0,password:0,patch:0,path:0,pct:0,perl:0,pip3:0,pip:0,place:0,plain:0,pleas:0,policy_evalu:0,policy_override_com:0,policy_override_reason:0,policy_publish:0,policyoverrid:[],port:0,posit:0,prefer:[],premad:0,print:[],privaci:0,process:0,produc:0,program:0,project:0,provid:[],publicli:0,publishedpolici:[],python3:0,python:0,queri:0,rais:0,rather:[],readabl:0,reason:0,receiv:0,recipi:[],record:[],regardless:0,releas:0,report_id:0,report_metadata:0,report_typ:0,reports_fold:0,requir:0,respons:0,result:0,reverse_dn:0,rfc:0,run:0,sample_headers_onli:0,save:0,save_aggregate_report_to_elasticsearch:0,save_forensic_report_to_elasticsearch:0,save_output:0,schema:0,scope:0,screen:[],search:0,second:0,selector:0,send:0,server:0,set:0,set_host:0,share:0,show:0,shv:0,simpl:0,simpli:0,singl:0,sister:0,skip:[],smtp:0,smtperror:0,some:0,sourc:0,source_base_domain:0,source_countri:0,source_ip_address:0,source_reverse_dn:0,specifi:0,spf:[],spf_align:0,spf_domain:0,spf_result:0,spf_scope:0,spfresult:[],ssl:0,ssl_context:0,stabl:0,standard:0,start:0,starttl:0,still:0,str:0,string:0,structur:0,subject:0,substitut:0,sudLine truncated