This commit is contained in:
Sean Whalen
2018-09-06 12:49:15 -04:00
parent 74a8a2aec2
commit fea3ca8b82
10 changed files with 98 additions and 43 deletions
+3 -3
View File
@@ -8,7 +8,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Overview: module code &mdash; parsedmarc 3.9.0 documentation</title>
<title>Overview: module code &mdash; parsedmarc 3.9.1 documentation</title>
@@ -56,7 +56,7 @@
<div class="version">
3.9.0
3.9.1
</div>
@@ -177,7 +177,7 @@
<script type="text/javascript">
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../',
VERSION:'3.9.0',
VERSION:'3.9.1',
LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
+61 -23
View File
@@ -8,7 +8,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>parsedmarc &mdash; parsedmarc 3.9.0 documentation</title>
<title>parsedmarc &mdash; parsedmarc 3.9.1 documentation</title>
@@ -56,7 +56,7 @@
<div class="version">
3.9.0
3.9.1
</div>
@@ -188,7 +188,7 @@
<span class="kn">import</span> <span class="nn">dateparser</span>
<span class="kn">import</span> <span class="nn">mailparser</span>
<span class="n">__version__</span> <span class="o">=</span> <span class="s2">&quot;3.9.0&quot;</span>
<span class="n">__version__</span> <span class="o">=</span> <span class="s2">&quot;3.9.1&quot;</span>
<span class="n">logger</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="vm">__name__</span><span class="p">)</span>
<span class="n">logger</span><span class="o">.</span><span class="n">setLevel</span><span class="p">(</span><span class="n">logging</span><span class="o">.</span><span class="n">INFO</span><span class="p">)</span>
@@ -1228,6 +1228,23 @@
<span class="k">return</span> <span class="n">results</span></div>
<div class="viewcode-block" id="get_imap_capabilities"><a class="viewcode-back" href="../index.html#parsedmarc.get_imap_capabilities">[docs]</a><span class="k">def</span> <span class="nf">get_imap_capabilities</span><span class="p">(</span><span class="n">server</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Returns a list of an IMAP server&#39;s capabilities</span>
<span class="sd"> Args:</span>
<span class="sd"> server (imapclient.IMAPClient): An instance of imapclient.IMAPClient</span>
<span class="sd"> Returns (list): A list of capabilities</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">capabilities</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="nb">str</span><span class="p">,</span> <span class="nb">list</span><span class="p">(</span><span class="n">server</span><span class="o">.</span><span class="n">capabilities</span><span class="p">())))</span>
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">capabilities</span><span class="p">)):</span>
<span class="n">capabilities</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">capabilities</span><span class="p">[</span><span class="n">i</span><span class="p">])</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;b&#39;&quot;</span><span class="p">,</span>
<span class="s2">&quot;&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;&#39;&quot;</span><span class="p">,</span>
<span class="s2">&quot;&quot;</span><span class="p">)</span>
<span class="k">return</span> <span class="n">capabilities</span></div>
<div class="viewcode-block" id="get_dmarc_reports_from_inbox"><a class="viewcode-back" href="../index.html#parsedmarc.get_dmarc_reports_from_inbox">[docs]</a><span class="k">def</span> <span class="nf">get_dmarc_reports_from_inbox</span><span class="p">(</span><span class="n">host</span><span class="p">,</span> <span class="n">user</span><span class="p">,</span> <span class="n">password</span><span class="p">,</span>
<span class="n">reports_folder</span><span class="o">=</span><span class="s2">&quot;INBOX&quot;</span><span class="p">,</span>
<span class="n">archive_folder</span><span class="o">=</span><span class="s2">&quot;Archive&quot;</span><span class="p">,</span>
@@ -1271,6 +1288,28 @@
<span class="k">try</span><span class="p">:</span>
<span class="n">server</span> <span class="o">=</span> <span class="n">imapclient</span><span class="o">.</span><span class="n">IMAPClient</span><span class="p">(</span><span class="n">host</span><span class="p">,</span> <span class="n">use_uid</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="n">server</span><span class="o">.</span><span class="n">login</span><span class="p">(</span><span class="n">user</span><span class="p">,</span> <span class="n">password</span><span class="p">)</span>
<span class="n">server_capabilities</span> <span class="o">=</span> <span class="n">get_imap_capabilities</span><span class="p">(</span><span class="n">server</span><span class="p">)</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;IMAP server supports: </span><span class="si">{0}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">server_capabilities</span><span class="p">))</span>
<span class="n">move_supported</span> <span class="o">=</span> <span class="s2">&quot;MOVE&quot;</span> <span class="ow">in</span> <span class="n">server_capabilities</span>
<span class="k">def</span> <span class="nf">delete_messages</span><span class="p">(</span><span class="n">msg_uids</span><span class="p">):</span>
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">msg_uids</span><span class="p">)</span> <span class="o">==</span> <span class="nb">str</span><span class="p">:</span>
<span class="n">msg_uids</span> <span class="o">=</span> <span class="p">[</span><span class="n">msg_uids</span><span class="p">]</span>
<span class="n">server</span><span class="o">.</span><span class="n">add_flags</span><span class="p">(</span><span class="n">msg_uids</span><span class="p">,</span> <span class="p">[</span><span class="n">imapclient</span><span class="o">.</span><span class="n">DELETED</span><span class="p">])</span>
<span class="n">server</span><span class="o">.</span><span class="n">expunge</span><span class="p">()</span>
<span class="k">def</span> <span class="nf">move_messages</span><span class="p">(</span><span class="n">msg_uids</span><span class="p">,</span> <span class="n">folder</span><span class="p">):</span>
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">msg_uids</span><span class="p">)</span> <span class="o">==</span> <span class="nb">str</span><span class="p">:</span>
<span class="n">msg_uids</span> <span class="o">=</span> <span class="p">[</span><span class="n">msg_uids</span><span class="p">]</span>
<span class="k">for</span> <span class="n">chunk</span> <span class="ow">in</span> <span class="n">chunks</span><span class="p">(</span><span class="n">msg_uids</span><span class="p">,</span> <span class="mi">100</span><span class="p">):</span>
<span class="k">if</span> <span class="n">move_supported</span><span class="p">:</span>
<span class="n">server</span><span class="o">.</span><span class="n">move</span><span class="p">(</span><span class="n">chunk</span><span class="p">,</span> <span class="n">folder</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">server</span><span class="o">.</span><span class="n">copy</span><span class="p">(</span><span class="n">msg_uids</span><span class="p">,</span> <span class="n">folder</span><span class="p">)</span>
<span class="n">delete_messages</span><span class="p">(</span><span class="n">msg_uids</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">server</span><span class="o">.</span><span class="n">folder_exists</span><span class="p">(</span><span class="n">archive_folder</span><span class="p">):</span>
<span class="n">server</span><span class="o">.</span><span class="n">create_folder</span><span class="p">(</span><span class="n">archive_folder</span><span class="p">)</span>
<span class="k">try</span><span class="p">:</span>
@@ -1312,26 +1351,21 @@
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">test</span><span class="p">:</span>
<span class="k">if</span> <span class="n">delete</span><span class="p">:</span>
<span class="n">server</span><span class="o">.</span><span class="n">add_flags</span><span class="p">([</span><span class="n">message_uid</span><span class="p">],</span> <span class="p">[</span><span class="n">imapclient</span><span class="o">.</span><span class="n">DELETED</span><span class="p">])</span>
<span class="n">server</span><span class="o">.</span><span class="n">expunge</span><span class="p">()</span>
<span class="n">delete_messages</span><span class="p">([</span><span class="n">message_uid</span><span class="p">])</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">server</span><span class="o">.</span><span class="n">move</span><span class="p">([</span><span class="n">message_uid</span><span class="p">],</span> <span class="n">invalid_reports_folder</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">test</span><span class="p">:</span>
<span class="k">if</span> <span class="n">delete</span><span class="p">:</span>
<span class="n">processed_messages</span> <span class="o">=</span> <span class="n">aggregate_report_msg_uids</span> <span class="o">+</span> \
<span class="n">forensic_report_msg_uids</span>
<span class="n">server</span><span class="o">.</span><span class="n">add_flags</span><span class="p">(</span><span class="n">processed_messages</span><span class="p">,</span> <span class="p">[</span><span class="n">imapclient</span><span class="o">.</span><span class="n">DELETED</span><span class="p">])</span>
<span class="n">server</span><span class="o">.</span><span class="n">expunge</span><span class="p">()</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">aggregate_report_msg_uids</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
<span class="k">for</span> <span class="n">chunk</span> <span class="ow">in</span> <span class="n">chunks</span><span class="p">(</span><span class="n">aggregate_report_msg_uids</span><span class="p">,</span> <span class="mi">100</span><span class="p">):</span>
<span class="n">server</span><span class="o">.</span><span class="n">move</span><span class="p">(</span><span class="n">chunk</span><span class="p">,</span>
<span class="n">aggregate_reports_folder</span><span class="p">)</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">forensic_report_msg_uids</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
<span class="k">for</span> <span class="n">chunk</span> <span class="ow">in</span> <span class="n">chunks</span><span class="p">(</span><span class="n">forensic_report_msg_uids</span><span class="p">,</span> <span class="mi">100</span><span class="p">):</span>
<span class="n">server</span><span class="o">.</span><span class="n">move</span><span class="p">(</span><span class="n">chunk</span><span class="p">,</span>
<span class="n">forensic_reports_folder</span><span class="p">)</span>
<span class="n">move_messages</span><span class="p">([</span><span class="n">message_uid</span><span class="p">],</span> <span class="n">invalid_reports_folder</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">test</span><span class="p">:</span>
<span class="k">if</span> <span class="n">delete</span><span class="p">:</span>
<span class="n">processed_messages</span> <span class="o">=</span> <span class="n">aggregate_report_msg_uids</span> <span class="o">+</span> \
<span class="n">forensic_report_msg_uids</span>
<span class="n">delete_messages</span><span class="p">(</span><span class="n">processed_messages</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">aggregate_report_msg_uids</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
<span class="n">move_messages</span><span class="p">(</span><span class="n">aggregate_report_msg_uids</span><span class="p">,</span>
<span class="n">aggregate_reports_folder</span><span class="p">)</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">forensic_report_msg_uids</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
<span class="n">move_messages</span><span class="p">(</span><span class="n">forensic_report_msg_uids</span><span class="p">,</span>
<span class="n">forensic_reports_folder</span><span class="p">)</span>
<span class="n">results</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">([(</span><span class="s2">&quot;aggregate_reports&quot;</span><span class="p">,</span> <span class="n">aggregate_reports</span><span class="p">),</span>
<span class="p">(</span><span class="s2">&quot;forensic_reports&quot;</span><span class="p">,</span> <span class="n">forensic_reports</span><span class="p">)])</span>
@@ -1572,6 +1606,10 @@
<span class="k">try</span><span class="p">:</span>
<span class="n">server</span><span class="o">.</span><span class="n">login</span><span class="p">(</span><span class="n">username</span><span class="p">,</span> <span class="n">password</span><span class="p">)</span>
<span class="k">if</span> <span class="s2">&quot;IDLE&quot;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">get_imap_capabilities</span><span class="p">(</span><span class="n">server</span><span class="p">):</span>
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;Cannot watch inbox: IMAP server does not support &quot;</span>
<span class="s2">&quot;the IDLE command&quot;</span><span class="p">)</span>
<span class="n">exit</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
<span class="n">server</span><span class="o">.</span><span class="n">select_folder</span><span class="p">(</span><span class="n">rf</span><span class="p">)</span>
<span class="n">idle_start_time</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">monotonic</span><span class="p">()</span>
<span class="n">server</span><span class="o">.</span><span class="n">idle</span><span class="p">()</span>
@@ -1680,7 +1718,7 @@
<script type="text/javascript">
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../',
VERSION:'3.9.0',
VERSION:'3.9.1',
LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
+3 -3
View File
@@ -8,7 +8,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>parsedmarc.elastic &mdash; parsedmarc 3.9.0 documentation</title>
<title>parsedmarc.elastic &mdash; parsedmarc 3.9.1 documentation</title>
@@ -56,7 +56,7 @@
<div class="version">
3.9.0
3.9.1
</div>
@@ -546,7 +546,7 @@
<script type="text/javascript">
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../',
VERSION:'3.9.0',
VERSION:'3.9.1',
LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
+1 -1
View File
@@ -1,6 +1,6 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT: document.getElementById("documentation_options").getAttribute('data-url_root'),
VERSION: '3.9.0',
VERSION: '3.9.1',
LANGUAGE: 'None',
COLLAPSE_INDEX: false,
FILE_SUFFIX: '.html',
+5 -3
View File
@@ -9,7 +9,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Index &mdash; parsedmarc 3.9.0 documentation</title>
<title>Index &mdash; parsedmarc 3.9.1 documentation</title>
@@ -57,7 +57,7 @@
<div class="version">
3.9.0
3.9.1
</div>
@@ -193,6 +193,8 @@
</li>
</ul></td>
<td style="width: 33%; vertical-align: top;"><ul>
<li><a href="index.html#parsedmarc.get_imap_capabilities">get_imap_capabilities() (in module parsedmarc)</a>
</li>
<li><a href="index.html#parsedmarc.get_report_zip">get_report_zip() (in module parsedmarc)</a>
</li>
</ul></td>
@@ -310,7 +312,7 @@
<script type="text/javascript">
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'./',
VERSION:'3.9.0',
VERSION:'3.9.1',
LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
+18 -3
View File
@@ -8,7 +8,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>parsedmarc documentation - Open source DMARC report analyzer and visualizer &mdash; parsedmarc 3.9.0 documentation</title>
<title>parsedmarc documentation - Open source DMARC report analyzer and visualizer &mdash; parsedmarc 3.9.1 documentation</title>
@@ -56,7 +56,7 @@
<div class="version">
3.9.0
3.9.1
</div>
@@ -948,6 +948,21 @@ or bytes.</p>
</table>
</dd></dl>
<dl class="function">
<dt id="parsedmarc.get_imap_capabilities">
<code class="descclassname">parsedmarc.</code><code class="descname">get_imap_capabilities</code><span class="sig-paren">(</span><em>server</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc.html#get_imap_capabilities"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#parsedmarc.get_imap_capabilities" title="Permalink to this definition"></a></dt>
<dd><p>Returns a list of an IMAP servers capabilities</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>server</strong> (<em>imapclient.IMAPClient</em>) An instance of imapclient.IMAPClient</td>
</tr>
</tbody>
</table>
<p>Returns (list): A list of capabilities</p>
</dd></dl>
<dl class="function">
<dt id="parsedmarc.get_report_zip">
<code class="descclassname">parsedmarc.</code><code class="descname">get_report_zip</code><span class="sig-paren">(</span><em>results</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc.html#get_report_zip"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#parsedmarc.get_report_zip" title="Permalink to this definition"></a></dt>
@@ -1313,7 +1328,7 @@ to a callback function</p>
<script type="text/javascript">
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'./',
VERSION:'3.9.0',
VERSION:'3.9.1',
LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
BIN
View File
Binary file not shown.
+3 -3
View File
@@ -8,7 +8,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Python Module Index &mdash; parsedmarc 3.9.0 documentation</title>
<title>Python Module Index &mdash; parsedmarc 3.9.1 documentation</title>
@@ -59,7 +59,7 @@
<div class="version">
3.9.0
3.9.1
</div>
@@ -200,7 +200,7 @@
<script type="text/javascript">
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'./',
VERSION:'3.9.0',
VERSION:'3.9.1',
LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
+3 -3
View File
@@ -8,7 +8,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Search &mdash; parsedmarc 3.9.0 documentation</title>
<title>Search &mdash; parsedmarc 3.9.1 documentation</title>
@@ -56,7 +56,7 @@
<div class="version">
3.9.0
3.9.1
</div>
@@ -188,7 +188,7 @@
<script type="text/javascript">
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'./',
VERSION:'3.9.0',
VERSION:'3.9.1',
LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
+1 -1
View File
File diff suppressed because one or more lines are too long