amorfo77/parsedmarc
1
0
Fork 0
mirror of https://github.com/domainaware/parsedmarc.git synced 2026-07-14 20:44:55 +00:00
Code Issues Projects Releases Wiki Activity

3.8.0

Browse Source
This commit is contained in:
Sean Whalen
2018-08-22 06:53:50 -04:00
parent 37d87f6e20
commit bed2f89031
11 changed files with 79 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
_modules/index.html
+3 -3
View File
@@ -8,7 +8,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Overview: module code &mdash; parsedmarc 3.7.3 documentation</title>
<title>Overview: module code &mdash; parsedmarc 3.8.0 documentation</title>
@@ -56,7 +56,7 @@
<div class="version">
3.7.3
3.8.0
</div>
@@ -177,7 +177,7 @@
<script type="text/javascript">
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../',
VERSION:'3.7.3',
VERSION:'3.8.0',
LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
_modules/parsedmarc.html
+32 -9
View File
@@ -8,7 +8,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>parsedmarc &mdash; parsedmarc 3.7.3 documentation</title>
<title>parsedmarc &mdash; parsedmarc 3.8.0 documentation</title>
@@ -56,7 +56,7 @@
<div class="version">
3.7.3
3.8.0
</div>
@@ -188,7 +188,7 @@
<span class="kn">import</span> <span class="nn">dateparser</span>
<span class="kn">import</span> <span class="nn">mailparser</span>
<span class="n">__version__</span> <span class="o">=</span> <span class="s2">&quot;3.7.3&quot;</span>
<span class="n">__version__</span> <span class="o">=</span> <span class="s2">&quot;3.8.0&quot;</span>
<span class="n">logger</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="vm">__name__</span><span class="p">)</span>
<span class="n">logger</span><span class="o">.</span><span class="n">setLevel</span><span class="p">(</span><span class="n">logging</span><span class="o">.</span><span class="n">INFO</span><span class="p">)</span>
@@ -937,8 +937,12 @@
<span class="k">else</span><span class="p">:</span>
<span class="n">parsed_sample</span><span class="p">[</span><span class="s2">&quot;reply_to&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">parsed_sample</span><span class="p">[</span><span class="s2">&quot;to&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">convert_address</span><span class="p">(</span><span class="n">x</span><span class="p">),</span>
<span class="n">parsed_sample</span><span class="p">[</span><span class="s2">&quot;to&quot;</span><span class="p">]))</span>
<span class="k">if</span> <span class="s2">&quot;to&quot;</span> <span class="ow">in</span> <span class="n">parsed_sample</span><span class="p">:</span>
<span class="n">parsed_sample</span><span class="p">[</span><span class="s2">&quot;to&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">convert_address</span><span class="p">(</span><span class="n">x</span><span class="p">),</span>
<span class="n">parsed_sample</span><span class="p">[</span><span class="s2">&quot;to&quot;</span><span class="p">]))</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">parsed_sample</span><span class="p">[</span><span class="s2">&quot;to&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">if</span> <span class="s2">&quot;cc&quot;</span> <span class="ow">in</span> <span class="n">parsed_sample</span><span class="p">:</span>
<span class="n">parsed_sample</span><span class="p">[</span><span class="s2">&quot;cc&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">convert_address</span><span class="p">(</span><span class="n">x</span><span class="p">),</span>
<span class="n">parsed_sample</span><span class="p">[</span><span class="s2">&quot;cc&quot;</span><span class="p">]))</span>
@@ -1120,7 +1124,14 @@
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">payload</span><span class="p">)</span> <span class="o">==</span> <span class="nb">list</span><span class="p">:</span>
<span class="n">payload</span> <span class="o">=</span> <span class="n">payload</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span>
<span class="k">if</span> <span class="n">content_type</span> <span class="o">==</span> <span class="s2">&quot;message/feedback-report&quot;</span><span class="p">:</span>
<span class="n">feedback_report</span> <span class="o">=</span> <span class="n">payload</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">feedback_report</span> <span class="o">=</span> <span class="n">b64decode</span><span class="p">(</span><span class="n">payload</span><span class="p">)</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span>
<span class="n">feedback_report</span> <span class="o">=</span> <span class="n">feedback_report</span><span class="o">.</span><span class="n">lstrip</span><span class="p">(</span><span class="s2">&quot;b&#39;&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s2">&quot;&#39;&quot;</span><span class="p">)</span>
<span class="n">feedback_report</span> <span class="o">=</span> <span class="n">feedback_report</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;</span><span class="se">\\</span><span class="s2">r&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">)</span>
<span class="n">feedback_report</span> <span class="o">=</span> <span class="n">feedback_report</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;</span><span class="se">\\</span><span class="s2">n&quot;</span><span class="p">,</span> <span class="s2">&quot;</span><span class="se">\n</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="k">except</span> <span class="p">(</span><span class="ne">ValueError</span><span class="p">,</span> <span class="ne">TypeError</span><span class="p">,</span> <span class="n">binascii</span><span class="o">.</span><span class="n">Error</span><span class="p">):</span>
<span class="n">feedback_report</span> <span class="o">=</span> <span class="n">payload</span>
<span class="k">elif</span> <span class="n">content_type</span> <span class="o">==</span> <span class="s2">&quot;text/rfc822-headers&quot;</span><span class="p">:</span>
<span class="n">sample</span> <span class="o">=</span> <span class="n">payload</span>
<span class="n">sample_headers_only</span> <span class="o">=</span> <span class="kc">True</span>
@@ -1148,7 +1159,7 @@
<span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">)</span>
<span class="n">result</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">([(</span><span class="s2">&quot;report_type&quot;</span><span class="p">,</span> <span class="s2">&quot;aggregate&quot;</span><span class="p">),</span>
<span class="p">(</span><span class="s2">&quot;report&quot;</span><span class="p">,</span> <span class="n">aggregate_report</span><span class="p">)])</span>
<span class="k">except</span> <span class="p">(</span><span class="ne">TypeError</span><span class="p">,</span> <span class="n">binascii</span><span class="o">.</span><span class="n">Error</span><span class="p">):</span>
<span class="k">except</span> <span class="p">(</span><span class="ne">TypeError</span><span class="p">,</span> <span class="ne">ValueError</span><span class="p">,</span> <span class="n">binascii</span><span class="o">.</span><span class="n">Error</span><span class="p">):</span>
<span class="k">pass</span>
<span class="k">if</span> <span class="n">result</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
@@ -1238,13 +1249,25 @@
<span class="k">try</span><span class="p">:</span>
<span class="n">server</span> <span class="o">=</span> <span class="n">imapclient</span><span class="o">.</span><span class="n">IMAPClient</span><span class="p">(</span><span class="n">host</span><span class="p">,</span> <span class="n">use_uid</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="n">server</span><span class="o">.</span><span class="n">login</span><span class="p">(</span><span class="n">user</span><span class="p">,</span> <span class="n">password</span><span class="p">)</span>
<span class="n">server</span><span class="o">.</span><span class="n">select_folder</span><span class="p">(</span><span class="n">reports_folder</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">server</span><span class="o">.</span><span class="n">folder_exists</span><span class="p">(</span><span class="n">archive_folder</span><span class="p">):</span>
<span class="n">server</span><span class="o">.</span><span class="n">create_folder</span><span class="p">(</span><span class="n">archive_folder</span><span class="p">)</span>
<span class="k">try</span><span class="p">:</span>
<span class="c1"># Test subfolder creation</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">server</span><span class="o">.</span><span class="n">folder_exists</span><span class="p">(</span><span class="n">aggregate_reports_folder</span><span class="p">):</span>
<span class="n">server</span><span class="o">.</span><span class="n">create_folder</span><span class="p">(</span><span class="n">aggregate_reports_folder</span><span class="p">)</span>
<span class="k">except</span> <span class="n">imapclient</span><span class="o">.</span><span class="n">exceptions</span><span class="o">.</span><span class="n">IMAPClientError</span><span class="p">:</span>
<span class="c1"># Only replace / with . when . doesn&#39;t work</span>
<span class="c1"># This usually indicates a dovecot IMAP server</span>
<span class="n">aggregate_reports_folder</span> <span class="o">=</span> <span class="n">aggregate_reports_folder</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;/&quot;</span><span class="p">,</span>
<span class="s2">&quot;.&quot;</span><span class="p">)</span>
<span class="n">forensic_reports_folder</span> <span class="o">=</span> <span class="n">forensic_reports_folder</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;/&quot;</span><span class="p">,</span>
<span class="s2">&quot;.&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">server</span><span class="o">.</span><span class="n">folder_exists</span><span class="p">(</span><span class="n">aggregate_reports_folder</span><span class="p">):</span>
<span class="n">server</span><span class="o">.</span><span class="n">create_folder</span><span class="p">(</span><span class="n">aggregate_reports_folder</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">server</span><span class="o">.</span><span class="n">folder_exists</span><span class="p">(</span><span class="n">forensic_reports_folder</span><span class="p">):</span>
<span class="n">server</span><span class="o">.</span><span class="n">create_folder</span><span class="p">(</span><span class="n">forensic_reports_folder</span><span class="p">)</span>
<span class="n">server</span><span class="o">.</span><span class="n">select_folder</span><span class="p">(</span><span class="n">reports_folder</span><span class="p">)</span>
<span class="n">messages</span> <span class="o">=</span> <span class="n">server</span><span class="o">.</span><span class="n">search</span><span class="p">()</span>
<span class="k">for</span> <span class="n">message_uid</span> <span class="ow">in</span> <span class="n">messages</span><span class="p">:</span>
<span class="n">raw_msg</span> <span class="o">=</span> <span class="n">server</span><span class="o">.</span><span class="n">fetch</span><span class="p">(</span><span class="n">message_uid</span><span class="p">,</span>
@@ -1627,7 +1650,7 @@
<script type="text/javascript">
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../',
VERSION:'3.7.3',
VERSION:'3.8.0',
LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
_modules/parsedmarc/elastic.html
+8 -7
View File
@@ -8,7 +8,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>parsedmarc.elastic &mdash; parsedmarc 3.7.3 documentation</title>
<title>parsedmarc.elastic &mdash; parsedmarc 3.8.0 documentation</title>
@@ -56,7 +56,7 @@
<div class="version">
3.7.3
3.8.0
</div>
@@ -443,12 +443,13 @@
<span class="n">arrival_date</span> <span class="o">=</span> <span class="n">parsedmarc</span><span class="o">.</span><span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">arrival_date_human</span><span class="p">)</span>
<span class="n">search</span> <span class="o">=</span> <span class="n">forensic_index</span><span class="o">.</span><span class="n">search</span><span class="p">()</span>
<span class="n">to_query</span> <span class="o">=</span> <span class="p">{</span><span class="s2">&quot;match&quot;</span><span class="p">:</span> <span class="p">{</span><span class="s2">&quot;sample.headers.to&quot;</span><span class="p">:</span> <span class="n">headers</span><span class="p">[</span><span class="s2">&quot;to&quot;</span><span class="p">]}}</span>
<span class="n">from_query</span> <span class="o">=</span> <span class="p">{</span><span class="s2">&quot;match&quot;</span><span class="p">:</span> <span class="p">{</span><span class="s2">&quot;sample.headers.from&quot;</span><span class="p">:</span> <span class="n">headers</span><span class="p">[</span><span class="s2">&quot;from&quot;</span><span class="p">]}}</span>
<span class="n">subject_query</span> <span class="o">=</span> <span class="p">{</span><span class="s2">&quot;match&quot;</span><span class="p">:</span> <span class="p">{</span><span class="s2">&quot;sample.headers.subject&quot;</span><span class="p">:</span> <span class="n">headers</span><span class="p">[</span><span class="s2">&quot;subject&quot;</span><span class="p">]}}</span>
<span class="n">arrival_date_query</span> <span class="o">=</span> <span class="p">{</span><span class="s2">&quot;match&quot;</span><span class="p">:</span> <span class="p">{</span><span class="s2">&quot;sample.headers.arrival_date&quot;</span><span class="p">:</span> <span class="n">arrival_date</span>
<span class="p">}}</span>
<span class="n">q</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="n">to_query</span><span class="p">)</span> <span class="o">&amp;</span> <span class="n">Q</span><span class="p">(</span><span class="n">from_query</span><span class="p">)</span> <span class="o">&amp;</span> <span class="n">Q</span><span class="p">(</span><span class="n">subject_query</span><span class="p">)</span> <span class="o">&amp;</span> <span class="n">Q</span><span class="p">(</span><span class="n">arrival_date_query</span><span class="p">)</span>
<span class="n">arrival_query</span> <span class="o">=</span> <span class="p">{</span><span class="s2">&quot;match&quot;</span><span class="p">:</span> <span class="p">{</span><span class="s2">&quot;sample.headers.arrival_date&quot;</span><span class="p">:</span> <span class="n">arrival_date</span><span class="p">}}</span>
<span class="n">q</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="n">from_query</span><span class="p">)</span> <span class="o">&amp;</span> <span class="n">Q</span><span class="p">(</span><span class="n">subject_query</span><span class="p">)</span> <span class="o">&amp;</span> <span class="n">Q</span><span class="p">(</span><span class="n">arrival_query</span><span class="p">)</span>
<span class="k">if</span> <span class="s2">&quot;to&quot;</span> <span class="ow">in</span> <span class="n">headers</span><span class="p">:</span>
<span class="n">to_query</span> <span class="o">=</span> <span class="p">{</span><span class="s2">&quot;match&quot;</span><span class="p">:</span> <span class="p">{</span><span class="s2">&quot;sample.headers.to&quot;</span><span class="p">:</span> <span class="n">headers</span><span class="p">[</span><span class="s2">&quot;to&quot;</span><span class="p">]}}</span>
<span class="n">q</span> <span class="o">&amp;</span> <span class="n">Q</span><span class="p">(</span><span class="n">to_query</span><span class="p">)</span>
<span class="n">search</span><span class="o">.</span><span class="n">query</span> <span class="o">=</span> <span class="n">q</span>
<span class="n">existing</span> <span class="o">=</span> <span class="n">search</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
@@ -545,7 +546,7 @@
<script type="text/javascript">
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../',
VERSION:'3.7.3',
VERSION:'3.8.0',
LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',