amorfo77/parsedmarc
1
0
Fork 0
mirror of https://github.com/domainaware/parsedmarc.git synced 2026-08-31 11:17:15 +00:00
Code Issues Projects Releases Wiki Activity

8.2.0

Browse Source
This commit is contained in:
Sean Whalen
2022-05-10 21:07:04 -04:00
parent a2cae44500
commit 38f5cc983e
9 changed files with 40 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
_modules/index.html
+2 -2
View File
@@ -3,7 +3,7 @@
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Overview: module code &mdash; parsedmarc 8.1.1 documentation</title>
<title>Overview: module code &mdash; parsedmarc 8.2.0 documentation</title>
<link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
<link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
<!--[if lt IE 9]>
@@ -27,7 +27,7 @@
<a href="../index.html" class="icon icon-home"> parsedmarc
</a>
<div class="version">
8.1.1
8.2.0
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
_modules/parsedmarc.html
+23 -6
View File
@@ -3,7 +3,7 @@
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>parsedmarc &mdash; parsedmarc 8.1.1 documentation</title>
<title>parsedmarc &mdash; parsedmarc 8.2.0 documentation</title>
<link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
<link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
<!--[if lt IE 9]>
@@ -27,7 +27,7 @@
<a href="../index.html" class="icon icon-home"> parsedmarc
</a>
<div class="version">
8.1.1
8.2.0
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
@@ -100,7 +100,7 @@
<span class="kn">from</span> <span class="nn">parsedmarc.utils</span> <span class="kn">import</span> <span class="n">parse_email</span>
<span class="kn">from</span> <span class="nn">parsedmarc.utils</span> <span class="kn">import</span> <span class="n">timestamp_to_human</span><span class="p">,</span> <span class="n">human_timestamp_to_datetime</span>
<span class="n">__version__</span> <span class="o">=</span> <span class="s2">&quot;8.1.1&quot;</span>
<span class="n">__version__</span> <span class="o">=</span> <span class="s2">&quot;8.2.0&quot;</span>
<span class="n">formatter</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">Formatter</span><span class="p">(</span>
<span class="n">fmt</span><span class="o">=</span><span class="s1">&#39;</span><span class="si">%(levelname)8s</span><span class="s1">:</span><span class="si">%(filename)s</span><span class="s1">:</span><span class="si">%(lineno)d</span><span class="s1">:</span><span class="si">%(message)s</span><span class="s1">&#39;</span><span class="p">,</span>
@@ -115,6 +115,7 @@
<span class="n">feedback_report_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">&quot;^([\w\-]+): (.+)$&quot;</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">MULTILINE</span><span class="p">)</span>
<span class="n">xml_header_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">&quot;^&lt;\?xml .*?&gt;&quot;</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">MULTILINE</span><span class="p">)</span>
<span class="n">xml_schema_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">&quot;&lt;/??xs:schema.*&gt;&quot;</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">MULTILINE</span><span class="p">)</span>
<span class="n">text_report_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">&quot;\s*([a-zA-Z\s]+):\s(.+)&quot;</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">MULTILINE</span><span class="p">)</span>
<span class="n">MAGIC_ZIP</span> <span class="o">=</span> <span class="sa">b</span><span class="s2">&quot;</span><span class="se">\x50\x4B\x03\x04</span><span class="s2">&quot;</span>
<span class="n">MAGIC_GZIP</span> <span class="o">=</span> <span class="sa">b</span><span class="s2">&quot;</span><span class="se">\x1F\x8B</span><span class="s2">&quot;</span>
@@ -674,7 +675,7 @@
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">&quot;arrival_date&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">msg_date</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span>
<span class="k">if</span> <span class="s2">&quot;version&quot;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_report</span><span class="p">:</span>
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">&quot;version&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="mi">1</span>
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">&quot;version&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">if</span> <span class="s2">&quot;user_agent&quot;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_report</span><span class="p">:</span>
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">&quot;user_agent&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
@@ -895,6 +896,21 @@
<span class="n">sample</span> <span class="o">=</span> <span class="n">payload</span>
<span class="k">elif</span> <span class="n">content_type</span> <span class="o">==</span> <span class="s2">&quot;message/rfc822&quot;</span><span class="p">:</span>
<span class="n">sample</span> <span class="o">=</span> <span class="n">payload</span>
<span class="k">elif</span> <span class="n">content_type</span> <span class="o">==</span> <span class="s2">&quot;text/plain&quot;</span><span class="p">:</span>
<span class="k">if</span> <span class="s2">&quot;A message claiming to be from you has failed&quot;</span> <span class="ow">in</span> <span class="n">payload</span><span class="p">:</span>
<span class="n">parts</span> <span class="o">=</span> <span class="n">payload</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">&quot;detected.&quot;</span><span class="p">)</span>
<span class="n">field_matches</span> <span class="o">=</span> <span class="n">text_report_regex</span><span class="o">.</span><span class="n">findall</span><span class="p">(</span><span class="n">parts</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span>
<span class="n">fields</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
<span class="k">for</span> <span class="n">match</span> <span class="ow">in</span> <span class="n">field_matches</span><span class="p">:</span>
<span class="n">field_name</span> <span class="o">=</span> <span class="n">match</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot; &quot;</span><span class="p">,</span> <span class="s2">&quot;-&quot;</span><span class="p">)</span>
<span class="n">fields</span><span class="p">[</span><span class="n">field_name</span><span class="p">]</span> <span class="o">=</span> <span class="n">match</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
<span class="n">feedback_report</span> <span class="o">=</span> <span class="s2">&quot;Arrival-Date: </span><span class="si">{}</span><span class="se">\n</span><span class="s2">&quot;</span> \
<span class="s2">&quot;Source-IP: </span><span class="si">{}</span><span class="s2">&quot;</span> \
<span class="s2">&quot;&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">fields</span><span class="p">[</span><span class="s2">&quot;received-date&quot;</span><span class="p">],</span>
<span class="n">fields</span><span class="p">[</span><span class="s2">&quot;sender-ip-address&quot;</span><span class="p">])</span>
<span class="n">sample</span> <span class="o">=</span> <span class="n">parts</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">lstrip</span><span class="p">()</span>
<span class="n">sample</span> <span class="o">=</span> <span class="n">sample</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;=</span><span class="se">\r\n</span><span class="s2">&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">)</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="n">sample</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">payload</span> <span class="o">=</span> <span class="n">b64decode</span><span class="p">(</span><span class="n">payload</span><span class="p">)</span>
@@ -923,7 +939,7 @@
<span class="s1">&#39;aggregate DMARC report: </span><span class="si">{1}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">subject</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
<span class="k">raise</span> <span class="n">InvalidAggregateReport</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">FileNotFoundError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="n">error</span> <span class="o">=</span> <span class="s1">&#39;Unable to parse message with &#39;</span> \
<span class="s1">&#39;subject &quot;</span><span class="si">{0}</span><span class="s1">&quot;: </span><span class="si">{1}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">subject</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
<span class="k">raise</span> <span class="n">InvalidDMARCReport</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
@@ -1084,7 +1100,7 @@
<span class="n">dns_timeout</span><span class="o">=</span><span class="mf">6.0</span><span class="p">,</span>
<span class="n">strip_attachment_payloads</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">results</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">batch_size</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">batch_size</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span>
<span class="n">create_folders</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Fetches and parses DMARC reports from a mailbox</span>
@@ -1103,6 +1119,7 @@
<span class="sd"> forensic report results</span>
<span class="sd"> results (dict): Results from the previous run</span>
<span class="sd"> batch_size (int): Number of messages to read and process before saving</span>
<span class="sd"> (use 0 for no limit)</span>
<span class="sd"> create_folders (bool): Whether to create the destination folders</span>
<span class="sd"> (not used in watch)</span>