amorfo77/parsedmarc
1
0
Fork 0
mirror of https://github.com/domainaware/parsedmarc.git synced 2026-05-20 10:55:24 +00:00
Code Issues Projects Releases Wiki Activity

4.3.0

Browse Source
This commit is contained in:
Sean Whalen
2018-10-12 12:37:10 -04:00
parent 49cf73723e
commit 906b9c7d70
13 changed files with 1840 additions and 583 deletions
Download Patch File Download Diff File Expand all files Collapse all files
_modules/index.html
+5 -3
View File
@@ -8,7 +8,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Overview: module code &mdash; parsedmarc 4.2.0k documentation</title>
<title>Overview: module code &mdash; parsedmarc 4.3.0 documentation</title>
@@ -56,7 +56,7 @@
<div class="version">
4.2.0k
4.3.0
</div>
@@ -143,6 +143,8 @@
<h1>All modules for which code is available</h1>
<ul><li><a href="parsedmarc.html">parsedmarc</a></li>
<ul><li><a href="parsedmarc/elastic.html">parsedmarc.elastic</a></li>
<li><a href="parsedmarc/splunk.html">parsedmarc.splunk</a></li>
<li><a href="parsedmarc/utils.html">parsedmarc.utils</a></li>
</ul></ul>
</div>
@@ -179,7 +181,7 @@
<script type="text/javascript">
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../',
VERSION:'4.2.0k',
VERSION:'4.3.0',
LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
_modules/parsedmarc.html
+83 -458
View File
@@ -8,7 +8,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>parsedmarc &mdash; parsedmarc 4.2.0k documentation</title>
<title>parsedmarc &mdash; parsedmarc 4.3.0 documentation</title>
@@ -56,7 +56,7 @@
<div class="version">
4.2.0k
4.3.0
</div>
@@ -149,49 +149,42 @@
<span class="kn">import</span> <span class="nn">logging</span>
<span class="kn">import</span> <span class="nn">os</span>
<span class="kn">import</span> <span class="nn">platform</span>
<span class="kn">import</span> <span class="nn">shutil</span>
<span class="kn">import</span> <span class="nn">xml.parsers.expat</span> <span class="k">as</span> <span class="nn">expat</span>
<span class="kn">import</span> <span class="nn">json</span>
<span class="kn">from</span> <span class="nn">datetime</span> <span class="k">import</span> <span class="n">datetime</span>
<span class="kn">from</span> <span class="nn">collections</span> <span class="k">import</span> <span class="n">OrderedDict</span>
<span class="kn">from</span> <span class="nn">datetime</span> <span class="k">import</span> <span class="n">timedelta</span>
<span class="kn">from</span> <span class="nn">io</span> <span class="k">import</span> <span class="n">BytesIO</span><span class="p">,</span> <span class="n">StringIO</span>
<span class="kn">from</span> <span class="nn">gzip</span> <span class="k">import</span> <span class="n">GzipFile</span>
<span class="kn">import</span> <span class="nn">tarfile</span>
<span class="kn">import</span> <span class="nn">zipfile</span>
<span class="kn">from</span> <span class="nn">csv</span> <span class="k">import</span> <span class="n">DictWriter</span>
<span class="kn">import</span> <span class="nn">re</span>
<span class="kn">from</span> <span class="nn">base64</span> <span class="k">import</span> <span class="n">b64decode</span>
<span class="kn">import</span> <span class="nn">binascii</span>
<span class="kn">import</span> <span class="nn">shutil</span>
<span class="kn">import</span> <span class="nn">email</span>
<span class="kn">import</span> <span class="nn">tempfile</span>
<span class="kn">import</span> <span class="nn">subprocess</span>
<span class="kn">import</span> <span class="nn">socket</span>
<span class="kn">from</span> <span class="nn">email.mime.application</span> <span class="k">import</span> <span class="n">MIMEApplication</span>
<span class="kn">from</span> <span class="nn">email.mime.multipart</span> <span class="k">import</span> <span class="n">MIMEMultipart</span>
<span class="kn">from</span> <span class="nn">email.mime.text</span> <span class="k">import</span> <span class="n">MIMEText</span>
<span class="kn">from</span> <span class="nn">email.utils</span> <span class="k">import</span> <span class="n">formatdate</span>
<span class="kn">import</span> <span class="nn">email.utils</span>
<span class="kn">import</span> <span class="nn">smtplib</span>
<span class="kn">from</span> <span class="nn">ssl</span> <span class="k">import</span> <span class="n">SSLError</span><span class="p">,</span> <span class="n">CertificateError</span><span class="p">,</span> <span class="n">create_default_context</span>
<span class="kn">import</span> <span class="nn">time</span>
<span class="kn">import</span> <span class="nn">requests</span>
<span class="kn">import</span> <span class="nn">publicsuffix</span>
<span class="kn">import</span> <span class="nn">xmltodict</span>
<span class="kn">import</span> <span class="nn">dns.reversename</span>
<span class="kn">import</span> <span class="nn">dns.resolver</span>
<span class="kn">import</span> <span class="nn">dns.exception</span>
<span class="kn">import</span> <span class="nn">geoip2.database</span>
<span class="kn">import</span> <span class="nn">geoip2.errors</span>
<span class="kn">import</span> <span class="nn">imapclient</span>
<span class="kn">import</span> <span class="nn">imapclient.exceptions</span>
<span class="kn">import</span> <span class="nn">dateparser</span>
<span class="kn">import</span> <span class="nn">mailparser</span>
<span class="n">__version__</span> <span class="o">=</span> <span class="s2">&quot;4.2.0k&quot;</span>
<span class="kn">from</span> <span class="nn">parsedmarc.__version__</span> <span class="k">import</span> <span class="n">__version__</span>
<span class="kn">from</span> <span class="nn">parsedmarc.utils</span> <span class="k">import</span> <span class="n">get_base_domain</span><span class="p">,</span> <span class="n">get_ip_address_info</span>
<span class="kn">from</span> <span class="nn">parsedmarc.utils</span> <span class="k">import</span> <span class="n">is_outlook_msg</span><span class="p">,</span> <span class="n">convert_outlook_msg</span>
<span class="kn">from</span> <span class="nn">parsedmarc.utils</span> <span class="k">import</span> <span class="n">timestamp_to_human</span><span class="p">,</span> <span class="n">human_timestamp_to_datetime</span>
<span class="kn">from</span> <span class="nn">parsedmarc.utils</span> <span class="k">import</span> <span class="n">parse_email</span>
<span class="n">logger</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s2">&quot;parsedmarc&quot;</span><span class="p">)</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;parsedmarc v</span><span class="si">{0}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">__version__</span><span class="p">))</span>
<span class="n">feedback_report_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">&quot;^([\w\-]+): (.+)$&quot;</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">MULTILINE</span><span class="p">)</span>
<span class="n">xml_header_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">&quot;^&lt;\?xml .*$&quot;</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">MULTILINE</span><span class="p">)</span>
@@ -202,13 +195,6 @@
<span class="n">MAGIC_XML</span> <span class="o">=</span> <span class="sa">b</span><span class="s2">&quot;</span><span class="se">\x3c\x3f\x78\x6d\x6c\x20</span><span class="s2">&quot;</span>
<span class="n">USER_AGENT</span> <span class="o">=</span> <span class="s2">&quot;Mozilla/5.0 ((0 </span><span class="si">{1}</span><span class="s2">)) parsedmarc/</span><span class="si">{2}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
<span class="n">platform</span><span class="o">.</span><span class="n">system</span><span class="p">(),</span>
<span class="n">platform</span><span class="o">.</span><span class="n">release</span><span class="p">(),</span>
<span class="n">__version__</span>
<span class="p">)</span>
<div class="viewcode-block" id="ParserError"><a class="viewcode-back" href="../index.html#parsedmarc.ParserError">[docs]</a><span class="k">class</span> <span class="nc">ParserError</span><span class="p">(</span><span class="ne">RuntimeError</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;Raised whenever the parser fails for some reason&quot;&quot;&quot;</span></div>
@@ -233,251 +219,6 @@
<span class="sd">&quot;&quot;&quot;Raised when an invalid DMARC forensic report is encountered&quot;&quot;&quot;</span></div>
<span class="k">def</span> <span class="nf">_get_base_domain</span><span class="p">(</span><span class="n">domain</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Gets the base domain name for the given domain</span>
<span class="sd"> .. note::</span>
<span class="sd"> Results are based on a list of public domain suffixes at</span>
<span class="sd"> https://publicsuffix.org/list/public_suffix_list.dat.</span>
<span class="sd"> This file is saved to the current working directory,</span>
<span class="sd"> where it is used as a cache file for 24 hours.</span>
<span class="sd"> Args:</span>
<span class="sd"> domain (str): A domain or subdomain</span>
<span class="sd"> Returns:</span>
<span class="sd"> str: The base domain of the given domain</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">psl_path</span> <span class="o">=</span> <span class="s2">&quot;.public_suffix_list.dat&quot;</span>
<span class="k">def</span> <span class="nf">download_psl</span><span class="p">():</span>
<span class="n">url</span> <span class="o">=</span> <span class="s2">&quot;https://publicsuffix.org/list/public_suffix_list.dat&quot;</span>
<span class="c1"># Use a browser-like user agent string to bypass some proxy blocks</span>
<span class="n">headers</span> <span class="o">=</span> <span class="p">{</span><span class="s2">&quot;User-Agent&quot;</span><span class="p">:</span> <span class="n">USER_AGENT</span><span class="p">}</span>
<span class="n">fresh_psl</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="n">headers</span><span class="o">=</span><span class="n">headers</span><span class="p">)</span><span class="o">.</span><span class="n">text</span>
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">psl_path</span><span class="p">,</span> <span class="s2">&quot;w&quot;</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s2">&quot;utf-8&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">fresh_psl_file</span><span class="p">:</span>
<span class="n">fresh_psl_file</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">fresh_psl</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">psl_path</span><span class="p">):</span>
<span class="n">download_psl</span><span class="p">()</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">psl_age</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span> <span class="o">-</span> <span class="n">datetime</span><span class="o">.</span><span class="n">fromtimestamp</span><span class="p">(</span>
<span class="n">os</span><span class="o">.</span><span class="n">stat</span><span class="p">(</span><span class="n">psl_path</span><span class="p">)</span><span class="o">.</span><span class="n">st_mtime</span><span class="p">)</span>
<span class="k">if</span> <span class="n">psl_age</span> <span class="o">&gt;</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">hours</span><span class="o">=</span><span class="mi">24</span><span class="p">):</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">download_psl</span><span class="p">()</span>
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
<span class="s2">&quot;Failed to download an updated PSL </span><span class="si">{0}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="p">))</span>
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">psl_path</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s2">&quot;utf-8&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">psl_file</span><span class="p">:</span>
<span class="n">psl</span> <span class="o">=</span> <span class="n">publicsuffix</span><span class="o">.</span><span class="n">PublicSuffixList</span><span class="p">(</span><span class="n">psl_file</span><span class="p">)</span>
<span class="k">return</span> <span class="n">psl</span><span class="o">.</span><span class="n">get_public_suffix</span><span class="p">(</span><span class="n">domain</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">_query_dns</span><span class="p">(</span><span class="n">domain</span><span class="p">,</span> <span class="n">record_type</span><span class="p">,</span> <span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mf">2.0</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Queries DNS</span>
<span class="sd"> Args:</span>
<span class="sd"> domain (str): The domain or subdomain to query about</span>
<span class="sd"> record_type (str): The record type to query for</span>
<span class="sd"> nameservers (list): A list of one or more nameservers to use</span>
<span class="sd"> (Cloudflare&#39;s public DNS resolvers by default)</span>
<span class="sd"> timeout (float): Sets the DNS timeout in seconds</span>
<span class="sd"> Returns:</span>
<span class="sd"> list: A list of answers</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">resolver</span> <span class="o">=</span> <span class="n">dns</span><span class="o">.</span><span class="n">resolver</span><span class="o">.</span><span class="n">Resolver</span><span class="p">()</span>
<span class="n">timeout</span> <span class="o">=</span> <span class="nb">float</span><span class="p">(</span><span class="n">timeout</span><span class="p">)</span>
<span class="k">if</span> <span class="n">nameservers</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">nameservers</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;1.1.1.1&quot;</span><span class="p">,</span> <span class="s2">&quot;1.0.0.1&quot;</span><span class="p">,</span>
<span class="s2">&quot;2606:4700:4700::1111&quot;</span><span class="p">,</span> <span class="s2">&quot;2606:4700:4700::1001&quot;</span><span class="p">,</span>
<span class="p">]</span>
<span class="n">resolver</span><span class="o">.</span><span class="n">nameservers</span> <span class="o">=</span> <span class="n">nameservers</span>
<span class="n">resolver</span><span class="o">.</span><span class="n">timeout</span> <span class="o">=</span> <span class="n">timeout</span>
<span class="n">resolver</span><span class="o">.</span><span class="n">lifetime</span> <span class="o">=</span> <span class="n">timeout</span>
<span class="k">return</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span>
<span class="k">lambda</span> <span class="n">r</span><span class="p">:</span> <span class="n">r</span><span class="o">.</span><span class="n">to_text</span><span class="p">()</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39; &quot;&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;&quot;&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s2">&quot;.&quot;</span><span class="p">),</span>
<span class="n">resolver</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">domain</span><span class="p">,</span> <span class="n">record_type</span><span class="p">,</span> <span class="n">tcp</span><span class="o">=</span><span class="kc">True</span><span class="p">)))</span>
<span class="k">def</span> <span class="nf">_get_reverse_dns</span><span class="p">(</span><span class="n">ip_address</span><span class="p">,</span> <span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mf">2.0</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Resolves an IP address to a hostname using a reverse DNS query</span>
<span class="sd"> Args:</span>
<span class="sd"> ip_address (str): The IP address to resolve</span>
<span class="sd"> nameservers (list): A list of one or more nameservers to use</span>
<span class="sd"> (Cloudflare&#39;s public DNS resolvers by default)</span>
<span class="sd"> timeout (float): Sets the DNS query timeout in seconds</span>
<span class="sd"> Returns:</span>
<span class="sd"> str: The reverse DNS hostname (if any)</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">hostname</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">address</span> <span class="o">=</span> <span class="n">dns</span><span class="o">.</span><span class="n">reversename</span><span class="o">.</span><span class="n">from_address</span><span class="p">(</span><span class="n">ip_address</span><span class="p">)</span>
<span class="n">hostname</span> <span class="o">=</span> <span class="n">_query_dns</span><span class="p">(</span><span class="n">address</span><span class="p">,</span> <span class="s2">&quot;PTR&quot;</span><span class="p">,</span>
<span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
<span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
<span class="k">except</span> <span class="n">dns</span><span class="o">.</span><span class="n">exception</span><span class="o">.</span><span class="n">DNSException</span><span class="p">:</span>
<span class="k">pass</span>
<span class="k">return</span> <span class="n">hostname</span>
<span class="k">def</span> <span class="nf">_timestamp_to_datetime</span><span class="p">(</span><span class="n">timestamp</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Converts a UNIX/DMARC timestamp to a Python ``DateTime`` object</span>
<span class="sd"> Args:</span>
<span class="sd"> timestamp (int): The timestamp</span>
<span class="sd"> Returns:</span>
<span class="sd"> DateTime: The converted timestamp as a Python ``DateTime`` object</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">return</span> <span class="n">datetime</span><span class="o">.</span><span class="n">fromtimestamp</span><span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="n">timestamp</span><span class="p">))</span>
<span class="k">def</span> <span class="nf">_timestamp_to_human</span><span class="p">(</span><span class="n">timestamp</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Converts a UNIX/DMARC timestamp to a human-readable string</span>
<span class="sd"> Args:</span>
<span class="sd"> timestamp: The timestamp</span>
<span class="sd"> Returns:</span>
<span class="sd"> str: The converted timestamp in ``YYYY-MM-DD HH:MM:SS`` format</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">return</span> <span class="n">_timestamp_to_datetime</span><span class="p">(</span><span class="n">timestamp</span><span class="p">)</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%S&quot;</span><span class="p">)</span>
<div class="viewcode-block" id="human_timestamp_to_datetime"><a class="viewcode-back" href="../index.html#parsedmarc.human_timestamp_to_datetime">[docs]</a><span class="k">def</span> <span class="nf">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">human_timestamp</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Converts a human-readable timestamp into a Python ``DateTime`` object</span>
<span class="sd"> Args:</span>
<span class="sd"> human_timestamp (str): A timestamp in `YYYY-MM-DD HH:MM:SS`` format</span>
<span class="sd"> Returns:</span>
<span class="sd"> DateTime: The converted timestamp</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">return</span> <span class="n">datetime</span><span class="o">.</span><span class="n">strptime</span><span class="p">(</span><span class="n">human_timestamp</span><span class="p">,</span> <span class="s2">&quot;%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%S&quot;</span><span class="p">)</span></div>
<div class="viewcode-block" id="human_timestamp_to_timestamp"><a class="viewcode-back" href="../index.html#parsedmarc.human_timestamp_to_timestamp">[docs]</a><span class="k">def</span> <span class="nf">human_timestamp_to_timestamp</span><span class="p">(</span><span class="n">human_timestamp</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Converts a human-readable timestamp into a into a UNIX timestamp</span>
<span class="sd"> Args:</span>
<span class="sd"> human_timestamp (str): A timestamp in `YYYY-MM-DD HH:MM:SS`` format</span>
<span class="sd"> Returns:</span>
<span class="sd"> float: The converted timestamp</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">return</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">human_timestamp</span><span class="p">)</span><span class="o">.</span><span class="n">timestamp</span><span class="p">()</span></div>
<span class="k">def</span> <span class="nf">_get_ip_address_country</span><span class="p">(</span><span class="n">ip_address</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Uses the MaxMind Geolite2 Country database to return the ISO code for the</span>
<span class="sd"> country associated with the given IPv4 or IPv6 address</span>
<span class="sd"> Args:</span>
<span class="sd"> ip_address (str): The IP address to query for</span>
<span class="sd"> Returns:</span>
<span class="sd"> str: And ISO country code associated with the given IP address</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">db_filename</span> <span class="o">=</span> <span class="s2">&quot;.GeoLite2-Country.mmdb&quot;</span>
<span class="k">def</span> <span class="nf">download_country_database</span><span class="p">(</span><span class="n">location</span><span class="o">=</span><span class="s2">&quot;.GeoLite2-Country.mmdb&quot;</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;Downloads the MaxMind Geolite2 Country database</span>
<span class="sd"> Args:</span>
<span class="sd"> location (str): Local location for the database file</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">url</span> <span class="o">=</span> <span class="s2">&quot;https://geolite.maxmind.com/download/geoip/database/&quot;</span> \
<span class="s2">&quot;GeoLite2-Country.tar.gz&quot;</span>
<span class="c1"># Use a browser-like user agent string to bypass some proxy blocks</span>
<span class="n">headers</span> <span class="o">=</span> <span class="p">{</span><span class="s2">&quot;User-Agent&quot;</span><span class="p">:</span> <span class="n">USER_AGENT</span><span class="p">}</span>
<span class="n">original_filename</span> <span class="o">=</span> <span class="s2">&quot;GeoLite2-Country.mmdb&quot;</span>
<span class="n">tar_bytes</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="n">headers</span><span class="o">=</span><span class="n">headers</span><span class="p">)</span><span class="o">.</span><span class="n">content</span>
<span class="n">tar_file</span> <span class="o">=</span> <span class="n">tarfile</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">fileobj</span><span class="o">=</span><span class="n">BytesIO</span><span class="p">(</span><span class="n">tar_bytes</span><span class="p">),</span> <span class="n">mode</span><span class="o">=</span><span class="s2">&quot;r:gz&quot;</span><span class="p">)</span>
<span class="n">tar_dir</span> <span class="o">=</span> <span class="n">tar_file</span><span class="o">.</span><span class="n">getnames</span><span class="p">()[</span><span class="mi">0</span><span class="p">]</span>
<span class="n">tar_path</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">{0}</span><span class="s2">/</span><span class="si">{1}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">tar_dir</span><span class="p">,</span> <span class="n">original_filename</span><span class="p">)</span>
<span class="n">tar_file</span><span class="o">.</span><span class="n">extract</span><span class="p">(</span><span class="n">tar_path</span><span class="p">)</span>
<span class="n">shutil</span><span class="o">.</span><span class="n">move</span><span class="p">(</span><span class="n">tar_path</span><span class="p">,</span> <span class="n">location</span><span class="p">)</span>
<span class="n">shutil</span><span class="o">.</span><span class="n">rmtree</span><span class="p">(</span><span class="n">tar_dir</span><span class="p">)</span>
<span class="n">system_paths</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;/usr/local/share/GeoIP/GeoLite2-Country.mmdb&quot;</span><span class="p">,</span>
<span class="s2">&quot;/usr/share/GeoIP/GeoLite2-Country.mmdb&quot;</span><span class="p">]</span>
<span class="n">db_path</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="k">for</span> <span class="n">system_path</span> <span class="ow">in</span> <span class="n">system_paths</span><span class="p">:</span>
<span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">system_path</span><span class="p">):</span>
<span class="n">db_path</span> <span class="o">=</span> <span class="n">system_path</span>
<span class="k">break</span>
<span class="k">if</span> <span class="n">db_path</span> <span class="o">==</span> <span class="s2">&quot;&quot;</span><span class="p">:</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">db_filename</span><span class="p">):</span>
<span class="n">download_country_database</span><span class="p">(</span><span class="n">db_filename</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">db_age</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span> <span class="o">-</span> <span class="n">datetime</span><span class="o">.</span><span class="n">fromtimestamp</span><span class="p">(</span>
<span class="n">os</span><span class="o">.</span><span class="n">stat</span><span class="p">(</span><span class="n">db_filename</span><span class="p">)</span><span class="o">.</span><span class="n">st_mtime</span><span class="p">)</span>
<span class="k">if</span> <span class="n">db_age</span> <span class="o">&gt;</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="mi">60</span><span class="p">):</span>
<span class="n">download_country_database</span><span class="p">()</span>
<span class="n">db_path</span> <span class="o">=</span> <span class="n">db_filename</span>
<span class="n">db_reader</span> <span class="o">=</span> <span class="n">geoip2</span><span class="o">.</span><span class="n">database</span><span class="o">.</span><span class="n">Reader</span><span class="p">(</span><span class="n">db_path</span><span class="p">)</span>
<span class="n">country</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">country</span> <span class="o">=</span> <span class="n">db_reader</span><span class="o">.</span><span class="n">country</span><span class="p">(</span><span class="n">ip_address</span><span class="p">)</span><span class="o">.</span><span class="n">country</span><span class="o">.</span><span class="n">iso_code</span>
<span class="k">except</span> <span class="n">geoip2</span><span class="o">.</span><span class="n">errors</span><span class="o">.</span><span class="n">AddressNotFoundError</span><span class="p">:</span>
<span class="k">pass</span>
<span class="k">return</span> <span class="n">country</span>
<span class="k">def</span> <span class="nf">_get_ip_address_info</span><span class="p">(</span><span class="n">ip_address</span><span class="p">,</span> <span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mf">2.0</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Returns reverse DNS and country information for the given IP address</span>
<span class="sd"> Args:</span>
<span class="sd"> ip_address (str): The IP address to check</span>
<span class="sd"> nameservers (list): A list of one or more nameservers to use</span>
<span class="sd"> (Cloudflare&#39;s public DNS resolvers by default)</span>
<span class="sd"> timeout (float): Sets the DNS timeout in seconds</span>
<span class="sd"> Returns:</span>
<span class="sd"> OrderedDict: ``ip_address``, ``reverse_dns``</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">ip_address</span> <span class="o">=</span> <span class="n">ip_address</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
<span class="n">info</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">()</span>
<span class="n">info</span><span class="p">[</span><span class="s2">&quot;ip_address&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">ip_address</span>
<span class="n">reverse_dns</span> <span class="o">=</span> <span class="n">_get_reverse_dns</span><span class="p">(</span><span class="n">ip_address</span><span class="p">,</span>
<span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
<span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">)</span>
<span class="n">country</span> <span class="o">=</span> <span class="n">_get_ip_address_country</span><span class="p">(</span><span class="n">ip_address</span><span class="p">)</span>
<span class="n">info</span><span class="p">[</span><span class="s2">&quot;country&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">country</span>
<span class="n">info</span><span class="p">[</span><span class="s2">&quot;reverse_dns&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">reverse_dns</span>
<span class="n">info</span><span class="p">[</span><span class="s2">&quot;base_domain&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">if</span> <span class="n">reverse_dns</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">base_domain</span> <span class="o">=</span> <span class="n">_get_base_domain</span><span class="p">(</span><span class="n">reverse_dns</span><span class="p">)</span>
<span class="n">info</span><span class="p">[</span><span class="s2">&quot;base_domain&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">base_domain</span>
<span class="k">return</span> <span class="n">info</span>
<span class="k">def</span> <span class="nf">_parse_report_record</span><span class="p">(</span><span class="n">record</span><span class="p">,</span> <span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mf">2.0</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Converts a record from a DMARC aggregate report into a more consistent</span>
@@ -496,9 +237,9 @@
<span class="n">nameservers</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;8.8.8.8&quot;</span><span class="p">,</span> <span class="s2">&quot;4.4.4.4&quot;</span><span class="p">]</span>
<span class="n">record</span> <span class="o">=</span> <span class="n">record</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="n">new_record</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">()</span>
<span class="n">new_record</span><span class="p">[</span><span class="s2">&quot;source&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">_get_ip_address_info</span><span class="p">(</span><span class="n">record</span><span class="p">[</span><span class="s2">&quot;row&quot;</span><span class="p">][</span><span class="s2">&quot;source_ip&quot;</span><span class="p">],</span>
<span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
<span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">)</span>
<span class="n">new_record</span><span class="p">[</span><span class="s2">&quot;source&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">get_ip_address_info</span><span class="p">(</span><span class="n">record</span><span class="p">[</span><span class="s2">&quot;row&quot;</span><span class="p">][</span><span class="s2">&quot;source_ip&quot;</span><span class="p">],</span>
<span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
<span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">)</span>
<span class="n">new_record</span><span class="p">[</span><span class="s2">&quot;count&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">record</span><span class="p">[</span><span class="s2">&quot;row&quot;</span><span class="p">][</span><span class="s2">&quot;count&quot;</span><span class="p">])</span>
<span class="n">policy_evaluated</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;row&quot;</span><span class="p">][</span><span class="s2">&quot;policy_evaluated&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="n">new_policy_evaluated</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">([(</span><span class="s2">&quot;disposition&quot;</span><span class="p">,</span> <span class="s2">&quot;none&quot;</span><span class="p">),</span>
@@ -637,7 +378,7 @@
<span class="s2">&quot;email&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">&quot;@&quot;</span><span class="p">)[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
<span class="n">org_name</span> <span class="o">=</span> <span class="n">report_metadata</span><span class="p">[</span><span class="s2">&quot;org_name&quot;</span><span class="p">]</span>
<span class="k">if</span> <span class="n">org_name</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">org_name</span> <span class="o">=</span> <span class="n">_get_base_domain</span><span class="p">(</span><span class="n">org_name</span><span class="p">)</span>
<span class="n">org_name</span> <span class="o">=</span> <span class="n">get_base_domain</span><span class="p">(</span><span class="n">org_name</span><span class="p">)</span>
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">&quot;org_name&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">org_name</span>
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">&quot;org_email&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">report_metadata</span><span class="p">[</span><span class="s2">&quot;email&quot;</span><span class="p">]</span>
<span class="n">extra</span> <span class="o">=</span> <span class="kc">None</span>
@@ -650,8 +391,8 @@
<span class="s2">&quot;&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;&gt;&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">&quot;@&quot;</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">&quot;report_id&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">report_id</span>
<span class="n">date_range</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">&quot;report_metadata&quot;</span><span class="p">][</span><span class="s2">&quot;date_range&quot;</span><span class="p">]</span>
<span class="n">date_range</span><span class="p">[</span><span class="s2">&quot;begin&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">_timestamp_to_human</span><span class="p">(</span><span class="n">date_range</span><span class="p">[</span><span class="s2">&quot;begin&quot;</span><span class="p">])</span>
<span class="n">date_range</span><span class="p">[</span><span class="s2">&quot;end&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">_timestamp_to_human</span><span class="p">(</span><span class="n">date_range</span><span class="p">[</span><span class="s2">&quot;end&quot;</span><span class="p">])</span>
<span class="n">date_range</span><span class="p">[</span><span class="s2">&quot;begin&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">timestamp_to_human</span><span class="p">(</span><span class="n">date_range</span><span class="p">[</span><span class="s2">&quot;begin&quot;</span><span class="p">])</span>
<span class="n">date_range</span><span class="p">[</span><span class="s2">&quot;end&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">timestamp_to_human</span><span class="p">(</span><span class="n">date_range</span><span class="p">[</span><span class="s2">&quot;end&quot;</span><span class="p">])</span>
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">&quot;begin_date&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">date_range</span><span class="p">[</span><span class="s2">&quot;begin&quot;</span><span class="p">]</span>
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">&quot;end_date&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">date_range</span><span class="p">[</span><span class="s2">&quot;end&quot;</span><span class="p">]</span>
<span class="k">if</span> <span class="s2">&quot;error&quot;</span> <span class="ow">in</span> <span class="n">report</span><span class="p">[</span><span class="s2">&quot;report_metadata&quot;</span><span class="p">]:</span>
@@ -897,15 +638,14 @@
<span class="k">return</span> <span class="n">csv_file_object</span><span class="o">.</span><span class="n">getvalue</span><span class="p">()</span></div>
<div class="viewcode-block" id="parse_forensic_report"><a class="viewcode-back" href="../index.html#parsedmarc.parse_forensic_report">[docs]</a><span class="k">def</span> <span class="nf">parse_forensic_report</span><span class="p">(</span><span class="n">feedback_report</span><span class="p">,</span> <span class="n">sample</span><span class="p">,</span> <span class="n">sample_headers_only</span><span class="p">,</span>
<span class="n">msg_date</span><span class="p">,</span> <span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mf">2.0</span><span class="p">):</span>
<div class="viewcode-block" id="parse_forensic_report"><a class="viewcode-back" href="../index.html#parsedmarc.parse_forensic_report">[docs]</a><span class="k">def</span> <span class="nf">parse_forensic_report</span><span class="p">(</span><span class="n">feedback_report</span><span class="p">,</span> <span class="n">sample</span><span class="p">,</span> <span class="n">msg_date</span><span class="p">,</span>
<span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mf">2.0</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Converts a DMARC forensic report and sample to a ``OrderedDict``</span>
<span class="sd"> Args:</span>
<span class="sd"> feedback_report (str): A message&#39;s feedback report as a string</span>
<span class="sd"> sample (str): The RFC 822 headers or RFC 822 message sample</span>
<span class="sd"> sample_headers_only (bool): Set true if the sample is only headers</span>
<span class="sd"> msg_date (str): The message&#39;s date header</span>
<span class="sd"> nameservers (list): A list of one or more nameservers to use</span>
<span class="sd"> (Cloudflare&#39;s public DNS resolvers by default)</span>
@@ -914,36 +654,6 @@
<span class="sd"> Returns:</span>
<span class="sd"> OrderedDict: A parsed report and sample</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">def</span> <span class="nf">convert_address</span><span class="p">(</span><span class="n">original_address</span><span class="p">):</span>
<span class="k">if</span> <span class="n">original_address</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="s2">&quot;&quot;</span><span class="p">:</span>
<span class="n">display_name</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">display_name</span> <span class="o">=</span> <span class="n">original_address</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
<span class="n">address</span> <span class="o">=</span> <span class="n">original_address</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
<span class="k">return</span> <span class="n">OrderedDict</span><span class="p">([(</span><span class="s2">&quot;display_name&quot;</span><span class="p">,</span> <span class="n">display_name</span><span class="p">),</span>
<span class="p">(</span><span class="s2">&quot;address&quot;</span><span class="p">,</span> <span class="n">address</span><span class="p">)])</span>
<span class="k">def</span> <span class="nf">get_filename_safe_subject</span><span class="p">(</span><span class="n">_subject</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Converts a message subject to a string that is safe for a filename</span>
<span class="sd"> Args:</span>
<span class="sd"> _subject (str): A message subject</span>
<span class="sd"> Returns:</span>
<span class="sd"> str: A string safe for a filename</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">invalid_filename_chars</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;</span><span class="se">\\</span><span class="s1">&#39;</span><span class="p">,</span> <span class="s1">&#39;/&#39;</span><span class="p">,</span> <span class="s1">&#39;:&#39;</span><span class="p">,</span> <span class="s1">&#39;&quot;&#39;</span><span class="p">,</span> <span class="s1">&#39;*&#39;</span><span class="p">,</span> <span class="s1">&#39;?&#39;</span><span class="p">,</span> <span class="s1">&#39;|&#39;</span><span class="p">,</span> <span class="s1">&#39;</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">,</span>
<span class="s1">&#39;</span><span class="se">\r</span><span class="s1">&#39;</span><span class="p">]</span>
<span class="k">if</span> <span class="n">_subject</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">_subject</span> <span class="o">=</span> <span class="s2">&quot;No Subject&quot;</span>
<span class="k">for</span> <span class="n">char</span> <span class="ow">in</span> <span class="n">invalid_filename_chars</span><span class="p">:</span>
<span class="n">_subject</span> <span class="o">=</span> <span class="n">_subject</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="n">char</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">)</span>
<span class="n">_subject</span> <span class="o">=</span> <span class="n">_subject</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s2">&quot;.&quot;</span><span class="p">)</span>
<span class="k">return</span> <span class="n">_subject</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">parsed_report</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">()</span>
<span class="n">report_values</span> <span class="o">=</span> <span class="n">feedback_report_regex</span><span class="o">.</span><span class="n">findall</span><span class="p">(</span><span class="n">feedback_report</span><span class="p">)</span>
@@ -951,18 +661,21 @@
<span class="n">key</span> <span class="o">=</span> <span class="n">report_value</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;-&quot;</span><span class="p">,</span> <span class="s2">&quot;_&quot;</span><span class="p">)</span>
<span class="n">parsed_report</span><span class="p">[</span><span class="n">key</span><span class="p">]</span> <span class="o">=</span> <span class="n">report_value</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
<span class="k">if</span> <span class="s2">&quot;content_type&quot;</span> <span class="ow">in</span> <span class="n">parsed_report</span><span class="p">:</span>
<span class="k">del</span> <span class="n">parsed_report</span><span class="p">[</span><span class="s2">&quot;content_type&quot;</span><span class="p">]</span>
<span class="k">if</span> <span class="s2">&quot;arrival_date&quot;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_report</span><span class="p">:</span>
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">&quot;arrival_date&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">msg_date</span>
<span class="n">arrival_utc</span> <span class="o">=</span> <span class="n">dateparser</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">parsed_report</span><span class="p">[</span><span class="s2">&quot;arrival_date&quot;</span><span class="p">],</span>
<span class="n">settings</span><span class="o">=</span><span class="p">{</span><span class="s2">&quot;TO_TIMEZONE&quot;</span><span class="p">:</span> <span class="s2">&quot;UTC&quot;</span><span class="p">})</span>
<span class="n">arrival_utc</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span>
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">&quot;arrival_date&quot;</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="n">arrival_utc</span> <span class="o">=</span> <span class="n">arrival_utc</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%S&quot;</span><span class="p">)</span>
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">&quot;arrival_date_utc&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">arrival_utc</span>
<span class="n">ip_address</span> <span class="o">=</span> <span class="n">parsed_report</span><span class="p">[</span><span class="s2">&quot;source_ip&quot;</span><span class="p">]</span>
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">&quot;source&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">_get_ip_address_info</span><span class="p">(</span><span class="n">ip_address</span><span class="p">,</span>
<span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
<span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">)</span>
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">&quot;source&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">get_ip_address_info</span><span class="p">(</span><span class="n">ip_address</span><span class="p">,</span>
<span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
<span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">)</span>
<span class="k">del</span> <span class="n">parsed_report</span><span class="p">[</span><span class="s2">&quot;source_ip&quot;</span><span class="p">]</span>
<span class="k">if</span> <span class="s2">&quot;identity_alignment&quot;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_report</span><span class="p">:</span>
@@ -987,67 +700,15 @@
<span class="k">if</span> <span class="n">optional_field</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_report</span><span class="p">:</span>
<span class="n">parsed_report</span><span class="p">[</span><span class="n">optional_field</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
<span class="n">parsed_mail</span> <span class="o">=</span> <span class="n">mailparser</span><span class="o">.</span><span class="n">parse_from_string</span><span class="p">(</span><span class="n">sample</span><span class="p">)</span>
<span class="n">parsed_headers</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">parsed_mail</span><span class="o">.</span><span class="n">headers_json</span><span class="p">)</span>
<span class="n">parsed_message</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">parsed_mail</span><span class="o">.</span><span class="n">mail_json</span><span class="p">)</span>
<span class="n">parsed_sample</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">([(</span><span class="s2">&quot;headers&quot;</span><span class="p">,</span> <span class="n">parsed_headers</span><span class="p">)])</span>
<span class="k">for</span> <span class="n">key</span> <span class="ow">in</span> <span class="n">parsed_message</span><span class="p">:</span>
<span class="n">parsed_sample</span><span class="p">[</span><span class="n">key</span><span class="p">]</span> <span class="o">=</span> <span class="n">parsed_message</span><span class="p">[</span><span class="n">key</span><span class="p">]</span>
<span class="n">parsed_sample</span> <span class="o">=</span> <span class="n">parse_email</span><span class="p">(</span><span class="n">sample</span><span class="p">)</span>
<span class="n">parsed_sample</span><span class="p">[</span><span class="s2">&quot;date&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">parsed_sample</span><span class="p">[</span><span class="s2">&quot;date&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;T&quot;</span><span class="p">,</span> <span class="s2">&quot; &quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="s2">&quot;received&quot;</span> <span class="ow">in</span> <span class="n">parsed_message</span><span class="p">:</span>
<span class="k">for</span> <span class="n">received</span> <span class="ow">in</span> <span class="n">parsed_message</span><span class="p">[</span><span class="s2">&quot;received&quot;</span><span class="p">]:</span>
<span class="k">if</span> <span class="s2">&quot;date_utc&quot;</span> <span class="ow">in</span> <span class="n">received</span><span class="p">:</span>
<span class="n">received</span><span class="p">[</span><span class="s2">&quot;date_utc&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">received</span><span class="p">[</span><span class="s2">&quot;date_utc&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;T&quot;</span><span class="p">,</span>
<span class="s2">&quot; &quot;</span><span class="p">)</span>
<span class="n">msg_from</span> <span class="o">=</span> <span class="n">convert_address</span><span class="p">(</span><span class="n">parsed_sample</span><span class="p">[</span><span class="s2">&quot;from&quot;</span><span class="p">][</span><span class="mi">0</span><span class="p">])</span>
<span class="n">parsed_sample</span><span class="p">[</span><span class="s2">&quot;from&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">msg_from</span>
<span class="k">if</span> <span class="s2">&quot;reported_domain&quot;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_report</span><span class="p">:</span>
<span class="n">domain</span> <span class="o">=</span> <span class="n">msg_from</span><span class="p">[</span><span class="s2">&quot;address&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">&quot;@&quot;</span><span class="p">)[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">&quot;reported_domain&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">domain</span>
<span class="k">if</span> <span class="s2">&quot;reply_to&quot;</span> <span class="ow">in</span> <span class="n">parsed_sample</span><span class="p">:</span>
<span class="n">parsed_sample</span><span class="p">[</span><span class="s2">&quot;reply_to&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">convert_address</span><span class="p">(</span><span class="n">x</span><span class="p">),</span>
<span class="n">parsed_sample</span><span class="p">[</span><span class="s2">&quot;reply_to&quot;</span><span class="p">]))</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">parsed_sample</span><span class="p">[</span><span class="s2">&quot;reply_to&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">if</span> <span class="s2">&quot;to&quot;</span> <span class="ow">in</span> <span class="n">parsed_sample</span><span class="p">:</span>
<span class="n">parsed_sample</span><span class="p">[</span><span class="s2">&quot;to&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">convert_address</span><span class="p">(</span><span class="n">x</span><span class="p">),</span>
<span class="n">parsed_sample</span><span class="p">[</span><span class="s2">&quot;to&quot;</span><span class="p">]))</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">parsed_sample</span><span class="p">[</span><span class="s2">&quot;to&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">if</span> <span class="s2">&quot;cc&quot;</span> <span class="ow">in</span> <span class="n">parsed_sample</span><span class="p">:</span>
<span class="n">parsed_sample</span><span class="p">[</span><span class="s2">&quot;cc&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">convert_address</span><span class="p">(</span><span class="n">x</span><span class="p">),</span>
<span class="n">parsed_sample</span><span class="p">[</span><span class="s2">&quot;cc&quot;</span><span class="p">]))</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">parsed_sample</span><span class="p">[</span><span class="s2">&quot;cc&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">if</span> <span class="s2">&quot;bcc&quot;</span> <span class="ow">in</span> <span class="n">parsed_sample</span><span class="p">:</span>
<span class="n">parsed_sample</span><span class="p">[</span><span class="s2">&quot;bcc&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">convert_address</span><span class="p">(</span><span class="n">x</span><span class="p">),</span>
<span class="n">parsed_sample</span><span class="p">[</span><span class="s2">&quot;bcc&quot;</span><span class="p">]))</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">parsed_sample</span><span class="p">[</span><span class="s2">&quot;bcc&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">if</span> <span class="s2">&quot;delivered_to&quot;</span> <span class="ow">in</span> <span class="n">parsed_sample</span><span class="p">:</span>
<span class="n">parsed_sample</span><span class="p">[</span><span class="s2">&quot;delivered_to&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span>
<span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">convert_address</span><span class="p">(</span><span class="n">x</span><span class="p">),</span>
<span class="n">parsed_sample</span><span class="p">[</span><span class="s2">&quot;delivered_to&quot;</span><span class="p">])</span>
<span class="p">)</span>
<span class="k">if</span> <span class="s2">&quot;attachments&quot;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_sample</span><span class="p">:</span>
<span class="n">parsed_sample</span><span class="p">[</span><span class="s2">&quot;attachments&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">if</span> <span class="s2">&quot;subject&quot;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_sample</span><span class="p">:</span>
<span class="n">parsed_sample</span><span class="p">[</span><span class="s2">&quot;subject&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
<span class="n">parsed_sample</span><span class="p">[</span><span class="s2">&quot;filename_safe_subject&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">get_filename_safe_subject</span><span class="p">(</span>
<span class="n">parsed_sample</span><span class="p">[</span><span class="s2">&quot;subject&quot;</span><span class="p">])</span>
<span class="k">if</span> <span class="s2">&quot;body&quot;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_sample</span><span class="p">:</span>
<span class="n">parsed_sample</span><span class="p">[</span><span class="s2">&quot;body&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">&quot;reported_domain&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">parsed_sample</span><span class="p">[</span><span class="s2">&quot;from&quot;</span><span class="p">][</span><span class="s2">&quot;domain&quot;</span><span class="p">]</span>
<span class="n">sample_headers_only</span> <span class="o">=</span> <span class="kc">False</span>
<span class="n">number_of_attachments</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">parsed_sample</span><span class="p">[</span><span class="s2">&quot;attachments&quot;</span><span class="p">])</span>
<span class="k">if</span> <span class="n">number_of_attachments</span> <span class="o">&lt;</span> <span class="mi">1</span> <span class="ow">and</span> <span class="n">parsed_sample</span><span class="p">[</span><span class="s2">&quot;body&quot;</span><span class="p">]</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">sample_headers_only</span> <span class="o">=</span> <span class="kc">True</span>
<span class="k">if</span> <span class="n">sample_headers_only</span> <span class="ow">and</span> <span class="n">parsed_sample</span><span class="p">[</span><span class="s2">&quot;has_defects&quot;</span><span class="p">]:</span>
<span class="k">del</span> <span class="n">parsed_sample</span><span class="p">[</span><span class="s2">&quot;defects&quot;</span><span class="p">]</span>
<span class="k">del</span> <span class="n">parsed_sample</span><span class="p">[</span><span class="s2">&quot;defects_categories&quot;</span><span class="p">]</span>
@@ -1124,102 +785,66 @@
<span class="sd"> * ``report_type``: ``aggregate`` or ``forensic``</span>
<span class="sd"> * ``report``: The parsed report</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">result</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">def</span> <span class="nf">is_outlook_msg</span><span class="p">(</span><span class="n">suspect_bytes</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;Checks if the given content is a Outlook msg OLE file&quot;&quot;&quot;</span>
<span class="k">return</span> <span class="n">suspect_bytes</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="sa">b</span><span class="s2">&quot;</span><span class="se">\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">convert_outlook_msg</span><span class="p">(</span><span class="n">msg_bytes</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Uses the ``msgconvert`` Perl utility to convert an Outlook MS file to</span>
<span class="sd"> standard RFC 822 format</span>
<span class="sd"> Args:</span>
<span class="sd"> msg_bytes (bytes): the content of the .msg file</span>
<span class="sd"> Returns:</span>
<span class="sd"> A RFC 822 string</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">is_outlook_msg</span><span class="p">(</span><span class="n">msg_bytes</span><span class="p">):</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;The supplied bytes are not an Outlook MSG file&quot;</span><span class="p">)</span>
<span class="n">orig_dir</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">getcwd</span><span class="p">()</span>
<span class="n">tmp_dir</span> <span class="o">=</span> <span class="n">tempfile</span><span class="o">.</span><span class="n">mkdtemp</span><span class="p">()</span>
<span class="n">os</span><span class="o">.</span><span class="n">chdir</span><span class="p">(</span><span class="n">tmp_dir</span><span class="p">)</span>
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="s2">&quot;sample.msg&quot;</span><span class="p">,</span> <span class="s2">&quot;wb&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">msg_file</span><span class="p">:</span>
<span class="n">msg_file</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">msg_bytes</span><span class="p">)</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">subprocess</span><span class="o">.</span><span class="n">check_call</span><span class="p">([</span><span class="s2">&quot;msgconvert&quot;</span><span class="p">,</span> <span class="s2">&quot;sample.msg&quot;</span><span class="p">])</span>
<span class="n">eml_path</span> <span class="o">=</span> <span class="s2">&quot;sample.eml&quot;</span>
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">eml_path</span><span class="p">,</span> <span class="s2">&quot;rb&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">eml_file</span><span class="p">:</span>
<span class="n">rfc822</span> <span class="o">=</span> <span class="n">eml_file</span><span class="o">.</span><span class="n">read</span><span class="p">()</span>
<span class="k">except</span> <span class="ne">FileNotFoundError</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">ParserError</span><span class="p">(</span><span class="s2">&quot;msgconvert utility not found&quot;</span><span class="p">)</span>
<span class="k">finally</span><span class="p">:</span>
<span class="n">os</span><span class="o">.</span><span class="n">chdir</span><span class="p">(</span><span class="n">orig_dir</span><span class="p">)</span>
<span class="n">shutil</span><span class="o">.</span><span class="n">rmtree</span><span class="p">(</span><span class="n">tmp_dir</span><span class="p">)</span>
<span class="k">return</span> <span class="n">rfc822</span>
<span class="k">def</span> <span class="nf">decode_header</span><span class="p">(</span><span class="n">header</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;Decodes a RFC 822 email header&quot;&quot;&quot;</span>
<span class="n">header</span> <span class="o">=</span> <span class="n">header</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;</span><span class="se">\r</span><span class="s2">&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;</span><span class="se">\n</span><span class="s2">&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">)</span>
<span class="n">decoded_header</span> <span class="o">=</span> <span class="n">email</span><span class="o">.</span><span class="n">header</span><span class="o">.</span><span class="n">decode_header</span><span class="p">(</span><span class="n">header</span><span class="p">)</span>
<span class="n">header</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="k">for</span> <span class="n">header_part</span> <span class="ow">in</span> <span class="n">decoded_header</span><span class="p">:</span>
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">header_part</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span> <span class="o">==</span> <span class="nb">bytes</span><span class="p">:</span>
<span class="n">encoding</span> <span class="o">=</span> <span class="n">header_part</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="ow">or</span> <span class="s2">&quot;ascii&quot;</span>
<span class="n">header_part</span> <span class="o">=</span> <span class="n">header_part</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="n">encoding</span><span class="o">=</span><span class="n">encoding</span><span class="p">,</span>
<span class="n">errors</span><span class="o">=</span><span class="s2">&quot;replace&quot;</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">header_part</span> <span class="o">=</span> <span class="n">header_part</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
<span class="n">header</span> <span class="o">+=</span> <span class="n">header_part</span>
<span class="n">header</span> <span class="o">=</span> <span class="n">header</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;</span><span class="se">\r</span><span class="s2">&quot;</span><span class="p">,</span> <span class="s2">&quot; &quot;</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;</span><span class="se">\n</span><span class="s2">&quot;</span><span class="p">,</span> <span class="s2">&quot; &quot;</span><span class="p">)</span>
<span class="k">return</span> <span class="n">header</span>
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">input_</span><span class="p">)</span> <span class="o">==</span> <span class="nb">bytes</span><span class="p">:</span>
<span class="k">try</span><span class="p">:</span>
<span class="k">if</span> <span class="n">is_outlook_msg</span><span class="p">(</span><span class="n">input_</span><span class="p">):</span>
<span class="n">input_</span> <span class="o">=</span> <span class="n">convert_outlook_msg</span><span class="p">(</span><span class="n">input_</span><span class="p">)</span>
<span class="n">input_</span> <span class="o">=</span> <span class="n">input_</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s2">&quot;utf-8&quot;</span><span class="p">,</span> <span class="n">errors</span><span class="o">=</span><span class="s2">&quot;replace&quot;</span><span class="p">)</span>
<span class="n">result</span> <span class="o">=</span> <span class="kc">None</span>
<span class="n">msg</span> <span class="o">=</span> <span class="n">email</span><span class="o">.</span><span class="n">message_from_string</span><span class="p">(</span><span class="n">input_</span><span class="p">)</span>
<span class="n">msg</span> <span class="o">=</span> <span class="n">mailparser</span><span class="o">.</span><span class="n">parse_from_string</span><span class="p">(</span><span class="n">input_</span><span class="p">)</span>
<span class="n">msg_headers</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">msg</span><span class="o">.</span><span class="n">headers_json</span><span class="p">)</span>
<span class="n">date</span> <span class="o">=</span> <span class="n">email</span><span class="o">.</span><span class="n">utils</span><span class="o">.</span><span class="n">format_datetime</span><span class="p">(</span><span class="n">datetime</span><span class="o">.</span><span class="n">utcnow</span><span class="p">())</span>
<span class="k">if</span> <span class="s2">&quot;Date&quot;</span> <span class="ow">in</span> <span class="n">msg_headers</span><span class="p">:</span>
<span class="n">date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span>
<span class="n">msg_headers</span><span class="p">[</span><span class="s2">&quot;Date&quot;</span><span class="p">])</span>
<span class="n">msg</span> <span class="o">=</span> <span class="n">email</span><span class="o">.</span><span class="n">message_from_string</span><span class="p">(</span><span class="n">input_</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">ParserError</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
<span class="n">subject</span> <span class="o">=</span> <span class="kc">None</span>
<span class="n">feedback_report</span> <span class="o">=</span> <span class="kc">None</span>
<span class="n">sample_headers_only</span> <span class="o">=</span> <span class="kc">False</span>
<span class="n">sample</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">if</span> <span class="s2">&quot;subject&quot;</span> <span class="ow">in</span> <span class="n">msg</span><span class="p">:</span>
<span class="n">subject</span> <span class="o">=</span> <span class="n">decode_header</span><span class="p">(</span><span class="n">msg</span><span class="p">[</span><span class="s2">&quot;subject&quot;</span><span class="p">])</span>
<span class="n">date</span> <span class="o">=</span> <span class="n">decode_header</span><span class="p">(</span><span class="n">msg</span><span class="p">[</span><span class="s2">&quot;date&quot;</span><span class="p">])</span>
<span class="k">if</span> <span class="s2">&quot;Subject&quot;</span> <span class="ow">in</span> <span class="n">msg_headers</span><span class="p">:</span>
<span class="n">subject</span> <span class="o">=</span> <span class="n">msg_headers</span><span class="p">[</span><span class="s2">&quot;Subject&quot;</span><span class="p">]</span>
<span class="k">for</span> <span class="n">part</span> <span class="ow">in</span> <span class="n">msg</span><span class="o">.</span><span class="n">walk</span><span class="p">():</span>
<span class="n">content_type</span> <span class="o">=</span> <span class="n">part</span><span class="o">.</span><span class="n">get_content_type</span><span class="p">()</span>
<span class="n">payload</span> <span class="o">=</span> <span class="n">part</span><span class="o">.</span><span class="n">get_payload</span><span class="p">()</span>
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">payload</span><span class="p">)</span> <span class="o">==</span> <span class="nb">list</span><span class="p">:</span>
<span class="n">payload</span> <span class="o">=</span> <span class="n">payload</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span>
<span class="k">if</span> <span class="n">content_type</span> <span class="o">==</span> <span class="s2">&quot;message/feedback-report&quot;</span><span class="p">:</span>
<span class="k">try</span><span class="p">:</span>
<span class="k">if</span> <span class="s2">&quot;Feedback-Type&quot;</span> <span class="ow">in</span> <span class="n">payload</span><span class="p">:</span>
<span class="n">feedback_report</span> <span class="o">=</span> <span class="n">payload</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">feedback_report</span> <span class="o">=</span> <span class="n">b64decode</span><span class="p">(</span><span class="n">payload</span><span class="p">)</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span>
<span class="n">feedback_report</span> <span class="o">=</span> <span class="n">feedback_report</span><span class="o">.</span><span class="n">lstrip</span><span class="p">(</span><span class="s2">&quot;b&#39;&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s2">&quot;&#39;&quot;</span><span class="p">)</span>
<span class="n">feedback_report</span> <span class="o">=</span> <span class="n">feedback_report</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;</span><span class="se">\\</span><span class="s2">r&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">)</span>
<span class="n">feedback_report</span> <span class="o">=</span> <span class="n">feedback_report</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;</span><span class="se">\\</span><span class="s2">n&quot;</span><span class="p">,</span> <span class="s2">&quot;</span><span class="se">\n</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="k">except</span> <span class="p">(</span><span class="ne">ValueError</span><span class="p">,</span> <span class="ne">TypeError</span><span class="p">,</span> <span class="n">binascii</span><span class="o">.</span><span class="n">Error</span><span class="p">):</span>
<span class="n">feedback_report</span> <span class="o">=</span> <span class="n">payload</span>
<span class="k">for</span> <span class="n">payload_</span> <span class="ow">in</span> <span class="n">payload</span><span class="p">:</span>
<span class="n">payload_</span> <span class="o">=</span> <span class="n">payload_</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span>
<span class="k">if</span> <span class="n">content_type</span> <span class="o">==</span> <span class="s2">&quot;message/feedback-report&quot;</span><span class="p">:</span>
<span class="k">try</span><span class="p">:</span>
<span class="k">if</span> <span class="s2">&quot;Feedback-Type&quot;</span> <span class="ow">in</span> <span class="n">payload_</span><span class="p">:</span>
<span class="n">feedback_report</span> <span class="o">=</span> <span class="n">payload_</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">feedback_report</span> <span class="o">=</span> <span class="n">b64decode</span><span class="p">(</span><span class="n">payload_</span><span class="p">)</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span>
<span class="n">feedback_report</span> <span class="o">=</span> <span class="n">feedback_report</span><span class="o">.</span><span class="n">lstrip</span><span class="p">(</span>
<span class="s2">&quot;b&#39;&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s2">&quot;&#39;&quot;</span><span class="p">)</span>
<span class="n">feedback_report</span> <span class="o">=</span> <span class="n">feedback_report</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;</span><span class="se">\\</span><span class="s2">r&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">)</span>
<span class="n">feedback_report</span> <span class="o">=</span> <span class="n">feedback_report</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;</span><span class="se">\\</span><span class="s2">n&quot;</span><span class="p">,</span> <span class="s2">&quot;</span><span class="se">\n</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="k">except</span> <span class="p">(</span><span class="ne">ValueError</span><span class="p">,</span> <span class="ne">TypeError</span><span class="p">,</span> <span class="n">binascii</span><span class="o">.</span><span class="n">Error</span><span class="p">):</span>
<span class="n">feedback_report</span> <span class="o">=</span> <span class="n">payload</span>
<span class="k">elif</span> <span class="n">content_type</span> <span class="o">==</span> <span class="s2">&quot;text/rfc822-headers&quot;</span><span class="p">:</span>
<span class="n">sample</span> <span class="o">=</span> <span class="n">payload</span>
<span class="n">sample_headers_only</span> <span class="o">=</span> <span class="kc">True</span>
<span class="k">elif</span> <span class="n">content_type</span> <span class="o">==</span> <span class="s2">&quot;message/rfc822&quot;</span><span class="p">:</span>
<span class="n">sample</span> <span class="o">=</span> <span class="n">payload</span>
<span class="n">sample_headers_only</span> <span class="o">=</span> <span class="kc">False</span>
<span class="k">elif</span> <span class="n">content_type</span> <span class="o">==</span> <span class="s2">&quot;text/rfc822-headers&quot;</span><span class="p">:</span>
<span class="n">sample</span> <span class="o">=</span> <span class="n">payload</span>
<span class="k">elif</span> <span class="n">content_type</span> <span class="o">==</span> <span class="s2">&quot;message/rfc822&quot;</span><span class="p">:</span>
<span class="n">sample</span> <span class="o">=</span> <span class="n">payload</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span>
<span class="k">elif</span> <span class="n">content_type</span> <span class="o">==</span> <span class="s2">&quot;multipart/report&quot;</span><span class="p">:</span>
<span class="k">if</span> <span class="s2">&quot;Feedback-Type&quot;</span> <span class="ow">in</span> <span class="n">payload_</span><span class="p">:</span>
<span class="n">feedback_report</span> <span class="o">=</span> <span class="n">payload_</span>
<span class="k">elif</span> <span class="s2">&quot;message/rfc822&quot;</span> <span class="ow">in</span> <span class="n">payload_</span><span class="p">:</span>
<span class="n">sample</span> <span class="o">=</span> <span class="n">payload_</span>
<span class="k">elif</span> <span class="s2">&quot;text/rfc822-headers&quot;</span> <span class="ow">in</span> <span class="n">payload_</span><span class="p">:</span>
<span class="n">sample</span> <span class="o">=</span> <span class="n">payload_</span>
<span class="k">if</span> <span class="n">feedback_report</span> <span class="ow">and</span> <span class="n">sample</span><span class="p">:</span>
<span class="n">forensic_report</span> <span class="o">=</span> <span class="n">parse_forensic_report</span><span class="p">(</span><span class="n">feedback_report</span><span class="p">,</span>
<span class="n">sample</span><span class="p">,</span>
<span class="n">sample_headers_only</span><span class="p">,</span>
<span class="n">date</span><span class="p">,</span>
<span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
<span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">)</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">forensic_report</span> <span class="o">=</span> <span class="n">parse_forensic_report</span><span class="p">(</span>
<span class="n">feedback_report</span><span class="p">,</span>
<span class="n">sample</span><span class="p">,</span>
<span class="n">date</span><span class="p">,</span>
<span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
<span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">ParserError</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
<span class="n">result</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">([(</span><span class="s2">&quot;report_type&quot;</span><span class="p">,</span> <span class="s2">&quot;forensic&quot;</span><span class="p">),</span>
<span class="p">(</span><span class="s2">&quot;report&quot;</span><span class="p">,</span> <span class="n">forensic_report</span><span class="p">)])</span>
@@ -1487,7 +1112,7 @@
<span class="k">else</span><span class="p">:</span>
<span class="n">move_messages</span><span class="p">([</span><span class="n">message_uid</span><span class="p">],</span> <span class="n">invalid_reports_folder</span><span class="p">)</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
<span class="s2">&quot;Moving message UID </span><span class="si">{0}</span><span class="s2"> to {1)&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
<span class="s2">&quot;Moving message UID </span><span class="si">{0}</span><span class="s2"> to </span><span class="si">{1}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
<span class="n">message_uid</span><span class="p">,</span> <span class="n">invalid_reports_folder</span><span class="p">))</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">test</span><span class="p">:</span>
@@ -1754,7 +1379,7 @@
<span class="n">msg</span> <span class="o">=</span> <span class="n">MIMEMultipart</span><span class="p">()</span>
<span class="n">msg</span><span class="p">[</span><span class="s1">&#39;From&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">mail_from</span>
<span class="n">msg</span><span class="p">[</span><span class="s1">&#39;To&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;, &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">mail_to</span><span class="p">)</span>
<span class="n">msg</span><span class="p">[</span><span class="s1">&#39;Date&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">formatdate</span><span class="p">(</span><span class="n">localtime</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="n">msg</span><span class="p">[</span><span class="s1">&#39;Date&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">email</span><span class="o">.</span><span class="n">utils</span><span class="o">.</span><span class="n">formatdate</span><span class="p">(</span><span class="n">localtime</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="n">msg</span><span class="p">[</span><span class="s1">&#39;Subject&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">subject</span> <span class="ow">or</span> <span class="s2">&quot;DMARC results for </span><span class="si">{0}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">date_string</span><span class="p">)</span>
<span class="n">text</span> <span class="o">=</span> <span class="n">message</span> <span class="ow">or</span> <span class="s2">&quot;Please see the attached zip file</span><span class="se">\n</span><span class="s2">&quot;</span>
@@ -2036,7 +1661,7 @@
<script type="text/javascript">
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../',
VERSION:'4.2.0k',
VERSION:'4.3.0',
LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
_modules/parsedmarc/elastic.html
+12 -9
View File
@@ -8,7 +8,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>parsedmarc.elastic &mdash; parsedmarc 4.2.0k documentation</title>
<title>parsedmarc.elastic &mdash; parsedmarc 4.3.0 documentation</title>
@@ -56,7 +56,7 @@
<div class="version">
4.2.0k
4.3.0
</div>
@@ -150,11 +150,12 @@
<span class="kn">import</span> <span class="nn">logging</span>
<span class="kn">from</span> <span class="nn">collections</span> <span class="k">import</span> <span class="n">OrderedDict</span>
<span class="kn">import</span> <span class="nn">parsedmarc</span>
<span class="kn">from</span> <span class="nn">elasticsearch_dsl.search</span> <span class="k">import</span> <span class="n">Q</span>
<span class="kn">from</span> <span class="nn">elasticsearch_dsl</span> <span class="k">import</span> <span class="n">connections</span><span class="p">,</span> <span class="n">Object</span><span class="p">,</span> <span class="n">Document</span><span class="p">,</span> <span class="n">Index</span><span class="p">,</span> <span class="n">Nested</span><span class="p">,</span> \
<span class="n">InnerDoc</span><span class="p">,</span> <span class="n">Integer</span><span class="p">,</span> <span class="n">Text</span><span class="p">,</span> <span class="n">Boolean</span><span class="p">,</span> <span class="n">DateRange</span><span class="p">,</span> <span class="n">Ip</span><span class="p">,</span> <span class="n">Date</span>
<span class="kn">from</span> <span class="nn">parsedmarc.utils</span> <span class="k">import</span> <span class="n">human_timestamp_to_datetime</span>
<span class="n">logger</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s2">&quot;parsedmarc&quot;</span><span class="p">)</span>
@@ -358,8 +359,8 @@
<span class="n">org_name</span> <span class="o">=</span> <span class="n">metadata</span><span class="p">[</span><span class="s2">&quot;org_name&quot;</span><span class="p">]</span>
<span class="n">report_id</span> <span class="o">=</span> <span class="n">metadata</span><span class="p">[</span><span class="s2">&quot;report_id&quot;</span><span class="p">]</span>
<span class="n">domain</span> <span class="o">=</span> <span class="n">aggregate_report</span><span class="p">[</span><span class="s2">&quot;policy_published&quot;</span><span class="p">][</span><span class="s2">&quot;domain&quot;</span><span class="p">]</span>
<span class="n">begin_date</span> <span class="o">=</span> <span class="n">parsedmarc</span><span class="o">.</span><span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">metadata</span><span class="p">[</span><span class="s2">&quot;begin_date&quot;</span><span class="p">])</span>
<span class="n">end_date</span> <span class="o">=</span> <span class="n">parsedmarc</span><span class="o">.</span><span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">metadata</span><span class="p">[</span><span class="s2">&quot;end_date&quot;</span><span class="p">])</span>
<span class="n">begin_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">metadata</span><span class="p">[</span><span class="s2">&quot;begin_date&quot;</span><span class="p">])</span>
<span class="n">end_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">metadata</span><span class="p">[</span><span class="s2">&quot;end_date&quot;</span><span class="p">])</span>
<span class="n">begin_date_human</span> <span class="o">=</span> <span class="n">begin_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%S&quot;</span><span class="p">)</span>
<span class="n">end_date_human</span> <span class="o">=</span> <span class="n">end_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%S&quot;</span><span class="p">)</span>
<span class="n">aggregate_report</span><span class="p">[</span><span class="s2">&quot;begin_date&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">begin_date</span>
@@ -453,15 +454,17 @@
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;Saving forensic report to Elasticsearch&quot;</span><span class="p">)</span>
<span class="n">forensic_report</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="n">sample_date</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;parsed_sample&quot;</span><span class="p">][</span><span class="s2">&quot;date&quot;</span><span class="p">]</span>
<span class="n">sample_date</span> <span class="o">=</span> <span class="n">parsedmarc</span><span class="o">.</span><span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">sample_date</span><span class="p">)</span>
<span class="n">sample_date</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">if</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;parsed_sample&quot;</span><span class="p">][</span><span class="s2">&quot;date&quot;</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">sample_date</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;parsed_sample&quot;</span><span class="p">][</span><span class="s2">&quot;date&quot;</span><span class="p">]</span>
<span class="n">sample_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">sample_date</span><span class="p">)</span>
<span class="n">original_headers</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;parsed_sample&quot;</span><span class="p">][</span><span class="s2">&quot;headers&quot;</span><span class="p">]</span>
<span class="n">headers</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">()</span>
<span class="k">for</span> <span class="n">original_header</span> <span class="ow">in</span> <span class="n">original_headers</span><span class="p">:</span>
<span class="n">headers</span><span class="p">[</span><span class="n">original_header</span><span class="o">.</span><span class="n">lower</span><span class="p">()]</span> <span class="o">=</span> <span class="n">original_headers</span><span class="p">[</span><span class="n">original_header</span><span class="p">]</span>
<span class="n">arrival_date_human</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;arrival_date_utc&quot;</span><span class="p">]</span>
<span class="n">arrival_date</span> <span class="o">=</span> <span class="n">parsedmarc</span><span class="o">.</span><span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">arrival_date_human</span><span class="p">)</span>
<span class="n">arrival_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">arrival_date_human</span><span class="p">)</span>
<span class="n">search</span> <span class="o">=</span> <span class="n">Index</span><span class="p">(</span><span class="n">index</span><span class="p">)</span><span class="o">.</span><span class="n">search</span><span class="p">()</span>
<span class="n">from_query</span> <span class="o">=</span> <span class="p">{</span><span class="s2">&quot;match&quot;</span><span class="p">:</span> <span class="p">{</span><span class="s2">&quot;sample.headers.from&quot;</span><span class="p">:</span> <span class="n">headers</span><span class="p">[</span><span class="s2">&quot;from&quot;</span><span class="p">]}}</span>
@@ -570,7 +573,7 @@
<script type="text/javascript">
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../',
VERSION:'4.2.0k',
VERSION:'4.3.0',
LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
_modules/parsedmarc/splunk.html
+355
View File
@@ -0,0 +1,355 @@
<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>parsedmarc.splunk &mdash; parsedmarc 4.3.0 documentation</title>
<link rel="stylesheet" href="../../_static/css/theme.css" type="text/css" />
<link rel="stylesheet" href="../../_static/pygments.css" type="text/css" />
<link rel="index" title="Index" href="../../genindex.html" />
<link rel="search" title="Search" href="../../search.html" />
<script src="../../_static/js/modernizr.min.js"></script>
</head>
<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
<div class="wy-side-scroll">
<div class="wy-side-nav-search">
<a href="../../index.html" class="icon icon-home"> parsedmarc
</a>
<div class="version">
4.3.0
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
<input type="text" name="q" placeholder="Search docs" />
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
</form>
</div>
</div>
<div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
<!-- Local TOC -->
<div class="local-toc"></div>
</div>
</div>
</nav>
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
<nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../index.html">parsedmarc</a>
</nav>
<div class="wy-nav-content">
<div class="rst-content">
<div role="navigation" aria-label="breadcrumbs navigation">
<ul class="wy-breadcrumbs">
<li><a href="../../index.html">Docs</a> &raquo;</li>
<li><a href="../index.html">Module code</a> &raquo;</li>
<li><a href="../parsedmarc.html">parsedmarc</a> &raquo;</li>
<li>parsedmarc.splunk</li>
<li class="wy-breadcrumbs-aside">
</li>
</ul>
<hr/>
</div>
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
<div itemprop="articleBody">
<h1>Source code for parsedmarc.splunk</h1><div class="highlight"><pre>
<span></span><span class="kn">import</span> <span class="nn">logging</span>
<span class="kn">from</span> <span class="nn">urllib.parse</span> <span class="k">import</span> <span class="n">urlparse</span>
<span class="kn">import</span> <span class="nn">socket</span>
<span class="kn">import</span> <span class="nn">json</span>
<span class="kn">import</span> <span class="nn">urllib3</span>
<span class="kn">import</span> <span class="nn">requests</span>
<span class="kn">from</span> <span class="nn">parsedmarc.__version__</span> <span class="k">import</span> <span class="n">__version__</span>
<span class="kn">from</span> <span class="nn">parsedmarc.utils</span> <span class="k">import</span> <span class="n">human_timestamp_to_timestamp</span>
<span class="n">urllib3</span><span class="o">.</span><span class="n">disable_warnings</span><span class="p">(</span><span class="n">urllib3</span><span class="o">.</span><span class="n">exceptions</span><span class="o">.</span><span class="n">InsecureRequestWarning</span><span class="p">)</span>
<span class="n">logger</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s2">&quot;parsedmarc&quot;</span><span class="p">)</span>
<div class="viewcode-block" id="SplunkError"><a class="viewcode-back" href="../../index.html#parsedmarc.splunk.SplunkError">[docs]</a><span class="k">class</span> <span class="nc">SplunkError</span><span class="p">(</span><span class="ne">RuntimeError</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;Raised when a Splunk API error occurs&quot;&quot;&quot;</span></div>
<div class="viewcode-block" id="HECClient"><a class="viewcode-back" href="../../index.html#parsedmarc.splunk.HECClient">[docs]</a><span class="k">class</span> <span class="nc">HECClient</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;A client for a Splunk HTTP Events Collector (HEC)&quot;&quot;&quot;</span>
<span class="c1"># http://docs.splunk.com/Documentation/Splunk/latest/Data/AboutHEC</span>
<span class="c1"># http://docs.splunk.com/Documentation/Splunk/latest/RESTREF/RESTinput#services.2Fcollector</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">url</span><span class="p">,</span> <span class="n">access_token</span><span class="p">,</span> <span class="n">index</span><span class="p">,</span>
<span class="n">source</span><span class="o">=</span><span class="s2">&quot;parsedmarc&quot;</span><span class="p">,</span> <span class="n">verify</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Initializes the HECClient</span>
<span class="sd"> Args:</span>
<span class="sd"> url (str): The URL of the HEC</span>
<span class="sd"> access_token (str): The HEC access token</span>
<span class="sd"> index (str): The name of the index</span>
<span class="sd"> source (str): The source name</span>
<span class="sd"> verify (bool): Verify SSL certificates</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">url</span> <span class="o">=</span> <span class="n">urlparse</span><span class="p">(</span><span class="n">url</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">url</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">{0}</span><span class="s2">://</span><span class="si">{1}</span><span class="s2">/services/collector/event/1.0&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">url</span><span class="o">.</span><span class="n">scheme</span><span class="p">,</span>
<span class="n">url</span><span class="o">.</span><span class="n">netloc</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">access_token</span> <span class="o">=</span> <span class="n">access_token</span><span class="o">.</span><span class="n">lstrip</span><span class="p">(</span><span class="s2">&quot;Splunk &quot;</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">index</span>
<span class="bp">self</span><span class="o">.</span><span class="n">host</span> <span class="o">=</span> <span class="n">socket</span><span class="o">.</span><span class="n">getfqdn</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">source</span> <span class="o">=</span> <span class="n">source</span>
<span class="bp">self</span><span class="o">.</span><span class="n">session</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">Session</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">verify</span> <span class="o">=</span> <span class="n">verify</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_common_data</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="n">host</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">host</span><span class="p">,</span> <span class="n">source</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">source</span><span class="p">,</span>
<span class="n">index</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">index</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">headers</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;User-Agent&quot;</span><span class="p">:</span> <span class="s2">&quot;parsedmarc/</span><span class="si">{0}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">__version__</span><span class="p">),</span>
<span class="s2">&quot;Authorization&quot;</span><span class="p">:</span> <span class="s2">&quot;Splunk </span><span class="si">{0}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">access_token</span><span class="p">)</span>
<span class="p">}</span>
<div class="viewcode-block" id="HECClient.save_aggregate_reports_to_splunk"><a class="viewcode-back" href="../../index.html#parsedmarc.splunk.HECClient.save_aggregate_reports_to_splunk">[docs]</a> <span class="k">def</span> <span class="nf">save_aggregate_reports_to_splunk</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">aggregate_reports</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Saves aggregate DMARC reports to Splunk</span>
<span class="sd"> Args:</span>
<span class="sd"> aggregate_reports: A list of aggregate report dictionaries</span>
<span class="sd"> to save in Splunk</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;Saving aggregate reports to Splunk&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">aggregate_reports</span><span class="p">)</span> <span class="o">==</span> <span class="nb">dict</span><span class="p">:</span>
<span class="n">aggregate_reports</span> <span class="o">=</span> <span class="p">[</span><span class="n">aggregate_reports</span><span class="p">]</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">aggregate_reports</span><span class="p">)</span> <span class="o">&lt;</span> <span class="mi">1</span><span class="p">:</span>
<span class="k">return</span>
<span class="n">data</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_common_data</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="n">json_str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="k">for</span> <span class="n">report</span> <span class="ow">in</span> <span class="n">aggregate_reports</span><span class="p">:</span>
<span class="k">for</span> <span class="n">record</span> <span class="ow">in</span> <span class="n">report</span><span class="p">[</span><span class="s2">&quot;records&quot;</span><span class="p">]:</span>
<span class="n">new_report</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
<span class="k">for</span> <span class="n">metadata</span> <span class="ow">in</span> <span class="n">report</span><span class="p">[</span><span class="s2">&quot;report_metadata&quot;</span><span class="p">]:</span>
<span class="n">new_report</span><span class="p">[</span><span class="n">metadata</span><span class="p">]</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">&quot;report_metadata&quot;</span><span class="p">][</span><span class="n">metadata</span><span class="p">]</span>
<span class="n">new_report</span><span class="p">[</span><span class="s2">&quot;published_policy&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">&quot;policy_published&quot;</span><span class="p">]</span>
<span class="n">new_report</span><span class="p">[</span><span class="s2">&quot;source_ip_address&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;source&quot;</span><span class="p">][</span>
<span class="s2">&quot;ip_address&quot;</span><span class="p">]</span>
<span class="n">new_report</span><span class="p">[</span><span class="s2">&quot;source_country&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;source&quot;</span><span class="p">][</span><span class="s2">&quot;country&quot;</span><span class="p">]</span>
<span class="n">new_report</span><span class="p">[</span><span class="s2">&quot;source_reverse_dns&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;source&quot;</span><span class="p">][</span>
<span class="s2">&quot;reverse_dns&quot;</span><span class="p">]</span>
<span class="n">new_report</span><span class="p">[</span><span class="s2">&quot;source_base_domain&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;source&quot;</span><span class="p">][</span>
<span class="s2">&quot;base_domain&quot;</span><span class="p">]</span>
<span class="n">new_report</span><span class="p">[</span><span class="s2">&quot;message_count&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;count&quot;</span><span class="p">]</span>
<span class="n">new_report</span><span class="p">[</span><span class="s2">&quot;disposition&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;policy_evaluated&quot;</span><span class="p">][</span>
<span class="s2">&quot;disposition&quot;</span>
<span class="p">]</span>
<span class="n">new_report</span><span class="p">[</span><span class="s2">&quot;spf_aligned&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;alignment&quot;</span><span class="p">][</span><span class="s2">&quot;spf&quot;</span><span class="p">]</span>
<span class="n">new_report</span><span class="p">[</span><span class="s2">&quot;dkim_aligned&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;alignment&quot;</span><span class="p">][</span><span class="s2">&quot;dkim&quot;</span><span class="p">]</span>
<span class="n">new_report</span><span class="p">[</span><span class="s2">&quot;passed_dmarc&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;alignment&quot;</span><span class="p">][</span><span class="s2">&quot;dmarc&quot;</span><span class="p">]</span>
<span class="n">new_report</span><span class="p">[</span><span class="s2">&quot;header_from&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;identifiers&quot;</span><span class="p">][</span>
<span class="s2">&quot;header_from&quot;</span><span class="p">]</span>
<span class="n">new_report</span><span class="p">[</span><span class="s2">&quot;envelope_from&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;identifiers&quot;</span><span class="p">][</span>
<span class="s2">&quot;envelope_from&quot;</span><span class="p">]</span>
<span class="k">if</span> <span class="s2">&quot;dkim&quot;</span> <span class="ow">in</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;auth_results&quot;</span><span class="p">]:</span>
<span class="n">new_report</span><span class="p">[</span><span class="s2">&quot;dkim_results&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;auth_results&quot;</span><span class="p">][</span>
<span class="s2">&quot;dkim&quot;</span><span class="p">]</span>
<span class="k">if</span> <span class="s2">&quot;spf&quot;</span> <span class="ow">in</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;auth_results&quot;</span><span class="p">]:</span>
<span class="n">new_report</span><span class="p">[</span><span class="s2">&quot;spf_results&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;auth_results&quot;</span><span class="p">][</span>
<span class="s2">&quot;spf&quot;</span><span class="p">]</span>
<span class="n">data</span><span class="p">[</span><span class="s2">&quot;sourcetype&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;dmarc:aggregate&quot;</span>
<span class="n">timestamp</span> <span class="o">=</span> <span class="n">human_timestamp_to_timestamp</span><span class="p">(</span>
<span class="n">new_report</span><span class="p">[</span><span class="s2">&quot;begin_date&quot;</span><span class="p">])</span>
<span class="n">data</span><span class="p">[</span><span class="s2">&quot;time&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">timestamp</span>
<span class="n">data</span><span class="p">[</span><span class="s2">&quot;event&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">new_report</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="n">json_str</span> <span class="o">+=</span> <span class="s2">&quot;</span><span class="si">{0}</span><span class="se">\n</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">data</span><span class="p">))</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">post</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">url</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="n">json_str</span><span class="p">)</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">SplunkError</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
<span class="k">if</span> <span class="n">response</span><span class="p">[</span><span class="s2">&quot;code&quot;</span><span class="p">]</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">SplunkError</span><span class="p">(</span><span class="n">response</span><span class="p">[</span><span class="s2">&quot;text&quot;</span><span class="p">])</span></div>
<div class="viewcode-block" id="HECClient.save_forensic_reports_to_splunk"><a class="viewcode-back" href="../../index.html#parsedmarc.splunk.HECClient.save_forensic_reports_to_splunk">[docs]</a> <span class="k">def</span> <span class="nf">save_forensic_reports_to_splunk</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">forensic_reports</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Saves forensic DMARC reports to Splunk</span>
<span class="sd"> Args:</span>
<span class="sd"> forensic_reports (list): A list of forensic report dictionaries</span>
<span class="sd"> to save in Splunk</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;Saving forensic reports to Splunk&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">forensic_reports</span><span class="p">)</span> <span class="o">==</span> <span class="nb">dict</span><span class="p">:</span>
<span class="n">forensic_reports</span> <span class="o">=</span> <span class="p">[</span><span class="n">forensic_reports</span><span class="p">]</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">forensic_reports</span><span class="p">)</span> <span class="o">&lt;</span> <span class="mi">1</span><span class="p">:</span>
<span class="k">return</span>
<span class="n">json_str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="k">for</span> <span class="n">report</span> <span class="ow">in</span> <span class="n">forensic_reports</span><span class="p">:</span>
<span class="n">data</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_common_data</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="n">data</span><span class="p">[</span><span class="s2">&quot;sourcetype&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;dmarc:forensic&quot;</span>
<span class="n">timestamp</span> <span class="o">=</span> <span class="n">human_timestamp_to_timestamp</span><span class="p">(</span>
<span class="n">report</span><span class="p">[</span><span class="s2">&quot;arrival_date_utc&quot;</span><span class="p">])</span>
<span class="n">data</span><span class="p">[</span><span class="s2">&quot;time&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">timestamp</span>
<span class="n">data</span><span class="p">[</span><span class="s2">&quot;event&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">report</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="n">json_str</span> <span class="o">+=</span> <span class="s2">&quot;</span><span class="si">{0}</span><span class="se">\n</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">data</span><span class="p">))</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">post</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">url</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="n">json_str</span><span class="p">)</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">SplunkError</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
<span class="k">if</span> <span class="n">response</span><span class="p">[</span><span class="s2">&quot;code&quot;</span><span class="p">]</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">SplunkError</span><span class="p">(</span><span class="n">response</span><span class="p">[</span><span class="s2">&quot;text&quot;</span><span class="p">])</span></div></div>
</pre></div>
</div>
</div>
<footer>
<hr/>
<div role="contentinfo">
<p>
&copy; Copyright 2018, Sean Whalen
</p>
</div>
Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
</div>
</div>
</section>
</div>
<script type="text/javascript">
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../',
VERSION:'4.3.0',
LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
SOURCELINK_SUFFIX: '.txt'
};
</script>
<script type="text/javascript" src="../../_static/jquery.js"></script>
<script type="text/javascript" src="../../_static/underscore.js"></script>
<script type="text/javascript" src="../../_static/doctools.js"></script>
<script type="text/javascript" src="../../_static/js/theme.js"></script>
<script type="text/javascript">
jQuery(function () {
SphinxRtdTheme.Navigation.enable(true);
});
</script>
</body>
</html>
_modules/parsedmarc/utils.html
+662
View File
@@ -0,0 +1,662 @@
<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>parsedmarc.utils &mdash; parsedmarc 4.3.0 documentation</title>
<link rel="stylesheet" href="../../_static/css/theme.css" type="text/css" />
<link rel="stylesheet" href="../../_static/pygments.css" type="text/css" />
<link rel="index" title="Index" href="../../genindex.html" />
<link rel="search" title="Search" href="../../search.html" />
<script src="../../_static/js/modernizr.min.js"></script>
</head>
<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
<div class="wy-side-scroll">
<div class="wy-side-nav-search">
<a href="../../index.html" class="icon icon-home"> parsedmarc
</a>
<div class="version">
4.3.0
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
<input type="text" name="q" placeholder="Search docs" />
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
</form>
</div>
</div>
<div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
<!-- Local TOC -->
<div class="local-toc"></div>
</div>
</div>
</nav>
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
<nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../index.html">parsedmarc</a>
</nav>
<div class="wy-nav-content">
<div class="rst-content">
<div role="navigation" aria-label="breadcrumbs navigation">
<ul class="wy-breadcrumbs">
<li><a href="../../index.html">Docs</a> &raquo;</li>
<li><a href="../index.html">Module code</a> &raquo;</li>
<li><a href="../parsedmarc.html">parsedmarc</a> &raquo;</li>
<li>parsedmarc.utils</li>
<li class="wy-breadcrumbs-aside">
</li>
</ul>
<hr/>
</div>
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
<div itemprop="articleBody">
<h1>Source code for parsedmarc.utils</h1><div class="highlight"><pre>
<span></span><span class="sd">&quot;&quot;&quot;Utility functions that might be useful for other projects&quot;&quot;&quot;</span>
<span class="kn">import</span> <span class="nn">logging</span>
<span class="kn">import</span> <span class="nn">os</span>
<span class="kn">from</span> <span class="nn">datetime</span> <span class="k">import</span> <span class="n">datetime</span>
<span class="kn">from</span> <span class="nn">datetime</span> <span class="k">import</span> <span class="n">timedelta</span>
<span class="kn">from</span> <span class="nn">collections</span> <span class="k">import</span> <span class="n">OrderedDict</span>
<span class="kn">from</span> <span class="nn">io</span> <span class="k">import</span> <span class="n">BytesIO</span>
<span class="kn">import</span> <span class="nn">tarfile</span>
<span class="kn">import</span> <span class="nn">tempfile</span>
<span class="kn">import</span> <span class="nn">subprocess</span>
<span class="kn">import</span> <span class="nn">shutil</span>
<span class="kn">import</span> <span class="nn">mailparser</span>
<span class="kn">import</span> <span class="nn">json</span>
<span class="kn">import</span> <span class="nn">dateparser</span>
<span class="kn">import</span> <span class="nn">dns.reversename</span>
<span class="kn">import</span> <span class="nn">dns.resolver</span>
<span class="kn">import</span> <span class="nn">dns.exception</span>
<span class="kn">import</span> <span class="nn">geoip2.database</span>
<span class="kn">import</span> <span class="nn">geoip2.errors</span>
<span class="kn">import</span> <span class="nn">requests</span>
<span class="kn">import</span> <span class="nn">publicsuffix</span>
<span class="kn">from</span> <span class="nn">parsedmarc.__version__</span> <span class="k">import</span> <span class="n">USER_AGENT</span>
<span class="n">logger</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s2">&quot;parsedmarc&quot;</span><span class="p">)</span>
<div class="viewcode-block" id="EmailParserError"><a class="viewcode-back" href="../../index.html#parsedmarc.utils.EmailParserError">[docs]</a><span class="k">class</span> <span class="nc">EmailParserError</span><span class="p">(</span><span class="ne">RuntimeError</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;Raised when an error parsing the email occurs&quot;&quot;&quot;</span></div>
<div class="viewcode-block" id="get_base_domain"><a class="viewcode-back" href="../../index.html#parsedmarc.utils.get_base_domain">[docs]</a><span class="k">def</span> <span class="nf">get_base_domain</span><span class="p">(</span><span class="n">domain</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Gets the base domain name for the given domain</span>
<span class="sd"> .. note::</span>
<span class="sd"> Results are based on a list of public domain suffixes at</span>
<span class="sd"> https://publicsuffix.org/list/public_suffix_list.dat.</span>
<span class="sd"> This file is saved to the current working directory,</span>
<span class="sd"> where it is used as a cache file for 24 hours.</span>
<span class="sd"> Args:</span>
<span class="sd"> domain (str): A domain or subdomain</span>
<span class="sd"> Returns:</span>
<span class="sd"> str: The base domain of the given domain</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">psl_path</span> <span class="o">=</span> <span class="s2">&quot;.public_suffix_list.dat&quot;</span>
<span class="k">def</span> <span class="nf">download_psl</span><span class="p">():</span>
<span class="n">url</span> <span class="o">=</span> <span class="s2">&quot;https://publicsuffix.org/list/public_suffix_list.dat&quot;</span>
<span class="c1"># Use a browser-like user agent string to bypass some proxy blocks</span>
<span class="n">headers</span> <span class="o">=</span> <span class="p">{</span><span class="s2">&quot;User-Agent&quot;</span><span class="p">:</span> <span class="n">USER_AGENT</span><span class="p">}</span>
<span class="n">fresh_psl</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="n">headers</span><span class="o">=</span><span class="n">headers</span><span class="p">)</span><span class="o">.</span><span class="n">text</span>
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">psl_path</span><span class="p">,</span> <span class="s2">&quot;w&quot;</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s2">&quot;utf-8&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">fresh_psl_file</span><span class="p">:</span>
<span class="n">fresh_psl_file</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">fresh_psl</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">psl_path</span><span class="p">):</span>
<span class="n">download_psl</span><span class="p">()</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">psl_age</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span> <span class="o">-</span> <span class="n">datetime</span><span class="o">.</span><span class="n">fromtimestamp</span><span class="p">(</span>
<span class="n">os</span><span class="o">.</span><span class="n">stat</span><span class="p">(</span><span class="n">psl_path</span><span class="p">)</span><span class="o">.</span><span class="n">st_mtime</span><span class="p">)</span>
<span class="k">if</span> <span class="n">psl_age</span> <span class="o">&gt;</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">hours</span><span class="o">=</span><span class="mi">24</span><span class="p">):</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">download_psl</span><span class="p">()</span>
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
<span class="s2">&quot;Failed to download an updated PSL </span><span class="si">{0}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="p">))</span>
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">psl_path</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s2">&quot;utf-8&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">psl_file</span><span class="p">:</span>
<span class="n">psl</span> <span class="o">=</span> <span class="n">publicsuffix</span><span class="o">.</span><span class="n">PublicSuffixList</span><span class="p">(</span><span class="n">psl_file</span><span class="p">)</span>
<span class="k">return</span> <span class="n">psl</span><span class="o">.</span><span class="n">get_public_suffix</span><span class="p">(</span><span class="n">domain</span><span class="p">)</span></div>
<div class="viewcode-block" id="query_dns"><a class="viewcode-back" href="../../index.html#parsedmarc.utils.query_dns">[docs]</a><span class="k">def</span> <span class="nf">query_dns</span><span class="p">(</span><span class="n">domain</span><span class="p">,</span> <span class="n">record_type</span><span class="p">,</span> <span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mf">2.0</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Queries DNS</span>
<span class="sd"> Args:</span>
<span class="sd"> domain (str): The domain or subdomain to query about</span>
<span class="sd"> record_type (str): The record type to query for</span>
<span class="sd"> nameservers (list): A list of one or more nameservers to use</span>
<span class="sd"> (Cloudflare&#39;s public DNS resolvers by default)</span>
<span class="sd"> timeout (float): Sets the DNS timeout in seconds</span>
<span class="sd"> Returns:</span>
<span class="sd"> list: A list of answers</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">resolver</span> <span class="o">=</span> <span class="n">dns</span><span class="o">.</span><span class="n">resolver</span><span class="o">.</span><span class="n">Resolver</span><span class="p">()</span>
<span class="n">timeout</span> <span class="o">=</span> <span class="nb">float</span><span class="p">(</span><span class="n">timeout</span><span class="p">)</span>
<span class="k">if</span> <span class="n">nameservers</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">nameservers</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;1.1.1.1&quot;</span><span class="p">,</span> <span class="s2">&quot;1.0.0.1&quot;</span><span class="p">,</span>
<span class="s2">&quot;2606:4700:4700::1111&quot;</span><span class="p">,</span> <span class="s2">&quot;2606:4700:4700::1001&quot;</span><span class="p">,</span>
<span class="p">]</span>
<span class="n">resolver</span><span class="o">.</span><span class="n">nameservers</span> <span class="o">=</span> <span class="n">nameservers</span>
<span class="n">resolver</span><span class="o">.</span><span class="n">timeout</span> <span class="o">=</span> <span class="n">timeout</span>
<span class="n">resolver</span><span class="o">.</span><span class="n">lifetime</span> <span class="o">=</span> <span class="n">timeout</span>
<span class="k">if</span> <span class="n">record_type</span> <span class="o">==</span> <span class="s2">&quot;TXT&quot;</span><span class="p">:</span>
<span class="n">resource_records</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span>
<span class="k">lambda</span> <span class="n">r</span><span class="p">:</span> <span class="n">r</span><span class="o">.</span><span class="n">strings</span><span class="p">,</span>
<span class="n">resolver</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">domain</span><span class="p">,</span> <span class="n">record_type</span><span class="p">,</span> <span class="n">tcp</span><span class="o">=</span><span class="kc">True</span><span class="p">)))</span>
<span class="n">_resource_record</span> <span class="o">=</span> <span class="p">[</span>
<span class="n">resource_record</span><span class="p">[</span><span class="mi">0</span><span class="p">][:</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">resource_record</span><span class="p">)</span>
<span class="k">for</span> <span class="n">resource_record</span> <span class="ow">in</span> <span class="n">resource_records</span> <span class="k">if</span> <span class="n">resource_record</span><span class="p">]</span>
<span class="k">return</span> <span class="p">[</span><span class="n">r</span><span class="o">.</span><span class="n">decode</span><span class="p">()</span> <span class="k">for</span> <span class="n">r</span> <span class="ow">in</span> <span class="n">_resource_record</span><span class="p">]</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">return</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span>
<span class="k">lambda</span> <span class="n">r</span><span class="p">:</span> <span class="n">r</span><span class="o">.</span><span class="n">to_text</span><span class="p">()</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;&quot;&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s2">&quot;.&quot;</span><span class="p">),</span>
<span class="n">resolver</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">domain</span><span class="p">,</span> <span class="n">record_type</span><span class="p">,</span> <span class="n">tcp</span><span class="o">=</span><span class="kc">True</span><span class="p">)))</span></div>
<div class="viewcode-block" id="get_reverse_dns"><a class="viewcode-back" href="../../index.html#parsedmarc.utils.get_reverse_dns">[docs]</a><span class="k">def</span> <span class="nf">get_reverse_dns</span><span class="p">(</span><span class="n">ip_address</span><span class="p">,</span> <span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mf">2.0</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Resolves an IP address to a hostname using a reverse DNS query</span>
<span class="sd"> Args:</span>
<span class="sd"> ip_address (str): The IP address to resolve</span>
<span class="sd"> nameservers (list): A list of one or more nameservers to use</span>
<span class="sd"> (Cloudflare&#39;s public DNS resolvers by default)</span>
<span class="sd"> timeout (float): Sets the DNS query timeout in seconds</span>
<span class="sd"> Returns:</span>
<span class="sd"> str: The reverse DNS hostname (if any)</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">hostname</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">address</span> <span class="o">=</span> <span class="n">dns</span><span class="o">.</span><span class="n">reversename</span><span class="o">.</span><span class="n">from_address</span><span class="p">(</span><span class="n">ip_address</span><span class="p">)</span>
<span class="n">hostname</span> <span class="o">=</span> <span class="n">query_dns</span><span class="p">(</span><span class="n">address</span><span class="p">,</span> <span class="s2">&quot;PTR&quot;</span><span class="p">,</span>
<span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
<span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
<span class="k">except</span> <span class="n">dns</span><span class="o">.</span><span class="n">exception</span><span class="o">.</span><span class="n">DNSException</span><span class="p">:</span>
<span class="k">pass</span>
<span class="k">return</span> <span class="n">hostname</span></div>
<div class="viewcode-block" id="timestamp_to_datetime"><a class="viewcode-back" href="../../index.html#parsedmarc.utils.timestamp_to_datetime">[docs]</a><span class="k">def</span> <span class="nf">timestamp_to_datetime</span><span class="p">(</span><span class="n">timestamp</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Converts a UNIX/DMARC timestamp to a Python ``DateTime`` object</span>
<span class="sd"> Args:</span>
<span class="sd"> timestamp (int): The timestamp</span>
<span class="sd"> Returns:</span>
<span class="sd"> DateTime: The converted timestamp as a Python ``DateTime`` object</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">return</span> <span class="n">datetime</span><span class="o">.</span><span class="n">fromtimestamp</span><span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="n">timestamp</span><span class="p">))</span></div>
<div class="viewcode-block" id="timestamp_to_human"><a class="viewcode-back" href="../../index.html#parsedmarc.utils.timestamp_to_human">[docs]</a><span class="k">def</span> <span class="nf">timestamp_to_human</span><span class="p">(</span><span class="n">timestamp</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Converts a UNIX/DMARC timestamp to a human-readable string</span>
<span class="sd"> Args:</span>
<span class="sd"> timestamp: The timestamp</span>
<span class="sd"> Returns:</span>
<span class="sd"> str: The converted timestamp in ``YYYY-MM-DD HH:MM:SS`` format</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">return</span> <span class="n">timestamp_to_datetime</span><span class="p">(</span><span class="n">timestamp</span><span class="p">)</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%S&quot;</span><span class="p">)</span></div>
<div class="viewcode-block" id="human_timestamp_to_datetime"><a class="viewcode-back" href="../../index.html#parsedmarc.utils.human_timestamp_to_datetime">[docs]</a><span class="k">def</span> <span class="nf">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">human_timestamp</span><span class="p">,</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Converts a human-readable timestamp into a Python ``DateTime`` object</span>
<span class="sd"> Args:</span>
<span class="sd"> human_timestamp (str): A timestamp string</span>
<span class="sd"> to_utc (bool): Convert the timestamp to UTC</span>
<span class="sd"> Returns:</span>
<span class="sd"> DateTime: The converted timestamp</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">settings</span> <span class="o">=</span> <span class="p">{}</span>
<span class="k">if</span> <span class="n">to_utc</span><span class="p">:</span>
<span class="n">settings</span> <span class="o">=</span> <span class="p">{</span><span class="s2">&quot;TO_TIMEZONE&quot;</span><span class="p">:</span> <span class="s2">&quot;UTC&quot;</span><span class="p">}</span>
<span class="k">return</span> <span class="n">dateparser</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">human_timestamp</span><span class="p">,</span> <span class="n">settings</span><span class="o">=</span><span class="n">settings</span><span class="p">)</span></div>
<div class="viewcode-block" id="human_timestamp_to_timestamp"><a class="viewcode-back" href="../../index.html#parsedmarc.utils.human_timestamp_to_timestamp">[docs]</a><span class="k">def</span> <span class="nf">human_timestamp_to_timestamp</span><span class="p">(</span><span class="n">human_timestamp</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Converts a human-readable timestamp into a into a UNIX timestamp</span>
<span class="sd"> Args:</span>
<span class="sd"> human_timestamp (str): A timestamp in `YYYY-MM-DD HH:MM:SS`` format</span>
<span class="sd"> Returns:</span>
<span class="sd"> float: The converted timestamp</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">human_timestamp</span> <span class="o">=</span> <span class="n">human_timestamp</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;T&quot;</span><span class="p">,</span> <span class="s2">&quot; &quot;</span><span class="p">)</span>
<span class="k">return</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">human_timestamp</span><span class="p">)</span><span class="o">.</span><span class="n">timestamp</span><span class="p">()</span></div>
<div class="viewcode-block" id="get_ip_address_country"><a class="viewcode-back" href="../../index.html#parsedmarc.utils.get_ip_address_country">[docs]</a><span class="k">def</span> <span class="nf">get_ip_address_country</span><span class="p">(</span><span class="n">ip_address</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Uses the MaxMind Geolite2 Country database to return the ISO code for the</span>
<span class="sd"> country associated with the given IPv4 or IPv6 address</span>
<span class="sd"> Args:</span>
<span class="sd"> ip_address (str): The IP address to query for</span>
<span class="sd"> Returns:</span>
<span class="sd"> str: And ISO country code associated with the given IP address</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">db_filename</span> <span class="o">=</span> <span class="s2">&quot;.GeoLite2-Country.mmdb&quot;</span>
<span class="k">def</span> <span class="nf">download_country_database</span><span class="p">(</span><span class="n">location</span><span class="o">=</span><span class="s2">&quot;.GeoLite2-Country.mmdb&quot;</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;Downloads the MaxMind Geolite2 Country database</span>
<span class="sd"> Args:</span>
<span class="sd"> location (str): Local location for the database file</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">url</span> <span class="o">=</span> <span class="s2">&quot;https://geolite.maxmind.com/download/geoip/database/&quot;</span> \
<span class="s2">&quot;GeoLite2-Country.tar.gz&quot;</span>
<span class="c1"># Use a browser-like user agent string to bypass some proxy blocks</span>
<span class="n">headers</span> <span class="o">=</span> <span class="p">{</span><span class="s2">&quot;User-Agent&quot;</span><span class="p">:</span> <span class="n">USER_AGENT</span><span class="p">}</span>
<span class="n">original_filename</span> <span class="o">=</span> <span class="s2">&quot;GeoLite2-Country.mmdb&quot;</span>
<span class="n">tar_bytes</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="n">headers</span><span class="o">=</span><span class="n">headers</span><span class="p">)</span><span class="o">.</span><span class="n">content</span>
<span class="n">tar_file</span> <span class="o">=</span> <span class="n">tarfile</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">fileobj</span><span class="o">=</span><span class="n">BytesIO</span><span class="p">(</span><span class="n">tar_bytes</span><span class="p">),</span> <span class="n">mode</span><span class="o">=</span><span class="s2">&quot;r:gz&quot;</span><span class="p">)</span>
<span class="n">tar_dir</span> <span class="o">=</span> <span class="n">tar_file</span><span class="o">.</span><span class="n">getnames</span><span class="p">()[</span><span class="mi">0</span><span class="p">]</span>
<span class="n">tar_path</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">{0}</span><span class="s2">/</span><span class="si">{1}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">tar_dir</span><span class="p">,</span> <span class="n">original_filename</span><span class="p">)</span>
<span class="n">tar_file</span><span class="o">.</span><span class="n">extract</span><span class="p">(</span><span class="n">tar_path</span><span class="p">)</span>
<span class="n">shutil</span><span class="o">.</span><span class="n">move</span><span class="p">(</span><span class="n">tar_path</span><span class="p">,</span> <span class="n">location</span><span class="p">)</span>
<span class="n">shutil</span><span class="o">.</span><span class="n">rmtree</span><span class="p">(</span><span class="n">tar_dir</span><span class="p">)</span>
<span class="n">system_paths</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;/usr/local/share/GeoIP/GeoLite2-Country.mmdb&quot;</span><span class="p">,</span>
<span class="s2">&quot;/usr/share/GeoIP/GeoLite2-Country.mmdb&quot;</span><span class="p">]</span>
<span class="n">db_path</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="k">for</span> <span class="n">system_path</span> <span class="ow">in</span> <span class="n">system_paths</span><span class="p">:</span>
<span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">system_path</span><span class="p">):</span>
<span class="n">db_path</span> <span class="o">=</span> <span class="n">system_path</span>
<span class="k">break</span>
<span class="k">if</span> <span class="n">db_path</span> <span class="o">==</span> <span class="s2">&quot;&quot;</span><span class="p">:</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">db_filename</span><span class="p">):</span>
<span class="n">download_country_database</span><span class="p">(</span><span class="n">db_filename</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">db_age</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span> <span class="o">-</span> <span class="n">datetime</span><span class="o">.</span><span class="n">fromtimestamp</span><span class="p">(</span>
<span class="n">os</span><span class="o">.</span><span class="n">stat</span><span class="p">(</span><span class="n">db_filename</span><span class="p">)</span><span class="o">.</span><span class="n">st_mtime</span><span class="p">)</span>
<span class="k">if</span> <span class="n">db_age</span> <span class="o">&gt;</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="mi">60</span><span class="p">):</span>
<span class="n">download_country_database</span><span class="p">()</span>
<span class="n">db_path</span> <span class="o">=</span> <span class="n">db_filename</span>
<span class="n">db_reader</span> <span class="o">=</span> <span class="n">geoip2</span><span class="o">.</span><span class="n">database</span><span class="o">.</span><span class="n">Reader</span><span class="p">(</span><span class="n">db_path</span><span class="p">)</span>
<span class="n">country</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">country</span> <span class="o">=</span> <span class="n">db_reader</span><span class="o">.</span><span class="n">country</span><span class="p">(</span><span class="n">ip_address</span><span class="p">)</span><span class="o">.</span><span class="n">country</span><span class="o">.</span><span class="n">iso_code</span>
<span class="k">except</span> <span class="n">geoip2</span><span class="o">.</span><span class="n">errors</span><span class="o">.</span><span class="n">AddressNotFoundError</span><span class="p">:</span>
<span class="k">pass</span>
<span class="k">return</span> <span class="n">country</span></div>
<div class="viewcode-block" id="get_ip_address_info"><a class="viewcode-back" href="../../index.html#parsedmarc.utils.get_ip_address_info">[docs]</a><span class="k">def</span> <span class="nf">get_ip_address_info</span><span class="p">(</span><span class="n">ip_address</span><span class="p">,</span> <span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mf">2.0</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Returns reverse DNS and country information for the given IP address</span>
<span class="sd"> Args:</span>
<span class="sd"> ip_address (str): The IP address to check</span>
<span class="sd"> nameservers (list): A list of one or more nameservers to use</span>
<span class="sd"> (Cloudflare&#39;s public DNS resolvers by default)</span>
<span class="sd"> timeout (float): Sets the DNS timeout in seconds</span>
<span class="sd"> Returns:</span>
<span class="sd"> OrderedDict: ``ip_address``, ``reverse_dns``</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">ip_address</span> <span class="o">=</span> <span class="n">ip_address</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
<span class="n">info</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">()</span>
<span class="n">info</span><span class="p">[</span><span class="s2">&quot;ip_address&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">ip_address</span>
<span class="n">reverse_dns</span> <span class="o">=</span> <span class="n">get_reverse_dns</span><span class="p">(</span><span class="n">ip_address</span><span class="p">,</span>
<span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
<span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">)</span>
<span class="n">country</span> <span class="o">=</span> <span class="n">get_ip_address_country</span><span class="p">(</span><span class="n">ip_address</span><span class="p">)</span>
<span class="n">info</span><span class="p">[</span><span class="s2">&quot;country&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">country</span>
<span class="n">info</span><span class="p">[</span><span class="s2">&quot;reverse_dns&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">reverse_dns</span>
<span class="n">info</span><span class="p">[</span><span class="s2">&quot;base_domain&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">if</span> <span class="n">reverse_dns</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">base_domain</span> <span class="o">=</span> <span class="n">get_base_domain</span><span class="p">(</span><span class="n">reverse_dns</span><span class="p">)</span>
<span class="n">info</span><span class="p">[</span><span class="s2">&quot;base_domain&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">base_domain</span>
<span class="k">return</span> <span class="n">info</span></div>
<span class="k">def</span> <span class="nf">parse_email_address</span><span class="p">(</span><span class="n">original_address</span><span class="p">):</span>
<span class="k">if</span> <span class="n">original_address</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="s2">&quot;&quot;</span><span class="p">:</span>
<span class="n">display_name</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">display_name</span> <span class="o">=</span> <span class="n">original_address</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
<span class="n">address</span> <span class="o">=</span> <span class="n">original_address</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
<span class="n">address_parts</span> <span class="o">=</span> <span class="n">address</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">&quot;@&quot;</span><span class="p">)</span>
<span class="n">local</span> <span class="o">=</span> <span class="kc">None</span>
<span class="n">domain</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">address_parts</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">1</span><span class="p">:</span>
<span class="n">local</span> <span class="o">=</span> <span class="n">address_parts</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
<span class="n">domain</span> <span class="o">=</span> <span class="n">address_parts</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
<span class="k">return</span> <span class="n">OrderedDict</span><span class="p">([(</span><span class="s2">&quot;display_name&quot;</span><span class="p">,</span> <span class="n">display_name</span><span class="p">),</span>
<span class="p">(</span><span class="s2">&quot;address&quot;</span><span class="p">,</span> <span class="n">address</span><span class="p">),</span>
<span class="p">(</span><span class="s2">&quot;local&quot;</span><span class="p">,</span> <span class="n">local</span><span class="p">),</span>
<span class="p">(</span><span class="s2">&quot;domain&quot;</span><span class="p">,</span> <span class="n">domain</span><span class="p">)])</span>
<div class="viewcode-block" id="get_filename_safe_string"><a class="viewcode-back" href="../../index.html#parsedmarc.utils.get_filename_safe_string">[docs]</a><span class="k">def</span> <span class="nf">get_filename_safe_string</span><span class="p">(</span><span class="n">string</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Converts a string to a string that is safe for a filename</span>
<span class="sd"> Args:</span>
<span class="sd"> string (str): A string to make safe for a filename</span>
<span class="sd"> Returns:</span>
<span class="sd"> str: A string safe for a filename</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">invalid_filename_chars</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;</span><span class="se">\\</span><span class="s1">&#39;</span><span class="p">,</span> <span class="s1">&#39;/&#39;</span><span class="p">,</span> <span class="s1">&#39;:&#39;</span><span class="p">,</span> <span class="s1">&#39;&quot;&#39;</span><span class="p">,</span> <span class="s1">&#39;*&#39;</span><span class="p">,</span> <span class="s1">&#39;?&#39;</span><span class="p">,</span> <span class="s1">&#39;|&#39;</span><span class="p">,</span> <span class="s1">&#39;</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">,</span>
<span class="s1">&#39;</span><span class="se">\r</span><span class="s1">&#39;</span><span class="p">]</span>
<span class="k">if</span> <span class="n">string</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">string</span> <span class="o">=</span> <span class="s2">&quot;None&quot;</span>
<span class="k">for</span> <span class="n">char</span> <span class="ow">in</span> <span class="n">invalid_filename_chars</span><span class="p">:</span>
<span class="n">string</span> <span class="o">=</span> <span class="n">string</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="n">char</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">)</span>
<span class="n">string</span> <span class="o">=</span> <span class="n">string</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s2">&quot;.&quot;</span><span class="p">)</span>
<span class="k">return</span> <span class="n">string</span></div>
<div class="viewcode-block" id="is_outlook_msg"><a class="viewcode-back" href="../../index.html#parsedmarc.utils.is_outlook_msg">[docs]</a><span class="k">def</span> <span class="nf">is_outlook_msg</span><span class="p">(</span><span class="n">content</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Checks if the given content is a Outlook msg OLE file</span>
<span class="sd"> Args:</span>
<span class="sd"> content: Content to check</span>
<span class="sd"> Returns:</span>
<span class="sd"> bool: A flag the indicates if a file is a Outlook MSG file</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">return</span> <span class="nb">type</span><span class="p">(</span><span class="n">content</span><span class="p">)</span> <span class="o">==</span> <span class="nb">bytes</span> <span class="ow">and</span> <span class="n">content</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span>
<span class="sa">b</span><span class="s2">&quot;</span><span class="se">\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1</span><span class="s2">&quot;</span><span class="p">)</span></div>
<div class="viewcode-block" id="convert_outlook_msg"><a class="viewcode-back" href="../../index.html#parsedmarc.utils.convert_outlook_msg">[docs]</a><span class="k">def</span> <span class="nf">convert_outlook_msg</span><span class="p">(</span><span class="n">msg_bytes</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Uses the ``msgconvert`` Perl utility to convert an Outlook MS file to</span>
<span class="sd"> standard RFC 822 format</span>
<span class="sd"> Args:</span>
<span class="sd"> msg_bytes (bytes): the content of the .msg file</span>
<span class="sd"> Returns:</span>
<span class="sd"> A RFC 822 string</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">is_outlook_msg</span><span class="p">(</span><span class="n">msg_bytes</span><span class="p">):</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;The supplied bytes are not an Outlook MSG file&quot;</span><span class="p">)</span>
<span class="n">orig_dir</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">getcwd</span><span class="p">()</span>
<span class="n">tmp_dir</span> <span class="o">=</span> <span class="n">tempfile</span><span class="o">.</span><span class="n">mkdtemp</span><span class="p">()</span>
<span class="n">os</span><span class="o">.</span><span class="n">chdir</span><span class="p">(</span><span class="n">tmp_dir</span><span class="p">)</span>
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="s2">&quot;sample.msg&quot;</span><span class="p">,</span> <span class="s2">&quot;wb&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">msg_file</span><span class="p">:</span>
<span class="n">msg_file</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">msg_bytes</span><span class="p">)</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">subprocess</span><span class="o">.</span><span class="n">check_call</span><span class="p">([</span><span class="s2">&quot;msgconvert&quot;</span><span class="p">,</span> <span class="s2">&quot;sample.msg&quot;</span><span class="p">])</span>
<span class="n">eml_path</span> <span class="o">=</span> <span class="s2">&quot;sample.eml&quot;</span>
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">eml_path</span><span class="p">,</span> <span class="s2">&quot;rb&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">eml_file</span><span class="p">:</span>
<span class="n">rfc822</span> <span class="o">=</span> <span class="n">eml_file</span><span class="o">.</span><span class="n">read</span><span class="p">()</span>
<span class="k">except</span> <span class="ne">FileNotFoundError</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">EmailParserError</span><span class="p">(</span>
<span class="s2">&quot;Failed to convert Outlook MSG: msgconvert utility not found&quot;</span><span class="p">)</span>
<span class="k">finally</span><span class="p">:</span>
<span class="n">os</span><span class="o">.</span><span class="n">chdir</span><span class="p">(</span><span class="n">orig_dir</span><span class="p">)</span>
<span class="n">shutil</span><span class="o">.</span><span class="n">rmtree</span><span class="p">(</span><span class="n">tmp_dir</span><span class="p">)</span>
<span class="k">return</span> <span class="n">rfc822</span></div>
<div class="viewcode-block" id="parse_email"><a class="viewcode-back" href="../../index.html#parsedmarc.utils.parse_email">[docs]</a><span class="k">def</span> <span class="nf">parse_email</span><span class="p">(</span><span class="n">data</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> A simplified email parser</span>
<span class="sd"> Args:</span>
<span class="sd"> data: The RFC 822 message string, or MSG binary</span>
<span class="sd"> Returns (dict): Parsed email data</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">data</span><span class="p">)</span> <span class="o">==</span> <span class="nb">bytes</span><span class="p">:</span>
<span class="k">if</span> <span class="n">is_outlook_msg</span><span class="p">(</span><span class="n">data</span><span class="p">):</span>
<span class="n">data</span> <span class="o">=</span> <span class="n">convert_outlook_msg</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
<span class="n">data</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s2">&quot;utf-8&quot;</span><span class="p">,</span> <span class="n">errors</span><span class="o">=</span><span class="s2">&quot;replace&quot;</span><span class="p">)</span>
<span class="n">parsed_email</span> <span class="o">=</span> <span class="n">mailparser</span><span class="o">.</span><span class="n">parse_from_string</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
<span class="n">headers</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">parsed_email</span><span class="o">.</span><span class="n">headers_json</span><span class="p">)</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="n">parsed_email</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">parsed_email</span><span class="o">.</span><span class="n">mail_json</span><span class="p">)</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;headers&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">headers</span>
<span class="k">if</span> <span class="s2">&quot;received&quot;</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">:</span>
<span class="k">for</span> <span class="n">received</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;received&quot;</span><span class="p">]:</span>
<span class="k">if</span> <span class="s2">&quot;date_utc&quot;</span> <span class="ow">in</span> <span class="n">received</span><span class="p">:</span>
<span class="n">received</span><span class="p">[</span><span class="s2">&quot;date_utc&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">received</span><span class="p">[</span><span class="s2">&quot;date_utc&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;T&quot;</span><span class="p">,</span>
<span class="s2">&quot; &quot;</span><span class="p">)</span>
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;from&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">parse_email_address</span><span class="p">(</span><span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;from&quot;</span><span class="p">][</span><span class="mi">0</span><span class="p">])</span>
<span class="k">if</span> <span class="s2">&quot;date&quot;</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">:</span>
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;date&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;date&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;T&quot;</span><span class="p">,</span> <span class="s2">&quot; &quot;</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;date&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">if</span> <span class="s2">&quot;reply_to&quot;</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">:</span>
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;reply_to&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">parse_email_address</span><span class="p">(</span><span class="n">x</span><span class="p">),</span>
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;reply_to&quot;</span><span class="p">]))</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;reply_to&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">if</span> <span class="s2">&quot;to&quot;</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">:</span>
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;to&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">parse_email_address</span><span class="p">(</span><span class="n">x</span><span class="p">),</span>
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;to&quot;</span><span class="p">]))</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;to&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">if</span> <span class="s2">&quot;cc&quot;</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">:</span>
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;cc&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">parse_email_address</span><span class="p">(</span><span class="n">x</span><span class="p">),</span>
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;cc&quot;</span><span class="p">]))</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;cc&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">if</span> <span class="s2">&quot;bcc&quot;</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">:</span>
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;bcc&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">parse_email_address</span><span class="p">(</span><span class="n">x</span><span class="p">),</span>
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;bcc&quot;</span><span class="p">]))</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;bcc&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">if</span> <span class="s2">&quot;delivered_to&quot;</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">:</span>
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;delivered_to&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span>
<span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">parse_email_address</span><span class="p">(</span><span class="n">x</span><span class="p">),</span>
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;delivered_to&quot;</span><span class="p">])</span>
<span class="p">)</span>
<span class="k">if</span> <span class="s2">&quot;attachments&quot;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">:</span>
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;attachments&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">if</span> <span class="s2">&quot;subject&quot;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">:</span>
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;subject&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;filename_safe_subject&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">get_filename_safe_string</span><span class="p">(</span>
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;subject&quot;</span><span class="p">])</span>
<span class="k">if</span> <span class="s2">&quot;body&quot;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">:</span>
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;body&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">return</span> <span class="n">parsed_email</span></div>
</pre></div>
</div>
</div>
<footer>
<hr/>
<div role="contentinfo">
<p>
&copy; Copyright 2018, Sean Whalen
</p>
</div>
Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
</div>
</div>
</section>
</div>
<script type="text/javascript">
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../',
VERSION:'4.3.0',
LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
SOURCELINK_SUFFIX: '.txt'
};
</script>
<script type="text/javascript" src="../../_static/jquery.js"></script>
<script type="text/javascript" src="../../_static/underscore.js"></script>
<script type="text/javascript" src="../../_static/doctools.js"></script>
<script type="text/javascript" src="../../_static/js/theme.js"></script>
<script type="text/javascript">
jQuery(function () {
SphinxRtdTheme.Navigation.enable(true);
});
</script>
</body>
</html>
_sources/index.rst.txt
+158 -28
View File
@@ -37,8 +37,24 @@ Features
Resources
=========
DMARC guides
------------
* `Demystifying DMARC`_ - A complete guide to SPF, DKIM, and DMARC
SPF and DMARC record validation
-------------------------------
If you are looking for SPF and DMARC record validation and parsing,
check out the sister project,
`checkdmarc <https://domainaware.github.io/checkdmarc/>`_.
Lookalike domains
-----------------
DMARC protects against domain spoofing, not lookalike domains. for open source
lookalike domain monitoring, check out `DomainAware <https://github.com/seanthegeek/domainaware>`_.
CLI help
========
@@ -53,9 +69,13 @@ CLI help
[--elasticsearch-index-prefix ELASTICSEARCH_INDEX_PREFIX]
[--elasticsearch-index-suffix ELASTICSEARCH_INDEX_SUFFIX]
[--hec HEC] [--hec-token HEC_TOKEN] [--hec-index HEC_INDEX]
[--hec-skip-certificate-verification] [--save-aggregate]
[--save-forensic] [-O OUTGOING_HOST] [-U OUTGOING_USER]
[-P OUTGOING_PASSWORD] [--outgoing-port OUTGOING_PORT]
[--hec-skip-certificate-verification]
[-K [KAFKA_HOSTS [KAFKA_HOSTS ...]]]
[--kafka-aggregate-topic KAFKA_AGGREGATE_TOPIC]
[--kafka-forensic_topic KAFKA_FORENSIC_TOPIC]
[--save-aggregate] [--save-forensic] [-O OUTGOING_HOST]
[-U OUTGOING_USER] [-P OUTGOING_PASSWORD]
[--outgoing-port OUTGOING_PORT]
[--outgoing-ssl OUTGOING_SSL] [-F OUTGOING_FROM]
[-T OUTGOING_TO [OUTGOING_TO ...]] [-S OUTGOING_SUBJECT]
[-A OUTGOING_ATTACHMENT] [-M OUTGOING_MESSAGE] [-w] [--test]
@@ -73,10 +93,11 @@ CLI help
-o OUTPUT, --output OUTPUT
Write output files to the given directory
-n NAMESERVERS [NAMESERVERS ...], --nameservers NAMESERVERS [NAMESERVERS ...]
nameservers to query (Default is Cloudflare's)
nameservers to query (Default is Cloudflare's
nameservers)
-t TIMEOUT, --timeout TIMEOUT
number of seconds to wait for an answer from DNS
(default 2.0)
(Default: 2.0)
-H HOST, --host HOST IMAP hostname or IP address
-u USER, --user USER IMAP user
-p PASSWORD, --password PASSWORD
@@ -85,14 +106,15 @@ CLI help
IMAP port
--imap-no-ssl Do not use SSL/TLS when connecting to IMAP
-r REPORTS_FOLDER, --reports-folder REPORTS_FOLDER
The IMAP folder containing the reports Default: INBOX
The IMAP folder containing the reports (Default:
INBOX)
-a ARCHIVE_FOLDER, --archive-folder ARCHIVE_FOLDER
Specifies the IMAP folder to move messages to after
processing them Default: Archive
processing them (Default: Archive)
-d, --delete Delete the reports after processing them
-E [ELASTICSEARCH_HOST [ELASTICSEARCH_HOST ...]], --elasticsearch-host [ELASTICSEARCH_HOST [ELASTICSEARCH_HOST ...]]
A list of one or more Elasticsearch hostnames or URLs
to use (e.g. localhost:9200)
One or more Elasticsearch hostnames or URLs to use
(e.g. localhost:9200)
--elasticsearch-index-prefix ELASTICSEARCH_INDEX_PREFIX
Prefix to add in front of the dmarc_aggregate and
dmarc_forensic Elasticsearch index names, joined by _
@@ -108,6 +130,14 @@ CLI help
HTTP Event Collector (HEC)
--hec-skip-certificate-verification
Skip certificate verification for Splunk HEC
-K [KAFKA_HOSTS [KAFKA_HOSTS ...]], --kafka-hosts [KAFKA_HOSTS [KAFKA_HOSTS ...]]
A list of one or more Kafka hostnames or URLs
--kafka-aggregate-topic KAFKA_AGGREGATE_TOPIC
The Kafka topic to publish aggregate reports to
(Default: dmarc_aggregate)
--kafka-forensic_topic KAFKA_FORENSIC_TOPIC
The Kafka topic to publish forensic reports to
(Default: dmarc_forensic)
--save-aggregate Save aggregate reports to search indexes
--save-forensic Save forensic reports to search indexes
-O OUTGOING_HOST, --outgoing-host OUTGOING_HOST
@@ -138,20 +168,6 @@ CLI help
--debug Print debugging information
-v, --version show program's version number and exit
SPF and DMARC record validation
===============================
If you are looking for SPF and DMARC record validation and parsing,
check out the sister project,
`checkdmarc <https://domainaware.github.io/checkdmarc/>`_.
SPF and DMARC record validation
===============================
If you are looking for SPF and DMARC record validation and parsing,
check out the sister project,
`checkdmarc <https://domainaware.github.io/checkdmarc/>`_.
Sample aggregate report output
==============================
@@ -242,12 +258,114 @@ CSV
xml_schema,org_name,org_email,org_extra_contact_info,report_id,begin_date,end_date,errors,domain,adkim,aspf,p,sp,pct,fo,source_ip_address,source_country,source_reverse_dns,source_base_domain,count,disposition,dkim_alignment,spf_alignment,policy_override_reasons,policy_override_comments,envelope_from,header_from,envelope_to,dkim_domains,dkim_selectors,dkim_results,spf_domains,spf_scopes,spf_results
draft,acme.com,noreply-dmarc-support@acme.com,http://acme.com/dmarc/support,9391651994964116463,2012-04-27 20:00:00,2012-04-28 19:59:59,,example.com,r,r,none,none,100,0,72.150.241.94,US,adsl-72-150-241-94.shv.bellsouth.net,bellsouth.net,2,none,fail,pass,,,example.com,example.com,,example.com,none,fail,example.com,mfrom,pass
Sample forensic report output
=============================
I don't have a sample I can share for privacy reasons. If you have a sample
forensic report that you can share publicly, please contact me!
Thanks to Github user `xennn <https://github.com/xennn>`_ for the anonymized
`forensic report email sample
<https://github.com/domainaware/parsedmarc/raw/master/samples/forensic/DMARC%20Failure%20Report%20for%20domain.de%20(mail-from%3Dsharepoint%40domain.de%2C%20ip%3D10.10.10.10).eml>`_.
JSON
----
.. code-block:: json
{
"feedback_type": "auth-failure",
"user_agent": "Lua/1.0",
"version": "1.0",
"original_mail_from": "sharepoint@domain.de",
"original_rcpt_to": "peter.pan@domain.de",
"arrival_date": "Mon, 01 Oct 2018 11:20:27 +0200",
"message_id": "<38.E7.30937.BD6E1BB5@ mailrelay.de>",
"authentication_results": "dmarc=fail (p=none, dis=none) header.from=domain.de",
"delivery_result": "smg-policy-action",
"auth_failure": [
"dmarc"
],
"reported_domain": "domain.de",
"arrival_date_utc": "2018-10-01 09:20:27",
"source": {
"ip_address": "10.10.10.10",
"country": null,
"reverse_dns": null,
"base_domain": null
},
"authentication_mechanisms": [],
"original_envelope_id": null,
"dkim_domain": null,
"sample_headers_only": false,
"sample": "Content-Type: message/rfc822\nContent-Disposition: inline\nReceived: from Servernameone.domain.local (Servernameone.domain.local [10.10.10.10])\n by mailrelay.de (mail.DOMAIN.de) with SMTP id 38.E7.30937.BD6E1BB5; Mon, 1 Oct 2018 11:20:27 +0200 (CEST)\nDate: 01 Oct 2018 11:20:27 +0200\nMessage-ID: <38.E7.30937.BD6E1BB5@ mailrelay.de>\nTo: <peter.pan@domain.de>\nfrom: \"=?utf-8?B?SW50ZXJha3RpdmUgV2V0dGJld2VyYmVyLcOcYmVyc2ljaHQ=?=\" <sharepoint@domain.de>\nSubject: Subject\nMIME-Version: 1.0\nX-Mailer: Microsoft SharePoint Foundation 2010\nContent-Type: text/html; charset=utf-8\nContent-Transfer-Encoding: quoted-printable\n\n\n<html><head><base href=3D'\nwettbewerb' /></head><body><!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 3.2//EN\"=\n><HTML><HEAD><META NAME=3D\"Generator\" CONTENT=3D\"MS Exchange Server version=\n 08.01.0240.003\"></html>\n ",
"parsed_sample": {
"content-transfer-encoding": "quoted-printable",
"x-mailer": "Microsoft SharePoint Foundation 2010",
"message-id": "<38.E7.30937.BD6E1BB5@ mailrelay.de>",
"body": "<html><head><base href=3D'\nwettbewerb' /></head><body><!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 3.2//EN\"=\n><HTML><HEAD><META NAME=3D\"Generator\" CONTENT=3D\"MS Exchange Server version=\n 08.01.0240.003\"></html>",
"to": [
{
"display_name": null,
"address": "peter.pan@domain.de",
"local": "peter.pan",
"domain": "domain.de"
}
],
"date": "2018-10-01 09:20:27",
"to_domains": [
"domain.de"
],
"received": [
{
"from": "Servernameone.domain.local Servernameone.domain.local 10.10.10.10",
"by": "mailrelay.de mail.DOMAIN.de",
"with": "SMTP id 38.E7.30937.BD6E1BB5",
"date": "Mon, 1 Oct 2018 11:20:27 +0200 CEST",
"hop": 1,
"date_utc": "2018-10-01 09:20:27",
"delay": 0
}
],
"content-disposition": "inline",
"mime-version": "1.0",
"subject": "Subject",
"timezone": "+2",
"from": {
"display_name": "Interaktive Wettbewerber-Übersicht",
"address": "sharepoint@domain.de",
"local": "sharepoint",
"domain": "domain.de"
},
"content-type": "message/rfc822",
"has_defects": false,
"headers": {
"Content-Type": "text/html; charset=utf-8",
"Content-Disposition": "inline",
"Received": "from Servernameone.domain.local (Servernameone.domain.local [10.10.10.10])\n by mailrelay.de (mail.DOMAIN.de) with SMTP id 38.E7.30937.BD6E1BB5; Mon, 1 Oct 2018 11:20:27 +0200 (CEST)",
"Date": "01 Oct 2018 11:20:27 +0200",
"Message-ID": "<38.E7.30937.BD6E1BB5@ mailrelay.de>",
"To": "<peter.pan@domain.de>",
"from": "\"Interaktive Wettbewerber-Übersicht\" <sharepoint@domain.de>",
"Subject": "Subject",
"MIME-Version": "1.0",
"X-Mailer": "Microsoft SharePoint Foundation 2010",
"Content-Transfer-Encoding": "quoted-printable"
},
"reply_to": [],
"cc": [],
"bcc": [],
"attachments": [],
"filename_safe_subject": "Subject"
}
}
CSV
---
::
feedback_type,user_agent,version,original_envelope_id,original_mail_from,original_rcpt_to,arrival_date,arrival_date_utc,subject,message_id,authentication_results,dkim_domain,source_ip_address,source_country,source_reverse_dns,source_base_domain,delivery_result,auth_failure,reported_domain,authentication_mechanisms,sample_headers_only
auth-failure,Lua/1.0,1.0,,sharepoint@domain.de,peter.pan@domain.de,"Mon, 01 Oct 2018 11:20:27 +0200",2018-10-01 09:20:27,Subject,<38.E7.30937.BD6E1BB5@ mailrelay.de>,"dmarc=fail (p=none, dis=none) header.from=domain.de",,10.10.10.10,,,,smg-policy-action,dmarc,domain.de,,False
Bug reports
===========
@@ -382,7 +500,7 @@ Elasticsearch and Kibana
.. note::
Splunk is also supported starting with ``parsedmarc`` 4.1.3
Splunk is also supported starting with ``parsedmarc`` 4.3.0
To set up visual dashboards of DMARC data, install Elasticsearch and Kibana.
@@ -616,11 +734,17 @@ select ``dmarc_aggregate`` for the other saved objects, as shown below.
:align: center
:target: _static/screenshots/index-pattern-conflicts.png
Records retention
~~~~~~~~~~~~~~~~~
To prevent your indexes from growing too large, or to comply with records
retention regulations such as GDPR, you need to use `time-based indexes
<https://www.elastic.co/blog/managing-time-based-indices-efficiently>`_.
Splunk
------
Starting in version 4.1.3 ``parsedmarc`` supports sending aggregate and/or
Starting in version 4.3.0 ``parsedmarc`` supports sending aggregate and/or
forensic DMARC data to a Splunk `HTTP Event collector (HEC)`_. Simply use the
following command line options, along with ``--save-aggregate`` and/or
``--save-forensic``:
@@ -902,6 +1026,12 @@ parsedmarc.elastic
.. automodule:: parsedmarc.elastic
:members:
.. automodule:: parsedmarc.splunk
:members:
.. automodule:: parsedmarc.utils
:members:
.. toctree::
:maxdepth: 2
:caption: Contents:
_static/documentation_options.js
+1 -1
View File
@@ -1,6 +1,6 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT: document.getElementById("documentation_options").getAttribute('data-url_root'),
VERSION: '4.2.0k',
VERSION: '4.3.0',
LANGUAGE: 'None',
COLLAPSE_INDEX: false,
FILE_SUFFIX: '.html',
genindex.html
+60 -6
View File
@@ -9,7 +9,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Index &mdash; parsedmarc 4.2.0k documentation</title>
<title>Index &mdash; parsedmarc 4.3.0 documentation</title>
@@ -57,7 +57,7 @@
<div class="version">
4.2.0k
4.3.0
</div>
@@ -154,7 +154,9 @@
| <a href="#H"><strong>H</strong></a>
| <a href="#I"><strong>I</strong></a>
| <a href="#P"><strong>P</strong></a>
| <a href="#Q"><strong>Q</strong></a>
| <a href="#S"><strong>S</strong></a>
| <a href="#T"><strong>T</strong></a>
| <a href="#W"><strong>W</strong></a>
</div>
@@ -168,6 +170,10 @@
<h2 id="C">C</h2>
<table style="width: 100%" class="indextable genindextable"><tr>
<td style="width: 33%; vertical-align: top;"><ul>
<li><a href="index.html#parsedmarc.utils.convert_outlook_msg">convert_outlook_msg() (in module parsedmarc.utils)</a>
</li>
</ul></td>
<td style="width: 33%; vertical-align: top;"><ul>
<li><a href="index.html#parsedmarc.elastic.create_indexes">create_indexes() (in module parsedmarc.elastic)</a>
</li>
@@ -181,6 +187,8 @@
</li>
</ul></td>
<td style="width: 33%; vertical-align: top;"><ul>
<li><a href="index.html#parsedmarc.utils.EmailParserError">EmailParserError</a>
</li>
<li><a href="index.html#parsedmarc.extract_xml">extract_xml() (in module parsedmarc)</a>
</li>
</ul></td>
@@ -189,13 +197,23 @@
<h2 id="G">G</h2>
<table style="width: 100%" class="indextable genindextable"><tr>
<td style="width: 33%; vertical-align: top;"><ul>
<li><a href="index.html#parsedmarc.utils.get_base_domain">get_base_domain() (in module parsedmarc.utils)</a>
</li>
<li><a href="index.html#parsedmarc.get_dmarc_reports_from_inbox">get_dmarc_reports_from_inbox() (in module parsedmarc)</a>
</li>
<li><a href="index.html#parsedmarc.utils.get_filename_safe_string">get_filename_safe_string() (in module parsedmarc.utils)</a>
</li>
<li><a href="index.html#parsedmarc.get_imap_capabilities">get_imap_capabilities() (in module parsedmarc)</a>
</li>
</ul></td>
<td style="width: 33%; vertical-align: top;"><ul>
<li><a href="index.html#parsedmarc.get_imap_capabilities">get_imap_capabilities() (in module parsedmarc)</a>
<li><a href="index.html#parsedmarc.utils.get_ip_address_country">get_ip_address_country() (in module parsedmarc.utils)</a>
</li>
<li><a href="index.html#parsedmarc.utils.get_ip_address_info">get_ip_address_info() (in module parsedmarc.utils)</a>
</li>
<li><a href="index.html#parsedmarc.get_report_zip">get_report_zip() (in module parsedmarc)</a>
</li>
<li><a href="index.html#parsedmarc.utils.get_reverse_dns">get_reverse_dns() (in module parsedmarc.utils)</a>
</li>
</ul></td>
</tr></table>
@@ -203,11 +221,13 @@
<h2 id="H">H</h2>
<table style="width: 100%" class="indextable genindextable"><tr>
<td style="width: 33%; vertical-align: top;"><ul>
<li><a href="index.html#parsedmarc.human_timestamp_to_datetime">human_timestamp_to_datetime() (in module parsedmarc)</a>
<li><a href="index.html#parsedmarc.splunk.HECClient">HECClient (class in parsedmarc.splunk)</a>
</li>
</ul></td>
<td style="width: 33%; vertical-align: top;"><ul>
<li><a href="index.html#parsedmarc.human_timestamp_to_timestamp">human_timestamp_to_timestamp() (in module parsedmarc)</a>
<li><a href="index.html#parsedmarc.utils.human_timestamp_to_datetime">human_timestamp_to_datetime() (in module parsedmarc.utils)</a>
</li>
<li><a href="index.html#parsedmarc.utils.human_timestamp_to_timestamp">human_timestamp_to_timestamp() (in module parsedmarc.utils)</a>
</li>
</ul></td>
</tr></table>
@@ -224,6 +244,8 @@
<li><a href="index.html#parsedmarc.InvalidDMARCReport">InvalidDMARCReport</a>
</li>
<li><a href="index.html#parsedmarc.InvalidForensicReport">InvalidForensicReport</a>
</li>
<li><a href="index.html#parsedmarc.utils.is_outlook_msg">is_outlook_msg() (in module parsedmarc.utils)</a>
</li>
</ul></td>
</tr></table>
@@ -234,6 +256,8 @@
<li><a href="index.html#parsedmarc.parse_aggregate_report_file">parse_aggregate_report_file() (in module parsedmarc)</a>
</li>
<li><a href="index.html#parsedmarc.parse_aggregate_report_xml">parse_aggregate_report_xml() (in module parsedmarc)</a>
</li>
<li><a href="index.html#parsedmarc.utils.parse_email">parse_email() (in module parsedmarc.utils)</a>
</li>
<li><a href="index.html#parsedmarc.parse_forensic_report">parse_forensic_report() (in module parsedmarc)</a>
</li>
@@ -250,18 +274,34 @@
<li><a href="index.html#module-parsedmarc">parsedmarc (module)</a>
</li>
<li><a href="index.html#module-parsedmarc.elastic">parsedmarc.elastic (module)</a>
</li>
<li><a href="index.html#module-parsedmarc.splunk">parsedmarc.splunk (module)</a>
</li>
<li><a href="index.html#module-parsedmarc.utils">parsedmarc.utils (module)</a>
</li>
<li><a href="index.html#parsedmarc.ParserError">ParserError</a>
</li>
</ul></td>
</tr></table>
<h2 id="Q">Q</h2>
<table style="width: 100%" class="indextable genindextable"><tr>
<td style="width: 33%; vertical-align: top;"><ul>
<li><a href="index.html#parsedmarc.utils.query_dns">query_dns() (in module parsedmarc.utils)</a>
</li>
</ul></td>
</tr></table>
<h2 id="S">S</h2>
<table style="width: 100%" class="indextable genindextable"><tr>
<td style="width: 33%; vertical-align: top;"><ul>
<li><a href="index.html#parsedmarc.elastic.save_aggregate_report_to_elasticsearch">save_aggregate_report_to_elasticsearch() (in module parsedmarc.elastic)</a>
</li>
<li><a href="index.html#parsedmarc.splunk.HECClient.save_aggregate_reports_to_splunk">save_aggregate_reports_to_splunk() (parsedmarc.splunk.HECClient method)</a>
</li>
<li><a href="index.html#parsedmarc.elastic.save_forensic_report_to_elasticsearch">save_forensic_report_to_elasticsearch() (in module parsedmarc.elastic)</a>
</li>
<li><a href="index.html#parsedmarc.splunk.HECClient.save_forensic_reports_to_splunk">save_forensic_reports_to_splunk() (parsedmarc.splunk.HECClient method)</a>
</li>
</ul></td>
<td style="width: 33%; vertical-align: top;"><ul>
@@ -270,6 +310,20 @@
<li><a href="index.html#parsedmarc.elastic.set_hosts">set_hosts() (in module parsedmarc.elastic)</a>
</li>
<li><a href="index.html#parsedmarc.SMTPError">SMTPError</a>
</li>
<li><a href="index.html#parsedmarc.splunk.SplunkError">SplunkError</a>
</li>
</ul></td>
</tr></table>
<h2 id="T">T</h2>
<table style="width: 100%" class="indextable genindextable"><tr>
<td style="width: 33%; vertical-align: top;"><ul>
<li><a href="index.html#parsedmarc.utils.timestamp_to_datetime">timestamp_to_datetime() (in module parsedmarc.utils)</a>
</li>
</ul></td>
<td style="width: 33%; vertical-align: top;"><ul>
<li><a href="index.html#parsedmarc.utils.timestamp_to_human">timestamp_to_human() (in module parsedmarc.utils)</a>
</li>
</ul></td>
</tr></table>
@@ -318,7 +372,7 @@
<script type="text/javascript">
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'./',
VERSION:'4.2.0k',
VERSION:'4.3.0',
LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
index.html
+487 -71
View File
@@ -8,7 +8,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>parsedmarc documentation - Open source DMARC report analyzer and visualizer &mdash; parsedmarc 4.2.0k documentation</title>
<title>parsedmarc documentation - Open source DMARC report analyzer and visualizer &mdash; parsedmarc 4.3.0 documentation</title>
@@ -56,7 +56,7 @@
<div class="version">
4.2.0k
4.3.0
</div>
@@ -84,23 +84,33 @@
<div class="local-toc"><ul>
<li><a class="reference internal" href="#">parsedmarc documentation - Open source DMARC report analyzer and visualizer</a><ul>
<li><a class="reference internal" href="#features">Features</a></li>
<li><a class="reference internal" href="#resources">Resources</a></li>
<li><a class="reference internal" href="#cli-help">CLI help</a></li>
<li><a class="reference internal" href="#resources">Resources</a><ul>
<li><a class="reference internal" href="#dmarc-guides">DMARC guides</a></li>
<li><a class="reference internal" href="#spf-and-dmarc-record-validation">SPF and DMARC record validation</a></li>
<li><a class="reference internal" href="#id1">SPF and DMARC record validation</a></li>
<li><a class="reference internal" href="#lookalike-domains">Lookalike domains</a></li>
</ul>
</li>
<li><a class="reference internal" href="#cli-help">CLI help</a></li>
<li><a class="reference internal" href="#sample-aggregate-report-output">Sample aggregate report output</a><ul>
<li><a class="reference internal" href="#json">JSON</a></li>
<li><a class="reference internal" href="#csv">CSV</a></li>
</ul>
</li>
<li><a class="reference internal" href="#sample-forensic-report-output">Sample forensic report output</a></li>
<li><a class="reference internal" href="#sample-forensic-report-output">Sample forensic report output</a><ul>
<li><a class="reference internal" href="#id1">JSON</a></li>
<li><a class="reference internal" href="#id2">CSV</a></li>
</ul>
</li>
<li><a class="reference internal" href="#bug-reports">Bug reports</a></li>
<li><a class="reference internal" href="#installation">Installation</a><ul>
<li><a class="reference internal" href="#installation-using-pypy3">Installation using pypy3</a></li>
<li><a class="reference internal" href="#optional-dependencies">Optional dependencies</a></li>
<li><a class="reference internal" href="#dns-performance">DNS performance</a></li>
<li><a class="reference internal" href="#testing-multiple-report-analyzers">Testing multiple report analyzers</a></li>
<li><a class="reference internal" href="#elasticsearch-and-kibana">Elasticsearch and Kibana</a></li>
<li><a class="reference internal" href="#elasticsearch-and-kibana">Elasticsearch and Kibana</a><ul>
<li><a class="reference internal" href="#records-retention">Records retention</a></li>
</ul>
</li>
<li><a class="reference internal" href="#splunk">Splunk</a></li>
<li><a class="reference internal" href="#running-parsedmarc-as-a-systemd-service">Running parsedmarc as a systemd service</a></li>
</ul>
@@ -208,10 +218,24 @@ premade dashboards</li>
</div>
<div class="section" id="resources">
<h2>Resources<a class="headerlink" href="#resources" title="Permalink to this headline"></a></h2>
<div class="section" id="dmarc-guides">
<h3>DMARC guides<a class="headerlink" href="#dmarc-guides" title="Permalink to this headline"></a></h3>
<ul class="simple">
<li><a class="reference external" href="https://seanthegeek.net/459/demystifying-dmarc/">Demystifying DMARC</a> - A complete guide to SPF, DKIM, and DMARC</li>
</ul>
</div>
<div class="section" id="spf-and-dmarc-record-validation">
<h3>SPF and DMARC record validation<a class="headerlink" href="#spf-and-dmarc-record-validation" title="Permalink to this headline"></a></h3>
<p>If you are looking for SPF and DMARC record validation and parsing,
check out the sister project,
<a class="reference external" href="https://domainaware.github.io/checkdmarc/">checkdmarc</a>.</p>
</div>
<div class="section" id="lookalike-domains">
<h3>Lookalike domains<a class="headerlink" href="#lookalike-domains" title="Permalink to this headline"></a></h3>
<p>DMARC protects against domain spoofing, not lookalike domains. for open source
lookalike domain monitoring, check out <a class="reference external" href="https://github.com/seanthegeek/domainaware">DomainAware</a>.</p>
</div>
</div>
<div class="section" id="cli-help">
<h2>CLI help<a class="headerlink" href="#cli-help" title="Permalink to this headline"></a></h2>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">usage</span><span class="p">:</span> <span class="n">parsedmarc</span> <span class="p">[</span><span class="o">-</span><span class="n">h</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">o</span> <span class="n">OUTPUT</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">n</span> <span class="n">NAMESERVERS</span> <span class="p">[</span><span class="n">NAMESERVERS</span> <span class="o">...</span><span class="p">]]</span>
@@ -222,9 +246,13 @@ premade dashboards</li>
<span class="p">[</span><span class="o">--</span><span class="n">elasticsearch</span><span class="o">-</span><span class="n">index</span><span class="o">-</span><span class="n">prefix</span> <span class="n">ELASTICSEARCH_INDEX_PREFIX</span><span class="p">]</span>
<span class="p">[</span><span class="o">--</span><span class="n">elasticsearch</span><span class="o">-</span><span class="n">index</span><span class="o">-</span><span class="n">suffix</span> <span class="n">ELASTICSEARCH_INDEX_SUFFIX</span><span class="p">]</span>
<span class="p">[</span><span class="o">--</span><span class="n">hec</span> <span class="n">HEC</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">hec</span><span class="o">-</span><span class="n">token</span> <span class="n">HEC_TOKEN</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">hec</span><span class="o">-</span><span class="n">index</span> <span class="n">HEC_INDEX</span><span class="p">]</span>
<span class="p">[</span><span class="o">--</span><span class="n">hec</span><span class="o">-</span><span class="n">skip</span><span class="o">-</span><span class="n">certificate</span><span class="o">-</span><span class="n">verification</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">save</span><span class="o">-</span><span class="n">aggregate</span><span class="p">]</span>
<span class="p">[</span><span class="o">--</span><span class="n">save</span><span class="o">-</span><span class="n">forensic</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">O</span> <span class="n">OUTGOING_HOST</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">U</span> <span class="n">OUTGOING_USER</span><span class="p">]</span>
<span class="p">[</span><span class="o">-</span><span class="n">P</span> <span class="n">OUTGOING_PASSWORD</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">outgoing</span><span class="o">-</span><span class="n">port</span> <span class="n">OUTGOING_PORT</span><span class="p">]</span>
<span class="p">[</span><span class="o">--</span><span class="n">hec</span><span class="o">-</span><span class="n">skip</span><span class="o">-</span><span class="n">certificate</span><span class="o">-</span><span class="n">verification</span><span class="p">]</span>
<span class="p">[</span><span class="o">-</span><span class="n">K</span> <span class="p">[</span><span class="n">KAFKA_HOSTS</span> <span class="p">[</span><span class="n">KAFKA_HOSTS</span> <span class="o">...</span><span class="p">]]]</span>
<span class="p">[</span><span class="o">--</span><span class="n">kafka</span><span class="o">-</span><span class="n">aggregate</span><span class="o">-</span><span class="n">topic</span> <span class="n">KAFKA_AGGREGATE_TOPIC</span><span class="p">]</span>
<span class="p">[</span><span class="o">--</span><span class="n">kafka</span><span class="o">-</span><span class="n">forensic_topic</span> <span class="n">KAFKA_FORENSIC_TOPIC</span><span class="p">]</span>
<span class="p">[</span><span class="o">--</span><span class="n">save</span><span class="o">-</span><span class="n">aggregate</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">save</span><span class="o">-</span><span class="n">forensic</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">O</span> <span class="n">OUTGOING_HOST</span><span class="p">]</span>
<span class="p">[</span><span class="o">-</span><span class="n">U</span> <span class="n">OUTGOING_USER</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">P</span> <span class="n">OUTGOING_PASSWORD</span><span class="p">]</span>
<span class="p">[</span><span class="o">--</span><span class="n">outgoing</span><span class="o">-</span><span class="n">port</span> <span class="n">OUTGOING_PORT</span><span class="p">]</span>
<span class="p">[</span><span class="o">--</span><span class="n">outgoing</span><span class="o">-</span><span class="n">ssl</span> <span class="n">OUTGOING_SSL</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">F</span> <span class="n">OUTGOING_FROM</span><span class="p">]</span>
<span class="p">[</span><span class="o">-</span><span class="n">T</span> <span class="n">OUTGOING_TO</span> <span class="p">[</span><span class="n">OUTGOING_TO</span> <span class="o">...</span><span class="p">]]</span> <span class="p">[</span><span class="o">-</span><span class="n">S</span> <span class="n">OUTGOING_SUBJECT</span><span class="p">]</span>
<span class="p">[</span><span class="o">-</span><span class="n">A</span> <span class="n">OUTGOING_ATTACHMENT</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">M</span> <span class="n">OUTGOING_MESSAGE</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">w</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">test</span><span class="p">]</span>
@@ -242,10 +270,11 @@ premade dashboards</li>
<span class="o">-</span><span class="n">o</span> <span class="n">OUTPUT</span><span class="p">,</span> <span class="o">--</span><span class="n">output</span> <span class="n">OUTPUT</span>
<span class="n">Write</span> <span class="n">output</span> <span class="n">files</span> <span class="n">to</span> <span class="n">the</span> <span class="n">given</span> <span class="n">directory</span>
<span class="o">-</span><span class="n">n</span> <span class="n">NAMESERVERS</span> <span class="p">[</span><span class="n">NAMESERVERS</span> <span class="o">...</span><span class="p">],</span> <span class="o">--</span><span class="n">nameservers</span> <span class="n">NAMESERVERS</span> <span class="p">[</span><span class="n">NAMESERVERS</span> <span class="o">...</span><span class="p">]</span>
<span class="n">nameservers</span> <span class="n">to</span> <span class="n">query</span> <span class="p">(</span><span class="n">Default</span> <span class="ow">is</span> <span class="n">Cloudflare</span><span class="s1">&#39;s)</span>
<span class="n">nameservers</span> <span class="n">to</span> <span class="n">query</span> <span class="p">(</span><span class="n">Default</span> <span class="ow">is</span> <span class="n">Cloudflare</span><span class="s1">&#39;s</span>
<span class="n">nameservers</span><span class="p">)</span>
<span class="o">-</span><span class="n">t</span> <span class="n">TIMEOUT</span><span class="p">,</span> <span class="o">--</span><span class="n">timeout</span> <span class="n">TIMEOUT</span>
<span class="n">number</span> <span class="n">of</span> <span class="n">seconds</span> <span class="n">to</span> <span class="n">wait</span> <span class="k">for</span> <span class="n">an</span> <span class="n">answer</span> <span class="kn">from</span> <span class="nn">DNS</span>
<span class="p">(</span><span class="n">default</span> <span class="mf">2.0</span><span class="p">)</span>
<span class="p">(</span><span class="n">Default</span><span class="p">:</span> <span class="mf">2.0</span><span class="p">)</span>
<span class="o">-</span><span class="n">H</span> <span class="n">HOST</span><span class="p">,</span> <span class="o">--</span><span class="n">host</span> <span class="n">HOST</span> <span class="n">IMAP</span> <span class="n">hostname</span> <span class="ow">or</span> <span class="n">IP</span> <span class="n">address</span>
<span class="o">-</span><span class="n">u</span> <span class="n">USER</span><span class="p">,</span> <span class="o">--</span><span class="n">user</span> <span class="n">USER</span> <span class="n">IMAP</span> <span class="n">user</span>
<span class="o">-</span><span class="n">p</span> <span class="n">PASSWORD</span><span class="p">,</span> <span class="o">--</span><span class="n">password</span> <span class="n">PASSWORD</span>
@@ -254,14 +283,15 @@ premade dashboards</li>
<span class="n">IMAP</span> <span class="n">port</span>
<span class="o">--</span><span class="n">imap</span><span class="o">-</span><span class="n">no</span><span class="o">-</span><span class="n">ssl</span> <span class="n">Do</span> <span class="ow">not</span> <span class="n">use</span> <span class="n">SSL</span><span class="o">/</span><span class="n">TLS</span> <span class="n">when</span> <span class="n">connecting</span> <span class="n">to</span> <span class="n">IMAP</span>
<span class="o">-</span><span class="n">r</span> <span class="n">REPORTS_FOLDER</span><span class="p">,</span> <span class="o">--</span><span class="n">reports</span><span class="o">-</span><span class="n">folder</span> <span class="n">REPORTS_FOLDER</span>
<span class="n">The</span> <span class="n">IMAP</span> <span class="n">folder</span> <span class="n">containing</span> <span class="n">the</span> <span class="n">reports</span> <span class="n">Default</span><span class="p">:</span> <span class="n">INBOX</span>
<span class="n">The</span> <span class="n">IMAP</span> <span class="n">folder</span> <span class="n">containing</span> <span class="n">the</span> <span class="n">reports</span> <span class="p">(</span><span class="n">Default</span><span class="p">:</span>
<span class="n">INBOX</span><span class="p">)</span>
<span class="o">-</span><span class="n">a</span> <span class="n">ARCHIVE_FOLDER</span><span class="p">,</span> <span class="o">--</span><span class="n">archive</span><span class="o">-</span><span class="n">folder</span> <span class="n">ARCHIVE_FOLDER</span>
<span class="n">Specifies</span> <span class="n">the</span> <span class="n">IMAP</span> <span class="n">folder</span> <span class="n">to</span> <span class="n">move</span> <span class="n">messages</span> <span class="n">to</span> <span class="n">after</span>
<span class="n">processing</span> <span class="n">them</span> <span class="n">Default</span><span class="p">:</span> <span class="n">Archive</span>
<span class="n">processing</span> <span class="n">them</span> <span class="p">(</span><span class="n">Default</span><span class="p">:</span> <span class="n">Archive</span><span class="p">)</span>
<span class="o">-</span><span class="n">d</span><span class="p">,</span> <span class="o">--</span><span class="n">delete</span> <span class="n">Delete</span> <span class="n">the</span> <span class="n">reports</span> <span class="n">after</span> <span class="n">processing</span> <span class="n">them</span>
<span class="o">-</span><span class="n">E</span> <span class="p">[</span><span class="n">ELASTICSEARCH_HOST</span> <span class="p">[</span><span class="n">ELASTICSEARCH_HOST</span> <span class="o">...</span><span class="p">]],</span> <span class="o">--</span><span class="n">elasticsearch</span><span class="o">-</span><span class="n">host</span> <span class="p">[</span><span class="n">ELASTICSEARCH_HOST</span> <span class="p">[</span><span class="n">ELASTICSEARCH_HOST</span> <span class="o">...</span><span class="p">]]</span>
<span class="n">A</span> <span class="nb">list</span> <span class="n">of</span> <span class="n">one</span> <span class="ow">or</span> <span class="n">more</span> <span class="n">Elasticsearch</span> <span class="n">hostnames</span> <span class="ow">or</span> <span class="n">URLs</span>
<span class="n">to</span> <span class="n">use</span> <span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="n">g</span><span class="o">.</span> <span class="n">localhost</span><span class="p">:</span><span class="mi">9200</span><span class="p">)</span>
<span class="n">One</span> <span class="ow">or</span> <span class="n">more</span> <span class="n">Elasticsearch</span> <span class="n">hostnames</span> <span class="ow">or</span> <span class="n">URLs</span> <span class="n">to</span> <span class="n">use</span>
<span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="n">g</span><span class="o">.</span> <span class="n">localhost</span><span class="p">:</span><span class="mi">9200</span><span class="p">)</span>
<span class="o">--</span><span class="n">elasticsearch</span><span class="o">-</span><span class="n">index</span><span class="o">-</span><span class="n">prefix</span> <span class="n">ELASTICSEARCH_INDEX_PREFIX</span>
<span class="n">Prefix</span> <span class="n">to</span> <span class="n">add</span> <span class="ow">in</span> <span class="n">front</span> <span class="n">of</span> <span class="n">the</span> <span class="n">dmarc_aggregate</span> <span class="ow">and</span>
<span class="n">dmarc_forensic</span> <span class="n">Elasticsearch</span> <span class="n">index</span> <span class="n">names</span><span class="p">,</span> <span class="n">joined</span> <span class="n">by</span> <span class="n">_</span>
@@ -277,6 +307,14 @@ premade dashboards</li>
<span class="n">HTTP</span> <span class="n">Event</span> <span class="n">Collector</span> <span class="p">(</span><span class="n">HEC</span><span class="p">)</span>
<span class="o">--</span><span class="n">hec</span><span class="o">-</span><span class="n">skip</span><span class="o">-</span><span class="n">certificate</span><span class="o">-</span><span class="n">verification</span>
<span class="n">Skip</span> <span class="n">certificate</span> <span class="n">verification</span> <span class="k">for</span> <span class="n">Splunk</span> <span class="n">HEC</span>
<span class="o">-</span><span class="n">K</span> <span class="p">[</span><span class="n">KAFKA_HOSTS</span> <span class="p">[</span><span class="n">KAFKA_HOSTS</span> <span class="o">...</span><span class="p">]],</span> <span class="o">--</span><span class="n">kafka</span><span class="o">-</span><span class="n">hosts</span> <span class="p">[</span><span class="n">KAFKA_HOSTS</span> <span class="p">[</span><span class="n">KAFKA_HOSTS</span> <span class="o">...</span><span class="p">]]</span>
<span class="n">A</span> <span class="nb">list</span> <span class="n">of</span> <span class="n">one</span> <span class="ow">or</span> <span class="n">more</span> <span class="n">Kafka</span> <span class="n">hostnames</span> <span class="ow">or</span> <span class="n">URLs</span>
<span class="o">--</span><span class="n">kafka</span><span class="o">-</span><span class="n">aggregate</span><span class="o">-</span><span class="n">topic</span> <span class="n">KAFKA_AGGREGATE_TOPIC</span>
<span class="n">The</span> <span class="n">Kafka</span> <span class="n">topic</span> <span class="n">to</span> <span class="n">publish</span> <span class="n">aggregate</span> <span class="n">reports</span> <span class="n">to</span>
<span class="p">(</span><span class="n">Default</span><span class="p">:</span> <span class="n">dmarc_aggregate</span><span class="p">)</span>
<span class="o">--</span><span class="n">kafka</span><span class="o">-</span><span class="n">forensic_topic</span> <span class="n">KAFKA_FORENSIC_TOPIC</span>
<span class="n">The</span> <span class="n">Kafka</span> <span class="n">topic</span> <span class="n">to</span> <span class="n">publish</span> <span class="n">forensic</span> <span class="n">reports</span> <span class="n">to</span>
<span class="p">(</span><span class="n">Default</span><span class="p">:</span> <span class="n">dmarc_forensic</span><span class="p">)</span>
<span class="o">--</span><span class="n">save</span><span class="o">-</span><span class="n">aggregate</span> <span class="n">Save</span> <span class="n">aggregate</span> <span class="n">reports</span> <span class="n">to</span> <span class="n">search</span> <span class="n">indexes</span>
<span class="o">--</span><span class="n">save</span><span class="o">-</span><span class="n">forensic</span> <span class="n">Save</span> <span class="n">forensic</span> <span class="n">reports</span> <span class="n">to</span> <span class="n">search</span> <span class="n">indexes</span>
<span class="o">-</span><span class="n">O</span> <span class="n">OUTGOING_HOST</span><span class="p">,</span> <span class="o">--</span><span class="n">outgoing</span><span class="o">-</span><span class="n">host</span> <span class="n">OUTGOING_HOST</span>
@@ -309,18 +347,6 @@ premade dashboards</li>
</pre></div>
</div>
</div>
<div class="section" id="spf-and-dmarc-record-validation">
<h2>SPF and DMARC record validation<a class="headerlink" href="#spf-and-dmarc-record-validation" title="Permalink to this headline"></a></h2>
<p>If you are looking for SPF and DMARC record validation and parsing,
check out the sister project,
<a class="reference external" href="https://domainaware.github.io/checkdmarc/">checkdmarc</a>.</p>
</div>
<div class="section" id="id1">
<h2>SPF and DMARC record validation<a class="headerlink" href="#id1" title="Permalink to this headline"></a></h2>
<p>If you are looking for SPF and DMARC record validation and parsing,
check out the sister project,
<a class="reference external" href="https://domainaware.github.io/checkdmarc/">checkdmarc</a>.</p>
</div>
<div class="section" id="sample-aggregate-report-output">
<h2>Sample aggregate report output<a class="headerlink" href="#sample-aggregate-report-output" title="Permalink to this headline"></a></h2>
<p>Here are the results from parsing the <a class="reference external" href="https://dmarc.org/wiki/FAQ#I_need_to_implement_aggregate_reports.2C_what_do_they_look_like.3F">example</a>
@@ -409,8 +435,106 @@ schema.</p>
</div>
<div class="section" id="sample-forensic-report-output">
<h2>Sample forensic report output<a class="headerlink" href="#sample-forensic-report-output" title="Permalink to this headline"></a></h2>
<p>I don’t have a sample I can share for privacy reasons. If you have a sample
forensic report that you can share publicly, please contact me!</p>
<p>Thanks to Github user <a class="reference external" href="https://github.com/xennn">xennn</a> for the anonymized
<a class="reference external" href="https://github.com/domainaware/parsedmarc/raw/master/samples/forensic/DMARC%20Failure%20Report%20for%20domain.de%20(mail-from%3Dsharepoint%40domain.de%2C%20ip%3D10.10.10.10).eml">forensic report email sample</a>.</p>
<div class="section" id="id1">
<h3>JSON<a class="headerlink" href="#id1" title="Permalink to this headline"></a></h3>
<div class="highlight-json notranslate"><div class="highlight"><pre><span></span><span class="p">{</span>
<span class="nt">&quot;feedback_type&quot;</span><span class="p">:</span> <span class="s2">&quot;auth-failure&quot;</span><span class="p">,</span>
<span class="nt">&quot;user_agent&quot;</span><span class="p">:</span> <span class="s2">&quot;Lua/1.0&quot;</span><span class="p">,</span>
<span class="nt">&quot;version&quot;</span><span class="p">:</span> <span class="s2">&quot;1.0&quot;</span><span class="p">,</span>
<span class="nt">&quot;original_mail_from&quot;</span><span class="p">:</span> <span class="s2">&quot;sharepoint@domain.de&quot;</span><span class="p">,</span>
<span class="nt">&quot;original_rcpt_to&quot;</span><span class="p">:</span> <span class="s2">&quot;peter.pan@domain.de&quot;</span><span class="p">,</span>
<span class="nt">&quot;arrival_date&quot;</span><span class="p">:</span> <span class="s2">&quot;Mon, 01 Oct 2018 11:20:27 +0200&quot;</span><span class="p">,</span>
<span class="nt">&quot;message_id&quot;</span><span class="p">:</span> <span class="s2">&quot;&lt;38.E7.30937.BD6E1BB5@ mailrelay.de&gt;&quot;</span><span class="p">,</span>
<span class="nt">&quot;authentication_results&quot;</span><span class="p">:</span> <span class="s2">&quot;dmarc=fail (p=none, dis=none) header.from=domain.de&quot;</span><span class="p">,</span>
<span class="nt">&quot;delivery_result&quot;</span><span class="p">:</span> <span class="s2">&quot;smg-policy-action&quot;</span><span class="p">,</span>
<span class="nt">&quot;auth_failure&quot;</span><span class="p">:</span> <span class="p">[</span>
<span class="s2">&quot;dmarc&quot;</span>
<span class="p">],</span>
<span class="nt">&quot;reported_domain&quot;</span><span class="p">:</span> <span class="s2">&quot;domain.de&quot;</span><span class="p">,</span>
<span class="nt">&quot;arrival_date_utc&quot;</span><span class="p">:</span> <span class="s2">&quot;2018-10-01 09:20:27&quot;</span><span class="p">,</span>
<span class="nt">&quot;source&quot;</span><span class="p">:</span> <span class="p">{</span>
<span class="nt">&quot;ip_address&quot;</span><span class="p">:</span> <span class="s2">&quot;10.10.10.10&quot;</span><span class="p">,</span>
<span class="nt">&quot;country&quot;</span><span class="p">:</span> <span class="kc">null</span><span class="p">,</span>
<span class="nt">&quot;reverse_dns&quot;</span><span class="p">:</span> <span class="kc">null</span><span class="p">,</span>
<span class="nt">&quot;base_domain&quot;</span><span class="p">:</span> <span class="kc">null</span>
<span class="p">},</span>
<span class="nt">&quot;authentication_mechanisms&quot;</span><span class="p">:</span> <span class="p">[],</span>
<span class="nt">&quot;original_envelope_id&quot;</span><span class="p">:</span> <span class="kc">null</span><span class="p">,</span>
<span class="nt">&quot;dkim_domain&quot;</span><span class="p">:</span> <span class="kc">null</span><span class="p">,</span>
<span class="nt">&quot;sample_headers_only&quot;</span><span class="p">:</span> <span class="kc">false</span><span class="p">,</span>
<span class="nt">&quot;sample&quot;</span><span class="p">:</span> <span class="s2">&quot;Content-Type: message/rfc822\nContent-Disposition: inline\nReceived: from Servernameone.domain.local (Servernameone.domain.local [10.10.10.10])\n by mailrelay.de (mail.DOMAIN.de) with SMTP id 38.E7.30937.BD6E1BB5; Mon, 1 Oct 2018 11:20:27 +0200 (CEST)\nDate: 01 Oct 2018 11:20:27 +0200\nMessage-ID: &lt;38.E7.30937.BD6E1BB5@ mailrelay.de&gt;\nTo: &lt;peter.pan@domain.de&gt;\nfrom: \&quot;=?utf-8?B?SW50ZXJha3RpdmUgV2V0dGJld2VyYmVyLcOcYmVyc2ljaHQ=?=\&quot; &lt;sharepoint@domain.de&gt;\nSubject: Subject\nMIME-Version: 1.0\nX-Mailer: Microsoft SharePoint Foundation 2010\nContent-Type: text/html; charset=utf-8\nContent-Transfer-Encoding: quoted-printable\n\n\n&lt;html&gt;&lt;head&gt;&lt;base href=3D&#39;\nwettbewerb&#39; /&gt;&lt;/head&gt;&lt;body&gt;&lt;!DOCTYPE HTML PUBLIC \&quot;-//W3C//DTD HTML 3.2//EN\&quot;=\n&gt;&lt;HTML&gt;&lt;HEAD&gt;&lt;META NAME=3D\&quot;Generator\&quot; CONTENT=3D\&quot;MS Exchange Server version=\n 08.01.0240.003\&quot;&gt;&lt;/html&gt;\n &quot;</span><span class="p">,</span>
<span class="nt">&quot;parsed_sample&quot;</span><span class="p">:</span> <span class="p">{</span>
<span class="nt">&quot;content-transfer-encoding&quot;</span><span class="p">:</span> <span class="s2">&quot;quoted-printable&quot;</span><span class="p">,</span>
<span class="nt">&quot;x-mailer&quot;</span><span class="p">:</span> <span class="s2">&quot;Microsoft SharePoint Foundation 2010&quot;</span><span class="p">,</span>
<span class="nt">&quot;message-id&quot;</span><span class="p">:</span> <span class="s2">&quot;&lt;38.E7.30937.BD6E1BB5@ mailrelay.de&gt;&quot;</span><span class="p">,</span>
<span class="nt">&quot;body&quot;</span><span class="p">:</span> <span class="s2">&quot;&lt;html&gt;&lt;head&gt;&lt;base href=3D&#39;\nwettbewerb&#39; /&gt;&lt;/head&gt;&lt;body&gt;&lt;!DOCTYPE HTML PUBLIC \&quot;-//W3C//DTD HTML 3.2//EN\&quot;=\n&gt;&lt;HTML&gt;&lt;HEAD&gt;&lt;META NAME=3D\&quot;Generator\&quot; CONTENT=3D\&quot;MS Exchange Server version=\n 08.01.0240.003\&quot;&gt;&lt;/html&gt;&quot;</span><span class="p">,</span>
<span class="nt">&quot;to&quot;</span><span class="p">:</span> <span class="p">[</span>
<span class="p">{</span>
<span class="nt">&quot;display_name&quot;</span><span class="p">:</span> <span class="kc">null</span><span class="p">,</span>
<span class="nt">&quot;address&quot;</span><span class="p">:</span> <span class="s2">&quot;peter.pan@domain.de&quot;</span><span class="p">,</span>
<span class="nt">&quot;local&quot;</span><span class="p">:</span> <span class="s2">&quot;peter.pan&quot;</span><span class="p">,</span>
<span class="nt">&quot;domain&quot;</span><span class="p">:</span> <span class="s2">&quot;domain.de&quot;</span>
<span class="p">}</span>
<span class="p">],</span>
<span class="nt">&quot;date&quot;</span><span class="p">:</span> <span class="s2">&quot;2018-10-01 09:20:27&quot;</span><span class="p">,</span>
<span class="nt">&quot;to_domains&quot;</span><span class="p">:</span> <span class="p">[</span>
<span class="s2">&quot;domain.de&quot;</span>
<span class="p">],</span>
<span class="nt">&quot;received&quot;</span><span class="p">:</span> <span class="p">[</span>
<span class="p">{</span>
<span class="nt">&quot;from&quot;</span><span class="p">:</span> <span class="s2">&quot;Servernameone.domain.local Servernameone.domain.local 10.10.10.10&quot;</span><span class="p">,</span>
<span class="nt">&quot;by&quot;</span><span class="p">:</span> <span class="s2">&quot;mailrelay.de mail.DOMAIN.de&quot;</span><span class="p">,</span>
<span class="nt">&quot;with&quot;</span><span class="p">:</span> <span class="s2">&quot;SMTP id 38.E7.30937.BD6E1BB5&quot;</span><span class="p">,</span>
<span class="nt">&quot;date&quot;</span><span class="p">:</span> <span class="s2">&quot;Mon, 1 Oct 2018 11:20:27 +0200 CEST&quot;</span><span class="p">,</span>
<span class="nt">&quot;hop&quot;</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
<span class="nt">&quot;date_utc&quot;</span><span class="p">:</span> <span class="s2">&quot;2018-10-01 09:20:27&quot;</span><span class="p">,</span>
<span class="nt">&quot;delay&quot;</span><span class="p">:</span> <span class="mi">0</span>
<span class="p">}</span>
<span class="p">],</span>
<span class="nt">&quot;content-disposition&quot;</span><span class="p">:</span> <span class="s2">&quot;inline&quot;</span><span class="p">,</span>
<span class="nt">&quot;mime-version&quot;</span><span class="p">:</span> <span class="s2">&quot;1.0&quot;</span><span class="p">,</span>
<span class="nt">&quot;subject&quot;</span><span class="p">:</span> <span class="s2">&quot;Subject&quot;</span><span class="p">,</span>
<span class="nt">&quot;timezone&quot;</span><span class="p">:</span> <span class="s2">&quot;+2&quot;</span><span class="p">,</span>
<span class="nt">&quot;from&quot;</span><span class="p">:</span> <span class="p">{</span>
<span class="nt">&quot;display_name&quot;</span><span class="p">:</span> <span class="s2">&quot;Interaktive Wettbewerber-Übersicht&quot;</span><span class="p">,</span>
<span class="nt">&quot;address&quot;</span><span class="p">:</span> <span class="s2">&quot;sharepoint@domain.de&quot;</span><span class="p">,</span>
<span class="nt">&quot;local&quot;</span><span class="p">:</span> <span class="s2">&quot;sharepoint&quot;</span><span class="p">,</span>
<span class="nt">&quot;domain&quot;</span><span class="p">:</span> <span class="s2">&quot;domain.de&quot;</span>
<span class="p">},</span>
<span class="nt">&quot;content-type&quot;</span><span class="p">:</span> <span class="s2">&quot;message/rfc822&quot;</span><span class="p">,</span>
<span class="nt">&quot;has_defects&quot;</span><span class="p">:</span> <span class="kc">false</span><span class="p">,</span>
<span class="nt">&quot;headers&quot;</span><span class="p">:</span> <span class="p">{</span>
<span class="nt">&quot;Content-Type&quot;</span><span class="p">:</span> <span class="s2">&quot;text/html; charset=utf-8&quot;</span><span class="p">,</span>
<span class="nt">&quot;Content-Disposition&quot;</span><span class="p">:</span> <span class="s2">&quot;inline&quot;</span><span class="p">,</span>
<span class="nt">&quot;Received&quot;</span><span class="p">:</span> <span class="s2">&quot;from Servernameone.domain.local (Servernameone.domain.local [10.10.10.10])\n by mailrelay.de (mail.DOMAIN.de) with SMTP id 38.E7.30937.BD6E1BB5; Mon, 1 Oct 2018 11:20:27 +0200 (CEST)&quot;</span><span class="p">,</span>
<span class="nt">&quot;Date&quot;</span><span class="p">:</span> <span class="s2">&quot;01 Oct 2018 11:20:27 +0200&quot;</span><span class="p">,</span>
<span class="nt">&quot;Message-ID&quot;</span><span class="p">:</span> <span class="s2">&quot;&lt;38.E7.30937.BD6E1BB5@ mailrelay.de&gt;&quot;</span><span class="p">,</span>
<span class="nt">&quot;To&quot;</span><span class="p">:</span> <span class="s2">&quot;&lt;peter.pan@domain.de&gt;&quot;</span><span class="p">,</span>
<span class="nt">&quot;from&quot;</span><span class="p">:</span> <span class="s2">&quot;\&quot;Interaktive Wettbewerber-Übersicht\&quot; &lt;sharepoint@domain.de&gt;&quot;</span><span class="p">,</span>
<span class="nt">&quot;Subject&quot;</span><span class="p">:</span> <span class="s2">&quot;Subject&quot;</span><span class="p">,</span>
<span class="nt">&quot;MIME-Version&quot;</span><span class="p">:</span> <span class="s2">&quot;1.0&quot;</span><span class="p">,</span>
<span class="nt">&quot;X-Mailer&quot;</span><span class="p">:</span> <span class="s2">&quot;Microsoft SharePoint Foundation 2010&quot;</span><span class="p">,</span>
<span class="nt">&quot;Content-Transfer-Encoding&quot;</span><span class="p">:</span> <span class="s2">&quot;quoted-printable&quot;</span>
<span class="p">},</span>
<span class="nt">&quot;reply_to&quot;</span><span class="p">:</span> <span class="p">[],</span>
<span class="nt">&quot;cc&quot;</span><span class="p">:</span> <span class="p">[],</span>
<span class="nt">&quot;bcc&quot;</span><span class="p">:</span> <span class="p">[],</span>
<span class="nt">&quot;attachments&quot;</span><span class="p">:</span> <span class="p">[],</span>
<span class="nt">&quot;filename_safe_subject&quot;</span><span class="p">:</span> <span class="s2">&quot;Subject&quot;</span>
<span class="p">}</span>
<span class="p">}</span>
</pre></div>
</div>
</div>
<div class="section" id="id2">
<h3>CSV<a class="headerlink" href="#id2" title="Permalink to this headline"></a></h3>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">feedback_type</span><span class="p">,</span><span class="n">user_agent</span><span class="p">,</span><span class="n">version</span><span class="p">,</span><span class="n">original_envelope_id</span><span class="p">,</span><span class="n">original_mail_from</span><span class="p">,</span><span class="n">original_rcpt_to</span><span class="p">,</span><span class="n">arrival_date</span><span class="p">,</span><span class="n">arrival_date_utc</span><span class="p">,</span><span class="n">subject</span><span class="p">,</span><span class="n">message_id</span><span class="p">,</span><span class="n">authentication_results</span><span class="p">,</span><span class="n">dkim_domain</span><span class="p">,</span><span class="n">source_ip_address</span><span class="p">,</span><span class="n">source_country</span><span class="p">,</span><span class="n">source_reverse_dns</span><span class="p">,</span><span class="n">source_base_domain</span><span class="p">,</span><span class="n">delivery_result</span><span class="p">,</span><span class="n">auth_failure</span><span class="p">,</span><span class="n">reported_domain</span><span class="p">,</span><span class="n">authentication_mechanisms</span><span class="p">,</span><span class="n">sample_headers_only</span>
<span class="n">auth</span><span class="o">-</span><span class="n">failure</span><span class="p">,</span><span class="n">Lua</span><span class="o">/</span><span class="mf">1.0</span><span class="p">,</span><span class="mf">1.0</span><span class="p">,,</span><span class="n">sharepoint</span><span class="nd">@domain</span><span class="o">.</span><span class="n">de</span><span class="p">,</span><span class="n">peter</span><span class="o">.</span><span class="n">pan</span><span class="nd">@domain</span><span class="o">.</span><span class="n">de</span><span class="p">,</span><span class="s2">&quot;Mon, 01 Oct 2018 11:20:27 +0200&quot;</span><span class="p">,</span><span class="mi">2018</span><span class="o">-</span><span class="mi">10</span><span class="o">-</span><span class="mi">01</span> <span class="mi">09</span><span class="p">:</span><span class="mi">20</span><span class="p">:</span><span class="mi">27</span><span class="p">,</span><span class="n">Subject</span><span class="p">,</span><span class="o">&lt;</span><span class="mf">38.E7</span><span class="o">.</span><span class="mf">30937.</span><span class="n">BD6E1BB5</span><span class="o">@</span> <span class="n">mailrelay</span><span class="o">.</span><span class="n">de</span><span class="o">&gt;</span><span class="p">,</span><span class="s2">&quot;dmarc=fail (p=none, dis=none) header.from=domain.de&quot;</span><span class="p">,,</span><span class="mf">10.10</span><span class="o">.</span><span class="mf">10.10</span><span class="p">,,,,</span><span class="n">smg</span><span class="o">-</span><span class="n">policy</span><span class="o">-</span><span class="n">action</span><span class="p">,</span><span class="n">dmarc</span><span class="p">,</span><span class="n">domain</span><span class="o">.</span><span class="n">de</span><span class="p">,,</span><span class="kc">False</span>
</pre></div>
</div>
</div>
</div>
<div class="section" id="bug-reports">
<h2>Bug reports<a class="headerlink" href="#bug-reports" title="Permalink to this headline"></a></h2>
@@ -509,7 +633,7 @@ tags in your DMARC record, separated by commas.</p>
<h3>Elasticsearch and Kibana<a class="headerlink" href="#elasticsearch-and-kibana" title="Permalink to this headline"></a></h3>
<div class="admonition note">
<p class="first admonition-title">Note</p>
<p class="last">Splunk is also supported starting with <code class="docutils literal notranslate"><span class="pre">parsedmarc</span></code> 4.1.3</p>
<p class="last">Splunk is also supported starting with <code class="docutils literal notranslate"><span class="pre">parsedmarc</span></code> 4.3.0</p>
</div>
<p>To set up visual dashboards of DMARC data, install Elasticsearch and Kibana.</p>
<div class="admonition note">
@@ -671,10 +795,15 @@ the commercial <a class="reference external" href="https://www.elastic.co/produc
patterns. Select <code class="docutils literal notranslate"><span class="pre">dmarc_forensic</span></code> for the set of forensic objects, and
select <code class="docutils literal notranslate"><span class="pre">dmarc_aggregate</span></code> for the other saved objects, as shown below.</p>
<a class="reference external image-reference" href="_static/screenshots/index-pattern-conflicts.png"><img alt="A screenshot showing how to resolve index pattern conflicts after importing saved objects" class="align-center" src="_images/index-pattern-conflicts.png" /></a>
<div class="section" id="records-retention">
<h4>Records retention<a class="headerlink" href="#records-retention" title="Permalink to this headline"></a></h4>
<p>To prevent your indexes from growing too large, or to comply with records
retention regulations such as GDPR, you need to use <a class="reference external" href="https://www.elastic.co/blog/managing-time-based-indices-efficiently">time-based indexes</a>.</p>
</div>
</div>
<div class="section" id="splunk">
<h3>Splunk<a class="headerlink" href="#splunk" title="Permalink to this headline"></a></h3>
<p>Starting in version 4.1.3 <code class="docutils literal notranslate"><span class="pre">parsedmarc</span></code> supports sending aggregate and/or
<p>Starting in version 4.3.0 <code class="docutils literal notranslate"><span class="pre">parsedmarc</span></code> supports sending aggregate and/or
forensic DMARC data to a Splunk <a class="reference external" href="http://docs.splunk.com/Documentation/Splunk/latest/Data/AboutHEC">HTTP Event collector (HEC)</a>. Simply use the
following command line options, along with <code class="docutils literal notranslate"><span class="pre">--save-aggregate</span></code> and/or
<code class="docutils literal notranslate"><span class="pre">--save-forensic</span></code>:</p>
@@ -1090,42 +1219,6 @@ or bytes.</p>
</table>
</dd></dl>
<dl class="function">
<dt id="parsedmarc.human_timestamp_to_datetime">
<code class="descclassname">parsedmarc.</code><code class="descname">human_timestamp_to_datetime</code><span class="sig-paren">(</span><em>human_timestamp</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc.html#human_timestamp_to_datetime"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#parsedmarc.human_timestamp_to_datetime" title="Permalink to this definition"></a></dt>
<dd><p>Converts a human-readable timestamp into a Python <code class="docutils literal notranslate"><span class="pre">DateTime</span></code> object</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>human_timestamp</strong> (<em>str</em>) – A timestamp in <cite>YYYY-MM-DD HH:MM:SS`</cite> format</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body">The converted timestamp</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body">DateTime</td>
</tr>
</tbody>
</table>
</dd></dl>
<dl class="function">
<dt id="parsedmarc.human_timestamp_to_timestamp">
<code class="descclassname">parsedmarc.</code><code class="descname">human_timestamp_to_timestamp</code><span class="sig-paren">(</span><em>human_timestamp</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc.html#human_timestamp_to_timestamp"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#parsedmarc.human_timestamp_to_timestamp" title="Permalink to this definition"></a></dt>
<dd><p>Converts a human-readable timestamp into a into a UNIX timestamp</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>human_timestamp</strong> (<em>str</em>) – A timestamp in <cite>YYYY-MM-DD HH:MM:SS`</cite> format</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body">The converted timestamp</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body">float</td>
</tr>
</tbody>
</table>
</dd></dl>
<dl class="function">
<dt id="parsedmarc.parse_aggregate_report_file">
<code class="descclassname">parsedmarc.</code><code class="descname">parse_aggregate_report_file</code><span class="sig-paren">(</span><em>_input</em>, <em>nameservers=None</em>, <em>timeout=2.0</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc.html#parse_aggregate_report_file"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#parsedmarc.parse_aggregate_report_file" title="Permalink to this definition"></a></dt>
@@ -1181,7 +1274,7 @@ aggregate DMARC report</p>
<dl class="function">
<dt id="parsedmarc.parse_forensic_report">
<code class="descclassname">parsedmarc.</code><code class="descname">parse_forensic_report</code><span class="sig-paren">(</span><em>feedback_report</em>, <em>sample</em>, <em>sample_headers_only</em>, <em>msg_date</em>, <em>nameservers=None</em>, <em>timeout=2.0</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc.html#parse_forensic_report"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#parsedmarc.parse_forensic_report" title="Permalink to this definition"></a></dt>
<code class="descclassname">parsedmarc.</code><code class="descname">parse_forensic_report</code><span class="sig-paren">(</span><em>feedback_report</em>, <em>sample</em>, <em>msg_date</em>, <em>nameservers=None</em>, <em>timeout=2.0</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc.html#parse_forensic_report"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#parsedmarc.parse_forensic_report" title="Permalink to this definition"></a></dt>
<dd><p>Converts a DMARC forensic report and sample to a <code class="docutils literal notranslate"><span class="pre">OrderedDict</span></code></p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
@@ -1190,7 +1283,6 @@ aggregate DMARC report</p>
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
<li><strong>feedback_report</strong> (<em>str</em>) – A message’s feedback report as a string</li>
<li><strong>sample</strong> (<em>str</em>) – The RFC 822 headers or RFC 822 message sample</li>
<li><strong>sample_headers_only</strong> (<em>bool</em>) – Set true if the sample is only headers</li>
<li><strong>msg_date</strong> (<em>str</em>) – The message’s date header</li>
<li><strong>nameservers</strong> (<em>list</em>) – A list of one or more nameservers to use</li>
<li><strong>public DNS resolvers by default</strong><strong>)</strong> (<em>(</em><em>Cloudflare's</em>) – </li>
@@ -1434,6 +1526,330 @@ to a callback function</p>
</table>
</dd></dl>
<span class="target" id="module-parsedmarc.splunk"></span><dl class="class">
<dt id="parsedmarc.splunk.HECClient">
<em class="property">class </em><code class="descclassname">parsedmarc.splunk.</code><code class="descname">HECClient</code><span class="sig-paren">(</span><em>url</em>, <em>access_token</em>, <em>index</em>, <em>source='parsedmarc'</em>, <em>verify=True</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc/splunk.html#HECClient"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#parsedmarc.splunk.HECClient" title="Permalink to this definition"></a></dt>
<dd><p>A client for a Splunk HTTP Events Collector (HEC)</p>
<dl class="method">
<dt id="parsedmarc.splunk.HECClient.save_aggregate_reports_to_splunk">
<code class="descname">save_aggregate_reports_to_splunk</code><span class="sig-paren">(</span><em>aggregate_reports</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc/splunk.html#HECClient.save_aggregate_reports_to_splunk"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#parsedmarc.splunk.HECClient.save_aggregate_reports_to_splunk" title="Permalink to this definition"></a></dt>
<dd><p>Saves aggregate DMARC reports to Splunk</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first last simple">
<li><strong>aggregate_reports</strong> – A list of aggregate report dictionaries</li>
<li><strong>save in Splunk</strong> (<em>to</em>) – </li>
</ul>
</td>
</tr>
</tbody>
</table>
</dd></dl>
<dl class="method">
<dt id="parsedmarc.splunk.HECClient.save_forensic_reports_to_splunk">
<code class="descname">save_forensic_reports_to_splunk</code><span class="sig-paren">(</span><em>forensic_reports</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc/splunk.html#HECClient.save_forensic_reports_to_splunk"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#parsedmarc.splunk.HECClient.save_forensic_reports_to_splunk" title="Permalink to this definition"></a></dt>
<dd><p>Saves forensic DMARC reports to Splunk</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first last simple">
<li><strong>forensic_reports</strong> (<em>list</em>) – A list of forensic report dictionaries</li>
<li><strong>save in Splunk</strong> (<em>to</em>) – </li>
</ul>
</td>
</tr>
</tbody>
</table>
</dd></dl>
</dd></dl>
<dl class="exception">
<dt id="parsedmarc.splunk.SplunkError">
<em class="property">exception </em><code class="descclassname">parsedmarc.splunk.</code><code class="descname">SplunkError</code><a class="reference internal" href="_modules/parsedmarc/splunk.html#SplunkError"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#parsedmarc.splunk.SplunkError" title="Permalink to this definition"></a></dt>
<dd><p>Raised when a Splunk API error occurs</p>
</dd></dl>
<span class="target" id="module-parsedmarc.utils"></span><p>Utility functions that might be useful for other projects</p>
<dl class="exception">
<dt id="parsedmarc.utils.EmailParserError">
<em class="property">exception </em><code class="descclassname">parsedmarc.utils.</code><code class="descname">EmailParserError</code><a class="reference internal" href="_modules/parsedmarc/utils.html#EmailParserError"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#parsedmarc.utils.EmailParserError" title="Permalink to this definition"></a></dt>
<dd><p>Raised when an error parsing the email occurs</p>
</dd></dl>
<dl class="function">
<dt id="parsedmarc.utils.convert_outlook_msg">
<code class="descclassname">parsedmarc.utils.</code><code class="descname">convert_outlook_msg</code><span class="sig-paren">(</span><em>msg_bytes</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc/utils.html#convert_outlook_msg"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#parsedmarc.utils.convert_outlook_msg" title="Permalink to this definition"></a></dt>
<dd><p>Uses the <code class="docutils literal notranslate"><span class="pre">msgconvert</span></code> Perl utility to convert an Outlook MS file to
standard RFC 822 format</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>msg_bytes</strong> (<em>bytes</em>) – the content of the .msg file</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body">A RFC 822 string</td>
</tr>
</tbody>
</table>
</dd></dl>
<dl class="function">
<dt id="parsedmarc.utils.get_base_domain">
<code class="descclassname">parsedmarc.utils.</code><code class="descname">get_base_domain</code><span class="sig-paren">(</span><em>domain</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc/utils.html#get_base_domain"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#parsedmarc.utils.get_base_domain" title="Permalink to this definition"></a></dt>
<dd><p>Gets the base domain name for the given domain</p>
<div class="admonition note">
<p class="first admonition-title">Note</p>
<p>Results are based on a list of public domain suffixes at
<a class="reference external" href="https://publicsuffix.org/list/public_suffix_list.dat">https://publicsuffix.org/list/public_suffix_list.dat</a>.</p>
<p class="last">This file is saved to the current working directory,
where it is used as a cache file for 24 hours.</p>
</div>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>domain</strong> (<em>str</em>) – A domain or subdomain</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body">The base domain of the given domain</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body">str</td>
</tr>
</tbody>
</table>
</dd></dl>
<dl class="function">
<dt id="parsedmarc.utils.get_filename_safe_string">
<code class="descclassname">parsedmarc.utils.</code><code class="descname">get_filename_safe_string</code><span class="sig-paren">(</span><em>string</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc/utils.html#get_filename_safe_string"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#parsedmarc.utils.get_filename_safe_string" title="Permalink to this definition"></a></dt>
<dd><p>Converts a string to a string that is safe for a filename
:param string: A string to make safe for a filename
:type string: str</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Returns:</th><td class="field-body">A string safe for a filename</td>
</tr>
<tr class="field-even field"><th class="field-name">Return type:</th><td class="field-body">str</td>
</tr>
</tbody>
</table>
</dd></dl>
<dl class="function">
<dt id="parsedmarc.utils.get_ip_address_country">
<code class="descclassname">parsedmarc.utils.</code><code class="descname">get_ip_address_country</code><span class="sig-paren">(</span><em>ip_address</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc/utils.html#get_ip_address_country"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#parsedmarc.utils.get_ip_address_country" title="Permalink to this definition"></a></dt>
<dd><p>Uses the MaxMind Geolite2 Country database to return the ISO code for the
country associated with the given IPv4 or IPv6 address</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>ip_address</strong> (<em>str</em>) – The IP address to query for</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body">And ISO country code associated with the given IP address</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body">str</td>
</tr>
</tbody>
</table>
</dd></dl>
<dl class="function">
<dt id="parsedmarc.utils.get_ip_address_info">
<code class="descclassname">parsedmarc.utils.</code><code class="descname">get_ip_address_info</code><span class="sig-paren">(</span><em>ip_address</em>, <em>nameservers=None</em>, <em>timeout=2.0</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc/utils.html#get_ip_address_info"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#parsedmarc.utils.get_ip_address_info" title="Permalink to this definition"></a></dt>
<dd><p>Returns reverse DNS and country information for the given IP address</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
<li><strong>ip_address</strong> (<em>str</em>) – The IP address to check</li>
<li><strong>nameservers</strong> (<em>list</em>) – A list of one or more nameservers to use</li>
<li><strong>public DNS resolvers by default</strong><strong>)</strong> (<em>(</em><em>Cloudflare's</em>) – </li>
<li><strong>timeout</strong> (<em>float</em>) – Sets the DNS timeout in seconds</li>
</ul>
</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body"><p class="first"><code class="docutils literal notranslate"><span class="pre">ip_address</span></code>, <code class="docutils literal notranslate"><span class="pre">reverse_dns</span></code></p>
</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body"><p class="first last">OrderedDict</p>
</td>
</tr>
</tbody>
</table>
</dd></dl>
<dl class="function">
<dt id="parsedmarc.utils.get_reverse_dns">
<code class="descclassname">parsedmarc.utils.</code><code class="descname">get_reverse_dns</code><span class="sig-paren">(</span><em>ip_address</em>, <em>nameservers=None</em>, <em>timeout=2.0</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc/utils.html#get_reverse_dns"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#parsedmarc.utils.get_reverse_dns" title="Permalink to this definition"></a></dt>
<dd><p>Resolves an IP address to a hostname using a reverse DNS query</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
<li><strong>ip_address</strong> (<em>str</em>) – The IP address to resolve</li>
<li><strong>nameservers</strong> (<em>list</em>) – A list of one or more nameservers to use</li>
<li><strong>public DNS resolvers by default</strong><strong>)</strong> (<em>(</em><em>Cloudflare's</em>) – </li>
<li><strong>timeout</strong> (<em>float</em>) – Sets the DNS query timeout in seconds</li>
</ul>
</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body"><p class="first">The reverse DNS hostname (if any)</p>
</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body"><p class="first last">str</p>
</td>
</tr>
</tbody>
</table>
</dd></dl>
<dl class="function">
<dt id="parsedmarc.utils.human_timestamp_to_datetime">
<code class="descclassname">parsedmarc.utils.</code><code class="descname">human_timestamp_to_datetime</code><span class="sig-paren">(</span><em>human_timestamp</em>, <em>to_utc=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc/utils.html#human_timestamp_to_datetime"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#parsedmarc.utils.human_timestamp_to_datetime" title="Permalink to this definition"></a></dt>
<dd><p>Converts a human-readable timestamp into a Python <code class="docutils literal notranslate"><span class="pre">DateTime</span></code> object</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
<li><strong>human_timestamp</strong> (<em>str</em>) – A timestamp string</li>
<li><strong>to_utc</strong> (<em>bool</em>) – Convert the timestamp to UTC</li>
</ul>
</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body"><p class="first">The converted timestamp</p>
</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body"><p class="first last">DateTime</p>
</td>
</tr>
</tbody>
</table>
</dd></dl>
<dl class="function">
<dt id="parsedmarc.utils.human_timestamp_to_timestamp">
<code class="descclassname">parsedmarc.utils.</code><code class="descname">human_timestamp_to_timestamp</code><span class="sig-paren">(</span><em>human_timestamp</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc/utils.html#human_timestamp_to_timestamp"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#parsedmarc.utils.human_timestamp_to_timestamp" title="Permalink to this definition"></a></dt>
<dd><p>Converts a human-readable timestamp into a into a UNIX timestamp</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>human_timestamp</strong> (<em>str</em>) – A timestamp in <cite>YYYY-MM-DD HH:MM:SS`</cite> format</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body">The converted timestamp</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body">float</td>
</tr>
</tbody>
</table>
</dd></dl>
<dl class="function">
<dt id="parsedmarc.utils.is_outlook_msg">
<code class="descclassname">parsedmarc.utils.</code><code class="descname">is_outlook_msg</code><span class="sig-paren">(</span><em>content</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc/utils.html#is_outlook_msg"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#parsedmarc.utils.is_outlook_msg" title="Permalink to this definition"></a></dt>
<dd><p>Checks if the given content is a Outlook msg OLE file</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>content</strong> – Content to check</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body">A flag the indicates if a file is a Outlook MSG file</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body">bool</td>
</tr>
</tbody>
</table>
</dd></dl>
<dl class="function">
<dt id="parsedmarc.utils.parse_email">
<code class="descclassname">parsedmarc.utils.</code><code class="descname">parse_email</code><span class="sig-paren">(</span><em>data</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc/utils.html#parse_email"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#parsedmarc.utils.parse_email" title="Permalink to this definition"></a></dt>
<dd><p>A simplified email parser</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>data</strong> – The RFC 822 message string, or MSG binary</td>
</tr>
</tbody>
</table>
<p>Returns (dict): Parsed email data</p>
</dd></dl>
<dl class="function">
<dt id="parsedmarc.utils.query_dns">
<code class="descclassname">parsedmarc.utils.</code><code class="descname">query_dns</code><span class="sig-paren">(</span><em>domain</em>, <em>record_type</em>, <em>nameservers=None</em>, <em>timeout=2.0</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc/utils.html#query_dns"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#parsedmarc.utils.query_dns" title="Permalink to this definition"></a></dt>
<dd><p>Queries DNS</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
<li><strong>domain</strong> (<em>str</em>) – The domain or subdomain to query about</li>
<li><strong>record_type</strong> (<em>str</em>) – The record type to query for</li>
<li><strong>nameservers</strong> (<em>list</em>) – A list of one or more nameservers to use</li>
<li><strong>public DNS resolvers by default</strong><strong>)</strong> (<em>(</em><em>Cloudflare's</em>) – </li>
<li><strong>timeout</strong> (<em>float</em>) – Sets the DNS timeout in seconds</li>
</ul>
</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body"><p class="first">A list of answers</p>
</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body"><p class="first last">list</p>
</td>
</tr>
</tbody>
</table>
</dd></dl>
<dl class="function">
<dt id="parsedmarc.utils.timestamp_to_datetime">
<code class="descclassname">parsedmarc.utils.</code><code class="descname">timestamp_to_datetime</code><span class="sig-paren">(</span><em>timestamp</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc/utils.html#timestamp_to_datetime"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#parsedmarc.utils.timestamp_to_datetime" title="Permalink to this definition"></a></dt>
<dd><p>Converts a UNIX/DMARC timestamp to a Python <code class="docutils literal notranslate"><span class="pre">DateTime</span></code> object</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>timestamp</strong> (<em>int</em>) – The timestamp</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body">The converted timestamp as a Python <code class="docutils literal notranslate"><span class="pre">DateTime</span></code> object</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body">DateTime</td>
</tr>
</tbody>
</table>
</dd></dl>
<dl class="function">
<dt id="parsedmarc.utils.timestamp_to_human">
<code class="descclassname">parsedmarc.utils.</code><code class="descname">timestamp_to_human</code><span class="sig-paren">(</span><em>timestamp</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc/utils.html#timestamp_to_human"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#parsedmarc.utils.timestamp_to_human" title="Permalink to this definition"></a></dt>
<dd><p>Converts a UNIX/DMARC timestamp to a human-readable string</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>timestamp</strong> – The timestamp</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body">The converted timestamp in <code class="docutils literal notranslate"><span class="pre">YYYY-MM-DD</span> <span class="pre">HH:MM:SS</span></code> format</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body">str</td>
</tr>
</tbody>
</table>
</dd></dl>
<div class="toctree-wrapper compound">
</div>
</div>
@@ -1483,7 +1899,7 @@ to a callback function</p>
<script type="text/javascript">
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'./',
VERSION:'4.2.0k',
VERSION:'4.3.0',
LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
objects.inv
BIN
View File
Binary file not shown.
py-modindex.html
+13 -3
View File
@@ -8,7 +8,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Python Module Index &mdash; parsedmarc 4.2.0k documentation</title>
<title>Python Module Index &mdash; parsedmarc 4.3.0 documentation</title>
@@ -59,7 +59,7 @@
<div class="version">
4.2.0k
4.3.0
</div>
@@ -165,6 +165,16 @@
<td>&#160;&#160;&#160;
<a href="index.html#module-parsedmarc.elastic"><code class="xref">parsedmarc.elastic</code></a></td><td>
<em></em></td></tr>
<tr class="cg-1">
<td></td>
<td>&#160;&#160;&#160;
<a href="index.html#module-parsedmarc.splunk"><code class="xref">parsedmarc.splunk</code></a></td><td>
<em></em></td></tr>
<tr class="cg-1">
<td></td>
<td>&#160;&#160;&#160;
<a href="index.html#module-parsedmarc.utils"><code class="xref">parsedmarc.utils</code></a></td><td>
<em></em></td></tr>
</table>
@@ -202,7 +212,7 @@
<script type="text/javascript">
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'./',
VERSION:'4.2.0k',
VERSION:'4.3.0',
LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
search.html
+3 -3
View File
@@ -8,7 +8,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Search &mdash; parsedmarc 4.2.0k documentation</title>
<title>Search &mdash; parsedmarc 4.3.0 documentation</title>
@@ -56,7 +56,7 @@
<div class="version">
4.2.0k
4.3.0
</div>
@@ -190,7 +190,7 @@
<script type="text/javascript">
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'./',
VERSION:'4.2.0k',
VERSION:'4.3.0',
LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
searchindex.js
+1 -1
View File
File diff suppressed because one or more lines are too long