mirror of
https://github.com/domainaware/parsedmarc.git
synced 2026-02-21 16:56:24 +00:00
2007 lines
296 KiB
HTML
2007 lines
296 KiB
HTML
|
|
|
|
<!DOCTYPE html>
|
|
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
|
|
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
|
|
<head>
|
|
<meta charset="utf-8">
|
|
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
|
|
<title>parsedmarc — parsedmarc 6.3.5 documentation</title>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<script type="text/javascript" src="../_static/js/modernizr.min.js"></script>
|
|
|
|
|
|
<script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script>
|
|
<script type="text/javascript" src="../_static/jquery.js"></script>
|
|
<script type="text/javascript" src="../_static/underscore.js"></script>
|
|
<script type="text/javascript" src="../_static/doctools.js"></script>
|
|
<script type="text/javascript" src="../_static/language_data.js"></script>
|
|
|
|
<script type="text/javascript" src="../_static/js/theme.js"></script>
|
|
|
|
|
|
|
|
|
|
<link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
|
|
<link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
|
|
<link rel="index" title="Index" href="../genindex.html" />
|
|
<link rel="search" title="Search" href="../search.html" />
|
|
</head>
|
|
|
|
<body class="wy-body-for-nav">
|
|
|
|
|
|
<div class="wy-grid-for-nav">
|
|
|
|
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
|
|
<div class="wy-side-scroll">
|
|
<div class="wy-side-nav-search" >
|
|
|
|
|
|
|
|
<a href="../index.html" class="icon icon-home"> parsedmarc
|
|
|
|
|
|
|
|
</a>
|
|
|
|
|
|
|
|
|
|
<div class="version">
|
|
6.3.5
|
|
</div>
|
|
|
|
|
|
|
|
|
|
<div role="search">
|
|
<form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
|
|
<input type="text" name="q" placeholder="Search docs" />
|
|
<input type="hidden" name="check_keywords" value="yes" />
|
|
<input type="hidden" name="area" value="default" />
|
|
</form>
|
|
</div>
|
|
|
|
|
|
</div>
|
|
|
|
<div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<!-- Local TOC -->
|
|
<div class="local-toc"></div>
|
|
|
|
|
|
</div>
|
|
</div>
|
|
</nav>
|
|
|
|
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
|
|
|
|
|
|
<nav class="wy-nav-top" aria-label="top navigation">
|
|
|
|
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
|
|
<a href="../index.html">parsedmarc</a>
|
|
|
|
</nav>
|
|
|
|
|
|
<div class="wy-nav-content">
|
|
|
|
<div class="rst-content">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<div role="navigation" aria-label="breadcrumbs navigation">
|
|
|
|
<ul class="wy-breadcrumbs">
|
|
|
|
<li><a href="../index.html">Docs</a> »</li>
|
|
|
|
<li><a href="index.html">Module code</a> »</li>
|
|
|
|
<li>parsedmarc</li>
|
|
|
|
|
|
<li class="wy-breadcrumbs-aside">
|
|
|
|
</li>
|
|
|
|
</ul>
|
|
|
|
|
|
<hr/>
|
|
</div>
|
|
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
|
|
<div itemprop="articleBody">
|
|
|
|
<h1>Source code for parsedmarc</h1><div class="highlight"><pre>
|
|
<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
|
|
|
|
<span class="sd">"""A Python package for parsing DMARC reports"""</span>
|
|
|
|
<span class="kn">import</span> <span class="nn">logging</span>
|
|
<span class="kn">import</span> <span class="nn">os</span>
|
|
<span class="kn">import</span> <span class="nn">shutil</span>
|
|
<span class="kn">import</span> <span class="nn">xml.parsers.expat</span> <span class="k">as</span> <span class="nn">expat</span>
|
|
<span class="kn">import</span> <span class="nn">json</span>
|
|
<span class="kn">from</span> <span class="nn">datetime</span> <span class="k">import</span> <span class="n">datetime</span>
|
|
<span class="kn">from</span> <span class="nn">collections</span> <span class="k">import</span> <span class="n">OrderedDict</span>
|
|
<span class="kn">from</span> <span class="nn">io</span> <span class="k">import</span> <span class="n">BytesIO</span><span class="p">,</span> <span class="n">StringIO</span>
|
|
<span class="kn">from</span> <span class="nn">gzip</span> <span class="k">import</span> <span class="n">GzipFile</span>
|
|
<span class="kn">import</span> <span class="nn">zipfile</span>
|
|
<span class="kn">from</span> <span class="nn">csv</span> <span class="k">import</span> <span class="n">DictWriter</span>
|
|
<span class="kn">import</span> <span class="nn">re</span>
|
|
<span class="kn">from</span> <span class="nn">base64</span> <span class="k">import</span> <span class="n">b64decode</span>
|
|
<span class="kn">import</span> <span class="nn">binascii</span>
|
|
<span class="kn">import</span> <span class="nn">email</span>
|
|
<span class="kn">import</span> <span class="nn">tempfile</span>
|
|
<span class="kn">import</span> <span class="nn">socket</span>
|
|
<span class="kn">from</span> <span class="nn">email.mime.application</span> <span class="k">import</span> <span class="n">MIMEApplication</span>
|
|
<span class="kn">from</span> <span class="nn">email.mime.multipart</span> <span class="k">import</span> <span class="n">MIMEMultipart</span>
|
|
<span class="kn">from</span> <span class="nn">email.mime.text</span> <span class="k">import</span> <span class="n">MIMEText</span>
|
|
<span class="kn">import</span> <span class="nn">email.utils</span>
|
|
<span class="kn">import</span> <span class="nn">smtplib</span>
|
|
<span class="kn">from</span> <span class="nn">ssl</span> <span class="k">import</span> <span class="n">SSLError</span><span class="p">,</span> <span class="n">CertificateError</span><span class="p">,</span> <span class="n">create_default_context</span>
|
|
<span class="kn">import</span> <span class="nn">time</span>
|
|
|
|
<span class="kn">from</span> <span class="nn">expiringdict</span> <span class="k">import</span> <span class="n">ExpiringDict</span>
|
|
<span class="kn">import</span> <span class="nn">xmltodict</span>
|
|
<span class="kn">import</span> <span class="nn">imapclient</span>
|
|
<span class="kn">import</span> <span class="nn">imapclient.exceptions</span>
|
|
<span class="kn">import</span> <span class="nn">mailparser</span>
|
|
|
|
<span class="kn">from</span> <span class="nn">parsedmarc.utils</span> <span class="k">import</span> <span class="n">get_base_domain</span><span class="p">,</span> <span class="n">get_ip_address_info</span>
|
|
<span class="kn">from</span> <span class="nn">parsedmarc.utils</span> <span class="k">import</span> <span class="n">is_outlook_msg</span><span class="p">,</span> <span class="n">convert_outlook_msg</span>
|
|
<span class="kn">from</span> <span class="nn">parsedmarc.utils</span> <span class="k">import</span> <span class="n">timestamp_to_human</span><span class="p">,</span> <span class="n">human_timestamp_to_datetime</span>
|
|
<span class="kn">from</span> <span class="nn">parsedmarc.utils</span> <span class="k">import</span> <span class="n">parse_email</span>
|
|
|
|
<span class="n">__version__</span> <span class="o">=</span> <span class="s2">"6.3.5"</span>
|
|
|
|
<span class="n">logging</span><span class="o">.</span><span class="n">basicConfig</span><span class="p">(</span>
|
|
<span class="nb">format</span><span class="o">=</span><span class="s1">'</span><span class="si">%(levelname)8s</span><span class="s1">:</span><span class="si">%(filename)s</span><span class="s1">:</span><span class="si">%(lineno)d</span><span class="s1">:'</span>
|
|
<span class="s1">'</span><span class="si">%(message)s</span><span class="s1">'</span><span class="p">,</span>
|
|
<span class="n">datefmt</span><span class="o">=</span><span class="s1">'%Y-%m-</span><span class="si">%d</span><span class="s1">:%H:%M:%S'</span><span class="p">)</span>
|
|
|
|
<span class="n">logger</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s2">"parsedmarc"</span><span class="p">)</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"parsedmarc v</span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">__version__</span><span class="p">))</span>
|
|
|
|
<span class="n">feedback_report_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">"^([\w\-]+): (.+)$"</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">MULTILINE</span><span class="p">)</span>
|
|
<span class="n">xml_header_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">"^<\?xml .*?>"</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">MULTILINE</span><span class="p">)</span>
|
|
<span class="n">xml_schema_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">"</??xs:schema.*>"</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">MULTILINE</span><span class="p">)</span>
|
|
|
|
<span class="n">MAGIC_ZIP</span> <span class="o">=</span> <span class="sa">b</span><span class="s2">"</span><span class="se">\x50\x4B\x03\x04</span><span class="s2">"</span>
|
|
<span class="n">MAGIC_GZIP</span> <span class="o">=</span> <span class="sa">b</span><span class="s2">"</span><span class="se">\x1F\x8B</span><span class="s2">"</span>
|
|
<span class="n">MAGIC_XML</span> <span class="o">=</span> <span class="sa">b</span><span class="s2">"</span><span class="se">\x3c\x3f\x78\x6d\x6c\x20</span><span class="s2">"</span>
|
|
|
|
<span class="n">IP_ADDRESS_CACHE</span> <span class="o">=</span> <span class="n">ExpiringDict</span><span class="p">(</span><span class="n">max_len</span><span class="o">=</span><span class="mi">10000</span><span class="p">,</span> <span class="n">max_age_seconds</span><span class="o">=</span><span class="mi">1800</span><span class="p">)</span>
|
|
|
|
|
|
<div class="viewcode-block" id="ParserError"><a class="viewcode-back" href="../index.html#parsedmarc.ParserError">[docs]</a><span class="k">class</span> <span class="nc">ParserError</span><span class="p">(</span><span class="ne">RuntimeError</span><span class="p">):</span>
|
|
<span class="sd">"""Raised whenever the parser fails for some reason"""</span></div>
|
|
|
|
|
|
<div class="viewcode-block" id="IMAPError"><a class="viewcode-back" href="../index.html#parsedmarc.IMAPError">[docs]</a><span class="k">class</span> <span class="nc">IMAPError</span><span class="p">(</span><span class="ne">RuntimeError</span><span class="p">):</span>
|
|
<span class="sd">"""Raised when an IMAP error occurs"""</span></div>
|
|
|
|
|
|
<div class="viewcode-block" id="SMTPError"><a class="viewcode-back" href="../index.html#parsedmarc.SMTPError">[docs]</a><span class="k">class</span> <span class="nc">SMTPError</span><span class="p">(</span><span class="ne">RuntimeError</span><span class="p">):</span>
|
|
<span class="sd">"""Raised when a SMTP error occurs"""</span></div>
|
|
|
|
|
|
<div class="viewcode-block" id="InvalidDMARCReport"><a class="viewcode-back" href="../index.html#parsedmarc.InvalidDMARCReport">[docs]</a><span class="k">class</span> <span class="nc">InvalidDMARCReport</span><span class="p">(</span><span class="n">ParserError</span><span class="p">):</span>
|
|
<span class="sd">"""Raised when an invalid DMARC report is encountered"""</span></div>
|
|
|
|
|
|
<div class="viewcode-block" id="InvalidAggregateReport"><a class="viewcode-back" href="../index.html#parsedmarc.InvalidAggregateReport">[docs]</a><span class="k">class</span> <span class="nc">InvalidAggregateReport</span><span class="p">(</span><span class="n">InvalidDMARCReport</span><span class="p">):</span>
|
|
<span class="sd">"""Raised when an invalid DMARC aggregate report is encountered"""</span></div>
|
|
|
|
|
|
<div class="viewcode-block" id="InvalidForensicReport"><a class="viewcode-back" href="../index.html#parsedmarc.InvalidForensicReport">[docs]</a><span class="k">class</span> <span class="nc">InvalidForensicReport</span><span class="p">(</span><span class="n">InvalidDMARCReport</span><span class="p">):</span>
|
|
<span class="sd">"""Raised when an invalid DMARC forensic report is encountered"""</span></div>
|
|
|
|
|
|
<span class="k">def</span> <span class="nf">_parse_report_record</span><span class="p">(</span><span class="n">record</span><span class="p">,</span> <span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">dns_timeout</span><span class="o">=</span><span class="mf">2.0</span><span class="p">,</span>
|
|
<span class="n">parallel</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
|
|
<span class="sd">"""</span>
|
|
<span class="sd"> Converts a record from a DMARC aggregate report into a more consistent</span>
|
|
<span class="sd"> format</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> record (OrderedDict): The record to convert</span>
|
|
<span class="sd"> nameservers (list): A list of one or more nameservers to use</span>
|
|
<span class="sd"> (Cloudflare's public DNS resolvers by default)</span>
|
|
<span class="sd"> dns_timeout (float): Sets the DNS timeout in seconds</span>
|
|
|
|
<span class="sd"> Returns:</span>
|
|
<span class="sd"> OrderedDict: The converted record</span>
|
|
<span class="sd"> """</span>
|
|
<span class="k">if</span> <span class="n">nameservers</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">nameservers</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"1.1.1.1"</span><span class="p">,</span> <span class="s2">"1.0.0.1"</span><span class="p">,</span>
|
|
<span class="s2">"2606:4700:4700::1111"</span><span class="p">,</span> <span class="s2">"2606:4700:4700::1001"</span><span class="p">,</span>
|
|
<span class="p">]</span>
|
|
<span class="n">record</span> <span class="o">=</span> <span class="n">record</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
|
<span class="n">new_record</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">()</span>
|
|
<span class="n">new_record_source</span> <span class="o">=</span> <span class="n">get_ip_address_info</span><span class="p">(</span><span class="n">record</span><span class="p">[</span><span class="s2">"row"</span><span class="p">][</span><span class="s2">"source_ip"</span><span class="p">],</span>
|
|
<span class="n">cache</span><span class="o">=</span><span class="n">IP_ADDRESS_CACHE</span><span class="p">,</span>
|
|
<span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
|
|
<span class="n">timeout</span><span class="o">=</span><span class="n">dns_timeout</span><span class="p">,</span>
|
|
<span class="n">parallel</span><span class="o">=</span><span class="n">parallel</span><span class="p">)</span>
|
|
<span class="n">new_record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">]</span> <span class="o">=</span> <span class="n">new_record_source</span>
|
|
<span class="n">new_record</span><span class="p">[</span><span class="s2">"count"</span><span class="p">]</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">record</span><span class="p">[</span><span class="s2">"row"</span><span class="p">][</span><span class="s2">"count"</span><span class="p">])</span>
|
|
<span class="n">policy_evaluated</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"row"</span><span class="p">][</span><span class="s2">"policy_evaluated"</span><span class="p">]</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
|
<span class="n">new_policy_evaluated</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">([(</span><span class="s2">"disposition"</span><span class="p">,</span> <span class="s2">"none"</span><span class="p">),</span>
|
|
<span class="p">(</span><span class="s2">"dkim"</span><span class="p">,</span> <span class="s2">"fail"</span><span class="p">),</span>
|
|
<span class="p">(</span><span class="s2">"spf"</span><span class="p">,</span> <span class="s2">"fail"</span><span class="p">),</span>
|
|
<span class="p">(</span><span class="s2">"policy_override_reasons"</span><span class="p">,</span> <span class="p">[])</span>
|
|
<span class="p">])</span>
|
|
<span class="k">if</span> <span class="s2">"disposition"</span> <span class="ow">in</span> <span class="n">policy_evaluated</span><span class="p">:</span>
|
|
<span class="n">new_policy_evaluated</span><span class="p">[</span><span class="s2">"disposition"</span><span class="p">]</span> <span class="o">=</span> <span class="n">policy_evaluated</span><span class="p">[</span><span class="s2">"disposition"</span><span class="p">]</span>
|
|
<span class="k">if</span> <span class="n">new_policy_evaluated</span><span class="p">[</span><span class="s2">"disposition"</span><span class="p">]</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s2">"pass"</span><span class="p">:</span>
|
|
<span class="n">new_policy_evaluated</span><span class="p">[</span><span class="s2">"disposition"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"none"</span>
|
|
<span class="k">if</span> <span class="s2">"dkim"</span> <span class="ow">in</span> <span class="n">policy_evaluated</span><span class="p">:</span>
|
|
<span class="n">new_policy_evaluated</span><span class="p">[</span><span class="s2">"dkim"</span><span class="p">]</span> <span class="o">=</span> <span class="n">policy_evaluated</span><span class="p">[</span><span class="s2">"dkim"</span><span class="p">]</span>
|
|
<span class="k">if</span> <span class="s2">"spf"</span> <span class="ow">in</span> <span class="n">policy_evaluated</span><span class="p">:</span>
|
|
<span class="n">new_policy_evaluated</span><span class="p">[</span><span class="s2">"spf"</span><span class="p">]</span> <span class="o">=</span> <span class="n">policy_evaluated</span><span class="p">[</span><span class="s2">"spf"</span><span class="p">]</span>
|
|
<span class="n">reasons</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="n">spf_aligned</span> <span class="o">=</span> <span class="n">policy_evaluated</span><span class="p">[</span><span class="s2">"spf"</span><span class="p">]</span> <span class="o">==</span> <span class="s2">"pass"</span>
|
|
<span class="n">dkim_aligned</span> <span class="o">=</span> <span class="n">policy_evaluated</span><span class="p">[</span><span class="s2">"dkim"</span><span class="p">]</span> <span class="o">==</span> <span class="s2">"pass"</span>
|
|
<span class="n">dmarc_aligned</span> <span class="o">=</span> <span class="n">spf_aligned</span> <span class="ow">or</span> <span class="n">dkim_aligned</span>
|
|
<span class="n">new_record</span><span class="p">[</span><span class="s2">"alignment"</span><span class="p">]</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
|
|
<span class="n">new_record</span><span class="p">[</span><span class="s2">"alignment"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">]</span> <span class="o">=</span> <span class="n">spf_aligned</span>
|
|
<span class="n">new_record</span><span class="p">[</span><span class="s2">"alignment"</span><span class="p">][</span><span class="s2">"dkim"</span><span class="p">]</span> <span class="o">=</span> <span class="n">dkim_aligned</span>
|
|
<span class="n">new_record</span><span class="p">[</span><span class="s2">"alignment"</span><span class="p">][</span><span class="s2">"dmarc"</span><span class="p">]</span> <span class="o">=</span> <span class="n">dmarc_aligned</span>
|
|
<span class="k">if</span> <span class="s2">"reason"</span> <span class="ow">in</span> <span class="n">policy_evaluated</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">policy_evaluated</span><span class="p">[</span><span class="s2">"reason"</span><span class="p">])</span> <span class="o">==</span> <span class="nb">list</span><span class="p">:</span>
|
|
<span class="n">reasons</span> <span class="o">=</span> <span class="n">policy_evaluated</span><span class="p">[</span><span class="s2">"reason"</span><span class="p">]</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">reasons</span> <span class="o">=</span> <span class="p">[</span><span class="n">policy_evaluated</span><span class="p">[</span><span class="s2">"reason"</span><span class="p">]]</span>
|
|
<span class="k">for</span> <span class="n">reason</span> <span class="ow">in</span> <span class="n">reasons</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="s2">"comment"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">reason</span><span class="p">:</span>
|
|
<span class="n">reason</span><span class="p">[</span><span class="s2">"comment"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="n">new_policy_evaluated</span><span class="p">[</span><span class="s2">"policy_override_reasons"</span><span class="p">]</span> <span class="o">=</span> <span class="n">reasons</span>
|
|
<span class="n">new_record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">]</span> <span class="o">=</span> <span class="n">new_policy_evaluated</span>
|
|
<span class="n">new_record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">]</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
|
<span class="n">new_record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">]</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">([(</span><span class="s2">"dkim"</span><span class="p">,</span> <span class="p">[]),</span> <span class="p">(</span><span class="s2">"spf"</span><span class="p">,</span> <span class="p">[])])</span>
|
|
<span class="k">if</span> <span class="n">record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">auth_results</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">]</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
|
<span class="k">if</span> <span class="s2">"spf"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">auth_results</span><span class="p">:</span>
|
|
<span class="n">auth_results</span><span class="p">[</span><span class="s2">"spf"</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="k">if</span> <span class="s2">"dkim"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">auth_results</span><span class="p">:</span>
|
|
<span class="n">auth_results</span><span class="p">[</span><span class="s2">"dkim"</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">auth_results</span> <span class="o">=</span> <span class="n">new_record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">]</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
|
|
|
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">auth_results</span><span class="p">[</span><span class="s2">"dkim"</span><span class="p">])</span> <span class="o">!=</span> <span class="nb">list</span><span class="p">:</span>
|
|
<span class="n">auth_results</span><span class="p">[</span><span class="s2">"dkim"</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="n">auth_results</span><span class="p">[</span><span class="s2">"dkim"</span><span class="p">]]</span>
|
|
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">auth_results</span><span class="p">[</span><span class="s2">"dkim"</span><span class="p">]:</span>
|
|
<span class="k">if</span> <span class="s2">"domain"</span> <span class="ow">in</span> <span class="n">result</span> <span class="ow">and</span> <span class="n">result</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">new_result</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">([(</span><span class="s2">"domain"</span><span class="p">,</span> <span class="n">result</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">])])</span>
|
|
<span class="k">if</span> <span class="s2">"selector"</span> <span class="ow">in</span> <span class="n">result</span> <span class="ow">and</span> <span class="n">result</span><span class="p">[</span><span class="s2">"selector"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">new_result</span><span class="p">[</span><span class="s2">"selector"</span><span class="p">]</span> <span class="o">=</span> <span class="n">result</span><span class="p">[</span><span class="s2">"selector"</span><span class="p">]</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">new_result</span><span class="p">[</span><span class="s2">"selector"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"none"</span>
|
|
<span class="k">if</span> <span class="s2">"result"</span> <span class="ow">in</span> <span class="n">result</span> <span class="ow">and</span> <span class="n">result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">new_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">]</span> <span class="o">=</span> <span class="n">result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">]</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">new_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"none"</span>
|
|
<span class="n">new_record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">][</span><span class="s2">"dkim"</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">new_result</span><span class="p">)</span>
|
|
|
|
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">auth_results</span><span class="p">[</span><span class="s2">"spf"</span><span class="p">])</span> <span class="o">!=</span> <span class="nb">list</span><span class="p">:</span>
|
|
<span class="n">auth_results</span><span class="p">[</span><span class="s2">"spf"</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="n">auth_results</span><span class="p">[</span><span class="s2">"spf"</span><span class="p">]]</span>
|
|
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">auth_results</span><span class="p">[</span><span class="s2">"spf"</span><span class="p">]:</span>
|
|
<span class="n">new_result</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">([(</span><span class="s2">"domain"</span><span class="p">,</span> <span class="n">result</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">])])</span>
|
|
<span class="k">if</span> <span class="s2">"scope"</span> <span class="ow">in</span> <span class="n">result</span> <span class="ow">and</span> <span class="n">result</span><span class="p">[</span><span class="s2">"scope"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">new_result</span><span class="p">[</span><span class="s2">"scope"</span><span class="p">]</span> <span class="o">=</span> <span class="n">result</span><span class="p">[</span><span class="s2">"scope"</span><span class="p">]</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">new_result</span><span class="p">[</span><span class="s2">"scope"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"mfrom"</span>
|
|
<span class="k">if</span> <span class="s2">"result"</span> <span class="ow">in</span> <span class="n">result</span> <span class="ow">and</span> <span class="n">result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">new_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">]</span> <span class="o">=</span> <span class="n">result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">]</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">new_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"none"</span>
|
|
<span class="n">new_record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">new_result</span><span class="p">)</span>
|
|
|
|
<span class="k">if</span> <span class="s2">"envelope_from"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">new_record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">]:</span>
|
|
<span class="n">envelope_from</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">auth_results</span><span class="p">[</span><span class="s2">"spf"</span><span class="p">])</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
|
<span class="n">envelope_from</span> <span class="o">=</span> <span class="n">new_record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">][</span><span class="o">-</span><span class="mi">1</span><span class="p">][</span><span class="s2">"domain"</span><span class="p">]</span>
|
|
<span class="k">if</span> <span class="n">envelope_from</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">envelope_from</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">envelope_from</span><span class="p">)</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
|
|
<span class="n">new_record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"envelope_from"</span><span class="p">]</span> <span class="o">=</span> <span class="n">envelope_from</span>
|
|
|
|
<span class="k">elif</span> <span class="n">new_record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"envelope_from"</span><span class="p">]</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">auth_results</span><span class="p">[</span><span class="s2">"spf"</span><span class="p">])</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
|
<span class="n">envelope_from</span> <span class="o">=</span> <span class="n">new_record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">][</span><span class="o">-</span><span class="mi">1</span><span class="p">][</span><span class="s2">"domain"</span><span class="p">]</span>
|
|
<span class="k">if</span> <span class="n">envelope_from</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">envelope_from</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">envelope_from</span><span class="p">)</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
|
|
<span class="n">new_record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"envelope_from"</span><span class="p">]</span> <span class="o">=</span> <span class="n">envelope_from</span>
|
|
|
|
<span class="n">envelope_to</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="k">if</span> <span class="s2">"envelope_to"</span> <span class="ow">in</span> <span class="n">new_record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">]:</span>
|
|
<span class="n">envelope_to</span> <span class="o">=</span> <span class="n">new_record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"envelope_to"</span><span class="p">]</span>
|
|
<span class="k">del</span> <span class="n">new_record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"envelope_to"</span><span class="p">]</span>
|
|
|
|
<span class="n">new_record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"envelope_to"</span><span class="p">]</span> <span class="o">=</span> <span class="n">envelope_to</span>
|
|
|
|
<span class="k">return</span> <span class="n">new_record</span>
|
|
|
|
|
|
<div class="viewcode-block" id="parse_aggregate_report_xml"><a class="viewcode-back" href="../index.html#parsedmarc.parse_aggregate_report_xml">[docs]</a><span class="k">def</span> <span class="nf">parse_aggregate_report_xml</span><span class="p">(</span><span class="n">xml</span><span class="p">,</span> <span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mf">2.0</span><span class="p">,</span>
|
|
<span class="n">parallel</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
|
|
<span class="sd">"""Parses a DMARC XML report string and returns a consistent OrderedDict</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> xml (str): A string of DMARC aggregate report XML</span>
|
|
<span class="sd"> nameservers (list): A list of one or more nameservers to use</span>
|
|
<span class="sd"> (Cloudflare's public DNS resolvers by default)</span>
|
|
<span class="sd"> timeout (float): Sets the DNS timeout in seconds</span>
|
|
<span class="sd"> parallel (bool): Parallel processing</span>
|
|
|
|
<span class="sd"> Returns:</span>
|
|
<span class="sd"> OrderedDict: The parsed aggregate DMARC report</span>
|
|
<span class="sd"> """</span>
|
|
<span class="n">errors</span> <span class="o">=</span> <span class="p">[]</span>
|
|
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">xmltodict</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">xml</span><span class="p">)[</span><span class="s2">"feedback"</span><span class="p">]</span>
|
|
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
|
<span class="n">errors</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
|
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="c1"># Replace XML header (sometimes they are invalid)</span>
|
|
<span class="n">xml</span> <span class="o">=</span> <span class="n">xml_header_regex</span><span class="o">.</span><span class="n">sub</span><span class="p">(</span><span class="s2">"<?xml version=</span><span class="se">\"</span><span class="s2">1.0</span><span class="se">\"</span><span class="s2">?>"</span><span class="p">,</span> <span class="n">xml</span><span class="p">)</span>
|
|
|
|
<span class="c1"># Remove invalid schema tags</span>
|
|
<span class="n">xml</span> <span class="o">=</span> <span class="n">xml_schema_regex</span><span class="o">.</span><span class="n">sub</span><span class="p">(</span><span class="s1">''</span><span class="p">,</span> <span class="n">xml</span><span class="p">)</span>
|
|
|
|
<span class="n">report</span> <span class="o">=</span> <span class="n">xmltodict</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">xml</span><span class="p">)[</span><span class="s2">"feedback"</span><span class="p">]</span>
|
|
<span class="n">report_metadata</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">]</span>
|
|
<span class="n">schema</span> <span class="o">=</span> <span class="s2">"draft"</span>
|
|
<span class="k">if</span> <span class="s2">"version"</span> <span class="ow">in</span> <span class="n">report</span><span class="p">:</span>
|
|
<span class="n">schema</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"version"</span><span class="p">]</span>
|
|
<span class="n">new_report</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">([(</span><span class="s2">"xml_schema"</span><span class="p">,</span> <span class="n">schema</span><span class="p">)])</span>
|
|
<span class="n">new_report_metadata</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">()</span>
|
|
<span class="k">if</span> <span class="n">report_metadata</span><span class="p">[</span><span class="s2">"org_name"</span><span class="p">]</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="n">report_metadata</span><span class="p">[</span><span class="s2">"email"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">report_metadata</span><span class="p">[</span><span class="s2">"org_name"</span><span class="p">]</span> <span class="o">=</span> <span class="n">report_metadata</span><span class="p">[</span>
|
|
<span class="s2">"email"</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">"@"</span><span class="p">)[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
|
|
<span class="n">org_name</span> <span class="o">=</span> <span class="n">report_metadata</span><span class="p">[</span><span class="s2">"org_name"</span><span class="p">]</span>
|
|
<span class="k">if</span> <span class="n">org_name</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">org_name</span> <span class="o">=</span> <span class="n">get_base_domain</span><span class="p">(</span><span class="n">org_name</span><span class="p">)</span>
|
|
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"org_name"</span><span class="p">]</span> <span class="o">=</span> <span class="n">org_name</span>
|
|
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"org_email"</span><span class="p">]</span> <span class="o">=</span> <span class="n">report_metadata</span><span class="p">[</span><span class="s2">"email"</span><span class="p">]</span>
|
|
<span class="n">extra</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="k">if</span> <span class="s2">"extra_contact_info"</span> <span class="ow">in</span> <span class="n">report_metadata</span><span class="p">:</span>
|
|
<span class="n">extra</span> <span class="o">=</span> <span class="n">report_metadata</span><span class="p">[</span><span class="s2">"extra_contact_info"</span><span class="p">]</span>
|
|
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"org_extra_contact_info"</span><span class="p">]</span> <span class="o">=</span> <span class="n">extra</span>
|
|
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"report_id"</span><span class="p">]</span> <span class="o">=</span> <span class="n">report_metadata</span><span class="p">[</span><span class="s2">"report_id"</span><span class="p">]</span>
|
|
<span class="n">report_id</span> <span class="o">=</span> <span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"report_id"</span><span class="p">]</span>
|
|
<span class="n">report_id</span> <span class="o">=</span> <span class="n">report_id</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"<"</span><span class="p">,</span>
|
|
<span class="s2">""</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">">"</span><span class="p">,</span> <span class="s2">""</span><span class="p">)</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">"@"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
|
|
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"report_id"</span><span class="p">]</span> <span class="o">=</span> <span class="n">report_id</span>
|
|
<span class="n">date_range</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">][</span><span class="s2">"date_range"</span><span class="p">]</span>
|
|
<span class="n">date_range</span><span class="p">[</span><span class="s2">"begin"</span><span class="p">]</span> <span class="o">=</span> <span class="n">timestamp_to_human</span><span class="p">(</span><span class="n">date_range</span><span class="p">[</span><span class="s2">"begin"</span><span class="p">])</span>
|
|
<span class="n">date_range</span><span class="p">[</span><span class="s2">"end"</span><span class="p">]</span> <span class="o">=</span> <span class="n">timestamp_to_human</span><span class="p">(</span><span class="n">date_range</span><span class="p">[</span><span class="s2">"end"</span><span class="p">])</span>
|
|
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">]</span> <span class="o">=</span> <span class="n">date_range</span><span class="p">[</span><span class="s2">"begin"</span><span class="p">]</span>
|
|
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">]</span> <span class="o">=</span> <span class="n">date_range</span><span class="p">[</span><span class="s2">"end"</span><span class="p">]</span>
|
|
<span class="k">if</span> <span class="s2">"error"</span> <span class="ow">in</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">]:</span>
|
|
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">][</span><span class="s2">"error"</span><span class="p">])</span> <span class="o">!=</span> <span class="nb">list</span><span class="p">:</span>
|
|
<span class="n">errors</span> <span class="o">=</span> <span class="p">[</span><span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">][</span><span class="s2">"error"</span><span class="p">]]</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">errors</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">][</span><span class="s2">"error"</span><span class="p">]</span>
|
|
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"errors"</span><span class="p">]</span> <span class="o">=</span> <span class="n">errors</span>
|
|
<span class="n">new_report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">]</span> <span class="o">=</span> <span class="n">new_report_metadata</span>
|
|
<span class="n">records</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="n">policy_published</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">]</span>
|
|
<span class="n">new_policy_published</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">()</span>
|
|
<span class="n">new_policy_published</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">]</span> <span class="o">=</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">]</span>
|
|
<span class="n">adkim</span> <span class="o">=</span> <span class="s2">"r"</span>
|
|
<span class="k">if</span> <span class="s2">"adkim"</span> <span class="ow">in</span> <span class="n">policy_published</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"adkim"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">adkim</span> <span class="o">=</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"adkim"</span><span class="p">]</span>
|
|
<span class="n">new_policy_published</span><span class="p">[</span><span class="s2">"adkim"</span><span class="p">]</span> <span class="o">=</span> <span class="n">adkim</span>
|
|
<span class="n">aspf</span> <span class="o">=</span> <span class="s2">"r"</span>
|
|
<span class="k">if</span> <span class="s2">"aspf"</span> <span class="ow">in</span> <span class="n">policy_published</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"aspf"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">aspf</span> <span class="o">=</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"aspf"</span><span class="p">]</span>
|
|
<span class="n">new_policy_published</span><span class="p">[</span><span class="s2">"aspf"</span><span class="p">]</span> <span class="o">=</span> <span class="n">aspf</span>
|
|
<span class="n">new_policy_published</span><span class="p">[</span><span class="s2">"p"</span><span class="p">]</span> <span class="o">=</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"p"</span><span class="p">]</span>
|
|
<span class="n">sp</span> <span class="o">=</span> <span class="n">new_policy_published</span><span class="p">[</span><span class="s2">"p"</span><span class="p">]</span>
|
|
<span class="k">if</span> <span class="s2">"sp"</span> <span class="ow">in</span> <span class="n">policy_published</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"sp"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">sp</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"sp"</span><span class="p">]</span>
|
|
<span class="n">new_policy_published</span><span class="p">[</span><span class="s2">"sp"</span><span class="p">]</span> <span class="o">=</span> <span class="n">sp</span>
|
|
<span class="n">pct</span> <span class="o">=</span> <span class="s2">"100"</span>
|
|
<span class="k">if</span> <span class="s2">"pct"</span> <span class="ow">in</span> <span class="n">policy_published</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"pct"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">pct</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"pct"</span><span class="p">]</span>
|
|
<span class="n">new_policy_published</span><span class="p">[</span><span class="s2">"pct"</span><span class="p">]</span> <span class="o">=</span> <span class="n">pct</span>
|
|
<span class="n">fo</span> <span class="o">=</span> <span class="s2">"0"</span>
|
|
<span class="k">if</span> <span class="s2">"fo"</span> <span class="ow">in</span> <span class="n">policy_published</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"fo"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">fo</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"fo"</span><span class="p">]</span>
|
|
<span class="n">new_policy_published</span><span class="p">[</span><span class="s2">"fo"</span><span class="p">]</span> <span class="o">=</span> <span class="n">fo</span>
|
|
<span class="n">new_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">]</span> <span class="o">=</span> <span class="n">new_policy_published</span>
|
|
|
|
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">"record"</span><span class="p">])</span> <span class="o">==</span> <span class="nb">list</span><span class="p">:</span>
|
|
<span class="k">for</span> <span class="n">record</span> <span class="ow">in</span> <span class="n">report</span><span class="p">[</span><span class="s2">"record"</span><span class="p">]:</span>
|
|
<span class="n">report_record</span> <span class="o">=</span> <span class="n">_parse_report_record</span><span class="p">(</span><span class="n">record</span><span class="p">,</span>
|
|
<span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
|
|
<span class="n">dns_timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">,</span>
|
|
<span class="n">parallel</span><span class="o">=</span><span class="n">parallel</span><span class="p">)</span>
|
|
<span class="n">records</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">report_record</span><span class="p">)</span>
|
|
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">report_record</span> <span class="o">=</span> <span class="n">_parse_report_record</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">"record"</span><span class="p">],</span>
|
|
<span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
|
|
<span class="n">dns_timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">,</span>
|
|
<span class="n">parallel</span><span class="o">=</span><span class="n">parallel</span><span class="p">)</span>
|
|
<span class="n">records</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">report_record</span><span class="p">)</span>
|
|
|
|
<span class="n">new_report</span><span class="p">[</span><span class="s2">"records"</span><span class="p">]</span> <span class="o">=</span> <span class="n">records</span>
|
|
|
|
<span class="k">return</span> <span class="n">new_report</span>
|
|
|
|
<span class="k">except</span> <span class="n">expat</span><span class="o">.</span><span class="n">ExpatError</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">InvalidAggregateReport</span><span class="p">(</span>
|
|
<span class="s2">"Invalid XML: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
|
|
|
<span class="k">except</span> <span class="ne">KeyError</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">InvalidAggregateReport</span><span class="p">(</span>
|
|
<span class="s2">"Missing field: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
|
<span class="k">except</span> <span class="ne">AttributeError</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">InvalidAggregateReport</span><span class="p">(</span><span class="s2">"Report missing required section"</span><span class="p">)</span>
|
|
|
|
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">InvalidAggregateReport</span><span class="p">(</span>
|
|
<span class="s2">"Unexpected error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
|
|
|
|
|
<div class="viewcode-block" id="extract_xml"><a class="viewcode-back" href="../index.html#parsedmarc.extract_xml">[docs]</a><span class="k">def</span> <span class="nf">extract_xml</span><span class="p">(</span><span class="n">input_</span><span class="p">):</span>
|
|
<span class="sd">"""</span>
|
|
<span class="sd"> Extracts xml from a zip or gzip file at the given path, file-like object,</span>
|
|
<span class="sd"> or bytes.</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> input_: A path to a file, a file like object, or bytes</span>
|
|
|
|
<span class="sd"> Returns:</span>
|
|
<span class="sd"> str: The extracted XML</span>
|
|
|
|
<span class="sd"> """</span>
|
|
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">input_</span><span class="p">)</span> <span class="o">==</span> <span class="nb">str</span><span class="p">:</span>
|
|
<span class="n">file_object</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="n">input_</span><span class="p">,</span> <span class="s2">"rb"</span><span class="p">)</span>
|
|
<span class="k">elif</span> <span class="nb">type</span><span class="p">(</span><span class="n">input_</span><span class="p">)</span> <span class="o">==</span> <span class="nb">bytes</span><span class="p">:</span>
|
|
<span class="n">file_object</span> <span class="o">=</span> <span class="n">BytesIO</span><span class="p">(</span><span class="n">input_</span><span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">file_object</span> <span class="o">=</span> <span class="n">input_</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">header</span> <span class="o">=</span> <span class="n">file_object</span><span class="o">.</span><span class="n">read</span><span class="p">(</span><span class="mi">6</span><span class="p">)</span>
|
|
<span class="n">file_object</span><span class="o">.</span><span class="n">seek</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">header</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="n">MAGIC_ZIP</span><span class="p">):</span>
|
|
<span class="n">_zip</span> <span class="o">=</span> <span class="n">zipfile</span><span class="o">.</span><span class="n">ZipFile</span><span class="p">(</span><span class="n">file_object</span><span class="p">)</span>
|
|
<span class="n">xml</span> <span class="o">=</span> <span class="n">_zip</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">_zip</span><span class="o">.</span><span class="n">namelist</span><span class="p">()[</span><span class="mi">0</span><span class="p">])</span><span class="o">.</span><span class="n">read</span><span class="p">()</span><span class="o">.</span><span class="n">decode</span><span class="p">()</span>
|
|
<span class="k">elif</span> <span class="n">header</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="n">MAGIC_GZIP</span><span class="p">):</span>
|
|
<span class="n">xml</span> <span class="o">=</span> <span class="n">GzipFile</span><span class="p">(</span><span class="n">fileobj</span><span class="o">=</span><span class="n">file_object</span><span class="p">)</span><span class="o">.</span><span class="n">read</span><span class="p">()</span><span class="o">.</span><span class="n">decode</span><span class="p">()</span>
|
|
<span class="k">elif</span> <span class="n">header</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="n">MAGIC_XML</span><span class="p">):</span>
|
|
<span class="n">xml</span> <span class="o">=</span> <span class="n">file_object</span><span class="o">.</span><span class="n">read</span><span class="p">()</span><span class="o">.</span><span class="n">decode</span><span class="p">()</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">file_object</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
|
|
<span class="k">raise</span> <span class="n">InvalidAggregateReport</span><span class="p">(</span><span class="s2">"Not a valid zip, gzip, or xml file"</span><span class="p">)</span>
|
|
|
|
<span class="n">file_object</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
|
|
|
|
<span class="k">except</span> <span class="ne">UnicodeDecodeError</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">InvalidAggregateReport</span><span class="p">(</span><span class="s2">"File objects must be opened in binary "</span>
|
|
<span class="s2">"(rb) mode"</span><span class="p">)</span>
|
|
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">InvalidAggregateReport</span><span class="p">(</span>
|
|
<span class="s2">"Invalid archive file: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
|
|
|
<span class="k">return</span> <span class="n">xml</span></div>
|
|
|
|
|
|
<div class="viewcode-block" id="parse_aggregate_report_file"><a class="viewcode-back" href="../index.html#parsedmarc.parse_aggregate_report_file">[docs]</a><span class="k">def</span> <span class="nf">parse_aggregate_report_file</span><span class="p">(</span><span class="n">_input</span><span class="p">,</span> <span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">dns_timeout</span><span class="o">=</span><span class="mf">2.0</span><span class="p">,</span>
|
|
<span class="n">parallel</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
|
|
<span class="sd">"""Parses a file at the given path, a file-like object. or bytes as a</span>
|
|
<span class="sd"> aggregate DMARC report</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> _input: A path to a file, a file like object, or bytes</span>
|
|
<span class="sd"> nameservers (list): A list of one or more nameservers to use</span>
|
|
<span class="sd"> (Cloudflare's public DNS resolvers by default)</span>
|
|
<span class="sd"> dns_timeout (float): Sets the DNS timeout in seconds</span>
|
|
<span class="sd"> parallel (bool): Parallel processing</span>
|
|
|
|
<span class="sd"> Returns:</span>
|
|
<span class="sd"> OrderedDict: The parsed DMARC aggregate report</span>
|
|
<span class="sd"> """</span>
|
|
<span class="n">xml</span> <span class="o">=</span> <span class="n">extract_xml</span><span class="p">(</span><span class="n">_input</span><span class="p">)</span>
|
|
|
|
<span class="k">return</span> <span class="n">parse_aggregate_report_xml</span><span class="p">(</span><span class="n">xml</span><span class="p">,</span>
|
|
<span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
|
|
<span class="n">timeout</span><span class="o">=</span><span class="n">dns_timeout</span><span class="p">,</span>
|
|
<span class="n">parallel</span><span class="o">=</span><span class="n">parallel</span><span class="p">)</span></div>
|
|
|
|
|
|
<div class="viewcode-block" id="parsed_aggregate_reports_to_csv"><a class="viewcode-back" href="../index.html#parsedmarc.parsed_aggregate_reports_to_csv">[docs]</a><span class="k">def</span> <span class="nf">parsed_aggregate_reports_to_csv</span><span class="p">(</span><span class="n">reports</span><span class="p">):</span>
|
|
<span class="sd">"""</span>
|
|
<span class="sd"> Converts one or more parsed aggregate reports to flat CSV format, including</span>
|
|
<span class="sd"> headers</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> reports: A parsed aggregate report or list of parsed aggregate reports</span>
|
|
|
|
<span class="sd"> Returns:</span>
|
|
<span class="sd"> str: Parsed aggregate report data in flat CSV format, including headers</span>
|
|
<span class="sd"> """</span>
|
|
|
|
<span class="k">def</span> <span class="nf">to_str</span><span class="p">(</span><span class="n">obj</span><span class="p">):</span>
|
|
<span class="k">return</span> <span class="nb">str</span><span class="p">(</span><span class="n">obj</span><span class="p">)</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
|
|
|
|
<span class="n">fields</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"xml_schema"</span><span class="p">,</span> <span class="s2">"org_name"</span><span class="p">,</span> <span class="s2">"org_email"</span><span class="p">,</span>
|
|
<span class="s2">"org_extra_contact_info"</span><span class="p">,</span> <span class="s2">"report_id"</span><span class="p">,</span> <span class="s2">"begin_date"</span><span class="p">,</span> <span class="s2">"end_date"</span><span class="p">,</span>
|
|
<span class="s2">"errors"</span><span class="p">,</span> <span class="s2">"domain"</span><span class="p">,</span> <span class="s2">"adkim"</span><span class="p">,</span> <span class="s2">"aspf"</span><span class="p">,</span> <span class="s2">"p"</span><span class="p">,</span> <span class="s2">"sp"</span><span class="p">,</span> <span class="s2">"pct"</span><span class="p">,</span> <span class="s2">"fo"</span><span class="p">,</span>
|
|
<span class="s2">"source_ip_address"</span><span class="p">,</span> <span class="s2">"source_country"</span><span class="p">,</span> <span class="s2">"source_reverse_dns"</span><span class="p">,</span>
|
|
<span class="s2">"source_base_domain"</span><span class="p">,</span> <span class="s2">"count"</span><span class="p">,</span> <span class="s2">"disposition"</span><span class="p">,</span> <span class="s2">"dkim_alignment"</span><span class="p">,</span>
|
|
<span class="s2">"spf_alignment"</span><span class="p">,</span> <span class="s2">"policy_override_reasons"</span><span class="p">,</span>
|
|
<span class="s2">"policy_override_comments"</span><span class="p">,</span> <span class="s2">"envelope_from"</span><span class="p">,</span> <span class="s2">"header_from"</span><span class="p">,</span>
|
|
<span class="s2">"envelope_to"</span><span class="p">,</span> <span class="s2">"dkim_domains"</span><span class="p">,</span> <span class="s2">"dkim_selectors"</span><span class="p">,</span> <span class="s2">"dkim_results"</span><span class="p">,</span>
|
|
<span class="s2">"spf_domains"</span><span class="p">,</span> <span class="s2">"spf_scopes"</span><span class="p">,</span> <span class="s2">"spf_results"</span><span class="p">]</span>
|
|
|
|
<span class="n">csv_file_object</span> <span class="o">=</span> <span class="n">StringIO</span><span class="p">(</span><span class="n">newline</span><span class="o">=</span><span class="s2">"</span><span class="se">\n</span><span class="s2">"</span><span class="p">)</span>
|
|
<span class="n">writer</span> <span class="o">=</span> <span class="n">DictWriter</span><span class="p">(</span><span class="n">csv_file_object</span><span class="p">,</span> <span class="n">fields</span><span class="p">)</span>
|
|
<span class="n">writer</span><span class="o">.</span><span class="n">writeheader</span><span class="p">()</span>
|
|
|
|
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">reports</span><span class="p">)</span> <span class="o">==</span> <span class="n">OrderedDict</span><span class="p">:</span>
|
|
<span class="n">reports</span> <span class="o">=</span> <span class="p">[</span><span class="n">reports</span><span class="p">]</span>
|
|
|
|
<span class="k">for</span> <span class="n">report</span> <span class="ow">in</span> <span class="n">reports</span><span class="p">:</span>
|
|
<span class="n">xml_schema</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"xml_schema"</span><span class="p">]</span>
|
|
<span class="n">org_name</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">][</span><span class="s2">"org_name"</span><span class="p">]</span>
|
|
<span class="n">org_email</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">][</span><span class="s2">"org_email"</span><span class="p">]</span>
|
|
<span class="n">org_extra_contact</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">][</span><span class="s2">"org_extra_contact_info"</span><span class="p">]</span>
|
|
<span class="n">report_id</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">][</span><span class="s2">"report_id"</span><span class="p">]</span>
|
|
<span class="n">begin_date</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">][</span><span class="s2">"begin_date"</span><span class="p">]</span>
|
|
<span class="n">end_date</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">][</span><span class="s2">"end_date"</span><span class="p">]</span>
|
|
<span class="n">errors</span> <span class="o">=</span> <span class="s2">"|"</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">][</span><span class="s2">"errors"</span><span class="p">])</span>
|
|
<span class="n">domain</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"domain"</span><span class="p">]</span>
|
|
<span class="n">adkim</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"adkim"</span><span class="p">]</span>
|
|
<span class="n">aspf</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"aspf"</span><span class="p">]</span>
|
|
<span class="n">p</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"p"</span><span class="p">]</span>
|
|
<span class="n">sp</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"sp"</span><span class="p">]</span>
|
|
<span class="n">pct</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"pct"</span><span class="p">]</span>
|
|
<span class="n">fo</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"fo"</span><span class="p">]</span>
|
|
|
|
<span class="n">report_dict</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="n">xml_schema</span><span class="o">=</span><span class="n">xml_schema</span><span class="p">,</span> <span class="n">org_name</span><span class="o">=</span><span class="n">org_name</span><span class="p">,</span>
|
|
<span class="n">org_email</span><span class="o">=</span><span class="n">org_email</span><span class="p">,</span>
|
|
<span class="n">org_extra_contact_info</span><span class="o">=</span><span class="n">org_extra_contact</span><span class="p">,</span>
|
|
<span class="n">report_id</span><span class="o">=</span><span class="n">report_id</span><span class="p">,</span> <span class="n">begin_date</span><span class="o">=</span><span class="n">begin_date</span><span class="p">,</span>
|
|
<span class="n">end_date</span><span class="o">=</span><span class="n">end_date</span><span class="p">,</span> <span class="n">errors</span><span class="o">=</span><span class="n">errors</span><span class="p">,</span> <span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span>
|
|
<span class="n">adkim</span><span class="o">=</span><span class="n">adkim</span><span class="p">,</span> <span class="n">aspf</span><span class="o">=</span><span class="n">aspf</span><span class="p">,</span> <span class="n">p</span><span class="o">=</span><span class="n">p</span><span class="p">,</span> <span class="n">sp</span><span class="o">=</span><span class="n">sp</span><span class="p">,</span> <span class="n">pct</span><span class="o">=</span><span class="n">pct</span><span class="p">,</span> <span class="n">fo</span><span class="o">=</span><span class="n">fo</span><span class="p">)</span>
|
|
|
|
<span class="k">for</span> <span class="n">record</span> <span class="ow">in</span> <span class="n">report</span><span class="p">[</span><span class="s2">"records"</span><span class="p">]:</span>
|
|
<span class="n">row</span> <span class="o">=</span> <span class="n">report_dict</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"source_ip_address"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"ip_address"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"source_country"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"country"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"source_reverse_dns"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"reverse_dns"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"source_base_domain"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"base_domain"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"count"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"count"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"disposition"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"disposition"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"spf_alignment"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"dkim_alignment"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"dkim"</span><span class="p">]</span>
|
|
<span class="n">policy_override_reasons</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span>
|
|
<span class="k">lambda</span> <span class="n">r</span><span class="p">:</span> <span class="n">r</span><span class="p">[</span><span class="s2">"type"</span><span class="p">],</span>
|
|
<span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">]</span>
|
|
<span class="p">[</span><span class="s2">"policy_override_reasons"</span><span class="p">]))</span>
|
|
<span class="n">policy_override_comments</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span>
|
|
<span class="k">lambda</span> <span class="n">r</span><span class="p">:</span> <span class="n">r</span><span class="p">[</span><span class="s2">"comment"</span><span class="p">]</span> <span class="ow">or</span> <span class="s2">"none"</span><span class="p">,</span>
|
|
<span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">]</span>
|
|
<span class="p">[</span><span class="s2">"policy_override_reasons"</span><span class="p">]))</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"policy_override_reasons"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">","</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
|
|
<span class="n">policy_override_reasons</span><span class="p">)</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"policy_override_comments"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"|"</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
|
|
<span class="n">policy_override_comments</span><span class="p">)</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"envelope_from"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"envelope_from"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"header_from"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"header_from"</span><span class="p">]</span>
|
|
<span class="n">envelope_to</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"envelope_to"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"envelope_to"</span><span class="p">]</span> <span class="o">=</span> <span class="n">envelope_to</span>
|
|
<span class="n">dkim_domains</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="n">dkim_selectors</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="n">dkim_results</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="k">for</span> <span class="n">dkim_result</span> <span class="ow">in</span> <span class="n">record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">][</span><span class="s2">"dkim"</span><span class="p">]:</span>
|
|
<span class="n">dkim_domains</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">])</span>
|
|
<span class="k">if</span> <span class="s2">"selector"</span> <span class="ow">in</span> <span class="n">dkim_result</span><span class="p">:</span>
|
|
<span class="n">dkim_selectors</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">"selector"</span><span class="p">])</span>
|
|
<span class="n">dkim_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">])</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"dkim_domains"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">","</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="n">to_str</span><span class="p">,</span> <span class="n">dkim_domains</span><span class="p">))</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"dkim_selectors"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">","</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="n">to_str</span><span class="p">,</span> <span class="n">dkim_selectors</span><span class="p">))</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"dkim_results"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">","</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="n">to_str</span><span class="p">,</span> <span class="n">dkim_results</span><span class="p">))</span>
|
|
<span class="n">spf_domains</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="n">spf_scopes</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="n">spf_results</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="k">for</span> <span class="n">spf_result</span> <span class="ow">in</span> <span class="n">record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">]:</span>
|
|
<span class="n">spf_domains</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">])</span>
|
|
<span class="n">spf_scopes</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">"scope"</span><span class="p">])</span>
|
|
<span class="n">spf_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">])</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"spf_domains"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">","</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="n">to_str</span><span class="p">,</span> <span class="n">spf_domains</span><span class="p">))</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"spf_scopes"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">","</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="n">to_str</span><span class="p">,</span> <span class="n">spf_scopes</span><span class="p">))</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"spf_results"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">","</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="n">to_str</span><span class="p">,</span> <span class="n">dkim_results</span><span class="p">))</span>
|
|
|
|
<span class="n">writer</span><span class="o">.</span><span class="n">writerow</span><span class="p">(</span><span class="n">row</span><span class="p">)</span>
|
|
<span class="n">csv_file_object</span><span class="o">.</span><span class="n">flush</span><span class="p">()</span>
|
|
|
|
<span class="k">return</span> <span class="n">csv_file_object</span><span class="o">.</span><span class="n">getvalue</span><span class="p">()</span></div>
|
|
|
|
|
|
<div class="viewcode-block" id="parse_forensic_report"><a class="viewcode-back" href="../index.html#parsedmarc.parse_forensic_report">[docs]</a><span class="k">def</span> <span class="nf">parse_forensic_report</span><span class="p">(</span><span class="n">feedback_report</span><span class="p">,</span> <span class="n">sample</span><span class="p">,</span> <span class="n">msg_date</span><span class="p">,</span>
|
|
<span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">dns_timeout</span><span class="o">=</span><span class="mf">2.0</span><span class="p">,</span>
|
|
<span class="n">strip_attachment_payloads</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
|
<span class="n">parallel</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
|
|
<span class="sd">"""</span>
|
|
<span class="sd"> Converts a DMARC forensic report and sample to a ``OrderedDict``</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> feedback_report (str): A message's feedback report as a string</span>
|
|
<span class="sd"> sample (str): The RFC 822 headers or RFC 822 message sample</span>
|
|
<span class="sd"> msg_date (str): The message's date header</span>
|
|
<span class="sd"> nameservers (list): A list of one or more nameservers to use</span>
|
|
<span class="sd"> (Cloudflare's public DNS resolvers by default)</span>
|
|
<span class="sd"> dns_timeout (float): Sets the DNS timeout in seconds</span>
|
|
<span class="sd"> strip_attachment_payloads (bool): Remove attachment payloads from</span>
|
|
<span class="sd"> forensic report results</span>
|
|
<span class="sd"> parallel (bool): Parallel processing</span>
|
|
|
|
<span class="sd"> Returns:</span>
|
|
<span class="sd"> OrderedDict: A parsed report and sample</span>
|
|
<span class="sd"> """</span>
|
|
<span class="n">delivery_results</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"delivered"</span><span class="p">,</span> <span class="s2">"spam"</span><span class="p">,</span> <span class="s2">"policy"</span><span class="p">,</span> <span class="s2">"reject"</span><span class="p">,</span> <span class="s2">"other"</span><span class="p">]</span>
|
|
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">parsed_report</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">()</span>
|
|
<span class="n">report_values</span> <span class="o">=</span> <span class="n">feedback_report_regex</span><span class="o">.</span><span class="n">findall</span><span class="p">(</span><span class="n">feedback_report</span><span class="p">)</span>
|
|
<span class="k">for</span> <span class="n">report_value</span> <span class="ow">in</span> <span class="n">report_values</span><span class="p">:</span>
|
|
<span class="n">key</span> <span class="o">=</span> <span class="n">report_value</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"-"</span><span class="p">,</span> <span class="s2">"_"</span><span class="p">)</span>
|
|
<span class="n">parsed_report</span><span class="p">[</span><span class="n">key</span><span class="p">]</span> <span class="o">=</span> <span class="n">report_value</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
|
|
|
|
<span class="k">if</span> <span class="s2">"arrival_date"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_report</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="n">msg_date</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">InvalidForensicReport</span><span class="p">(</span>
|
|
<span class="s2">"Forensic sample is not a valid email"</span><span class="p">)</span>
|
|
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"arrival_date"</span><span class="p">]</span> <span class="o">=</span> <span class="n">msg_date</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span>
|
|
|
|
<span class="k">if</span> <span class="s2">"version"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_report</span><span class="p">:</span>
|
|
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"version"</span><span class="p">]</span> <span class="o">=</span> <span class="mi">1</span>
|
|
|
|
<span class="k">if</span> <span class="s2">"user_agent"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_report</span><span class="p">:</span>
|
|
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"user_agent"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
|
|
|
<span class="k">if</span> <span class="s2">"delivery_result"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_report</span><span class="p">:</span>
|
|
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"delivery_result"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="k">for</span> <span class="n">delivery_result</span> <span class="ow">in</span> <span class="n">delivery_results</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="n">delivery_result</span> <span class="ow">in</span> <span class="n">parsed_report</span><span class="p">[</span><span class="s2">"delivery_result"</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">():</span>
|
|
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"delivery_result"</span><span class="p">]</span> <span class="o">=</span> <span class="n">delivery_result</span>
|
|
<span class="k">break</span>
|
|
<span class="k">if</span> <span class="n">parsed_report</span><span class="p">[</span><span class="s2">"delivery_result"</span><span class="p">]</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">delivery_results</span><span class="p">:</span>
|
|
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"delivery_result"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"other"</span>
|
|
|
|
<span class="n">arrival_utc</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span>
|
|
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"arrival_date"</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
|
<span class="n">arrival_utc</span> <span class="o">=</span> <span class="n">arrival_utc</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%S"</span><span class="p">)</span>
|
|
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"arrival_date_utc"</span><span class="p">]</span> <span class="o">=</span> <span class="n">arrival_utc</span>
|
|
|
|
<span class="n">ip_address</span> <span class="o">=</span> <span class="n">parsed_report</span><span class="p">[</span><span class="s2">"source_ip"</span><span class="p">]</span>
|
|
<span class="n">parsed_report_source</span> <span class="o">=</span> <span class="n">get_ip_address_info</span><span class="p">(</span><span class="n">ip_address</span><span class="p">,</span>
|
|
<span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
|
|
<span class="n">timeout</span><span class="o">=</span><span class="n">dns_timeout</span><span class="p">,</span>
|
|
<span class="n">parallel</span><span class="o">=</span><span class="n">parallel</span><span class="p">)</span>
|
|
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">]</span> <span class="o">=</span> <span class="n">parsed_report_source</span>
|
|
<span class="k">del</span> <span class="n">parsed_report</span><span class="p">[</span><span class="s2">"source_ip"</span><span class="p">]</span>
|
|
|
|
<span class="k">if</span> <span class="s2">"identity_alignment"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_report</span><span class="p">:</span>
|
|
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"authentication_mechanisms"</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="k">elif</span> <span class="n">parsed_report</span><span class="p">[</span><span class="s2">"identity_alignment"</span><span class="p">]</span> <span class="o">==</span> <span class="s2">"none"</span><span class="p">:</span>
|
|
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"authentication_mechanisms"</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="k">del</span> <span class="n">parsed_report</span><span class="p">[</span><span class="s2">"identity_alignment"</span><span class="p">]</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">auth_mechanisms</span> <span class="o">=</span> <span class="n">parsed_report</span><span class="p">[</span><span class="s2">"identity_alignment"</span><span class="p">]</span>
|
|
<span class="n">auth_mechanisms</span> <span class="o">=</span> <span class="n">auth_mechanisms</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">","</span><span class="p">)</span>
|
|
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"authentication_mechanisms"</span><span class="p">]</span> <span class="o">=</span> <span class="n">auth_mechanisms</span>
|
|
<span class="k">del</span> <span class="n">parsed_report</span><span class="p">[</span><span class="s2">"identity_alignment"</span><span class="p">]</span>
|
|
|
|
<span class="k">if</span> <span class="s2">"auth_failure"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_report</span><span class="p">:</span>
|
|
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"auth_failure"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"dmarc"</span>
|
|
<span class="n">auth_failure</span> <span class="o">=</span> <span class="n">parsed_report</span><span class="p">[</span><span class="s2">"auth_failure"</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">","</span><span class="p">)</span>
|
|
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"auth_failure"</span><span class="p">]</span> <span class="o">=</span> <span class="n">auth_failure</span>
|
|
|
|
<span class="n">optional_fields</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"original_envelope_id"</span><span class="p">,</span> <span class="s2">"dkim_domain"</span><span class="p">,</span>
|
|
<span class="s2">"original_mail_from"</span><span class="p">,</span> <span class="s2">"original_rcpt_to"</span><span class="p">]</span>
|
|
<span class="k">for</span> <span class="n">optional_field</span> <span class="ow">in</span> <span class="n">optional_fields</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="n">optional_field</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_report</span><span class="p">:</span>
|
|
<span class="n">parsed_report</span><span class="p">[</span><span class="n">optional_field</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
|
|
|
<span class="n">parsed_sample</span> <span class="o">=</span> <span class="n">parse_email</span><span class="p">(</span>
|
|
<span class="n">sample</span><span class="p">,</span>
|
|
<span class="n">strip_attachment_payloads</span><span class="o">=</span><span class="n">strip_attachment_payloads</span><span class="p">)</span>
|
|
|
|
<span class="k">if</span> <span class="s2">"reported_domain"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_report</span><span class="p">:</span>
|
|
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"reported_domain"</span><span class="p">]</span> <span class="o">=</span> <span class="n">parsed_sample</span><span class="p">[</span><span class="s2">"from"</span><span class="p">][</span><span class="s2">"domain"</span><span class="p">]</span>
|
|
|
|
<span class="n">sample_headers_only</span> <span class="o">=</span> <span class="kc">False</span>
|
|
<span class="n">number_of_attachments</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">parsed_sample</span><span class="p">[</span><span class="s2">"attachments"</span><span class="p">])</span>
|
|
<span class="k">if</span> <span class="n">number_of_attachments</span> <span class="o"><</span> <span class="mi">1</span> <span class="ow">and</span> <span class="n">parsed_sample</span><span class="p">[</span><span class="s2">"body"</span><span class="p">]</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">sample_headers_only</span> <span class="o">=</span> <span class="kc">True</span>
|
|
<span class="k">if</span> <span class="n">sample_headers_only</span> <span class="ow">and</span> <span class="n">parsed_sample</span><span class="p">[</span><span class="s2">"has_defects"</span><span class="p">]:</span>
|
|
<span class="k">del</span> <span class="n">parsed_sample</span><span class="p">[</span><span class="s2">"defects"</span><span class="p">]</span>
|
|
<span class="k">del</span> <span class="n">parsed_sample</span><span class="p">[</span><span class="s2">"defects_categories"</span><span class="p">]</span>
|
|
<span class="k">del</span> <span class="n">parsed_sample</span><span class="p">[</span><span class="s2">"has_defects"</span><span class="p">]</span>
|
|
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"sample_headers_only"</span><span class="p">]</span> <span class="o">=</span> <span class="n">sample_headers_only</span>
|
|
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"sample"</span><span class="p">]</span> <span class="o">=</span> <span class="n">sample</span>
|
|
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">]</span> <span class="o">=</span> <span class="n">parsed_sample</span>
|
|
|
|
<span class="k">return</span> <span class="n">parsed_report</span>
|
|
|
|
<span class="k">except</span> <span class="ne">KeyError</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">InvalidForensicReport</span><span class="p">(</span><span class="s2">"Missing value: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
|
<span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
|
|
|
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">InvalidForensicReport</span><span class="p">(</span>
|
|
<span class="s2">"Unexpected error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
|
|
|
|
|
<div class="viewcode-block" id="parsed_forensic_reports_to_csv"><a class="viewcode-back" href="../index.html#parsedmarc.parsed_forensic_reports_to_csv">[docs]</a><span class="k">def</span> <span class="nf">parsed_forensic_reports_to_csv</span><span class="p">(</span><span class="n">reports</span><span class="p">):</span>
|
|
<span class="sd">"""</span>
|
|
<span class="sd"> Converts one or more parsed forensic reports to flat CSV format, including</span>
|
|
<span class="sd"> headers</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> reports: A parsed forensic report or list of parsed forensic reports</span>
|
|
|
|
<span class="sd"> Returns:</span>
|
|
<span class="sd"> str: Parsed forensic report data in flat CSV format, including headers</span>
|
|
<span class="sd"> """</span>
|
|
<span class="n">fields</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"feedback_type"</span><span class="p">,</span> <span class="s2">"user_agent"</span><span class="p">,</span> <span class="s2">"version"</span><span class="p">,</span> <span class="s2">"original_envelope_id"</span><span class="p">,</span>
|
|
<span class="s2">"original_mail_from"</span><span class="p">,</span> <span class="s2">"original_rcpt_to"</span><span class="p">,</span> <span class="s2">"arrival_date"</span><span class="p">,</span>
|
|
<span class="s2">"arrival_date_utc"</span><span class="p">,</span> <span class="s2">"subject"</span><span class="p">,</span> <span class="s2">"message_id"</span><span class="p">,</span>
|
|
<span class="s2">"authentication_results"</span><span class="p">,</span> <span class="s2">"dkim_domain"</span><span class="p">,</span> <span class="s2">"source_ip_address"</span><span class="p">,</span>
|
|
<span class="s2">"source_country"</span><span class="p">,</span> <span class="s2">"source_reverse_dns"</span><span class="p">,</span> <span class="s2">"source_base_domain"</span><span class="p">,</span>
|
|
<span class="s2">"delivery_result"</span><span class="p">,</span> <span class="s2">"auth_failure"</span><span class="p">,</span> <span class="s2">"reported_domain"</span><span class="p">,</span>
|
|
<span class="s2">"authentication_mechanisms"</span><span class="p">,</span> <span class="s2">"sample_headers_only"</span><span class="p">]</span>
|
|
|
|
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">reports</span><span class="p">)</span> <span class="o">==</span> <span class="n">OrderedDict</span><span class="p">:</span>
|
|
<span class="n">reports</span> <span class="o">=</span> <span class="p">[</span><span class="n">reports</span><span class="p">]</span>
|
|
<span class="n">csv_file</span> <span class="o">=</span> <span class="n">StringIO</span><span class="p">()</span>
|
|
<span class="n">csv_writer</span> <span class="o">=</span> <span class="n">DictWriter</span><span class="p">(</span><span class="n">csv_file</span><span class="p">,</span> <span class="n">fieldnames</span><span class="o">=</span><span class="n">fields</span><span class="p">)</span>
|
|
<span class="n">csv_writer</span><span class="o">.</span><span class="n">writeheader</span><span class="p">()</span>
|
|
<span class="k">for</span> <span class="n">report</span> <span class="ow">in</span> <span class="n">reports</span><span class="p">:</span>
|
|
<span class="n">row</span> <span class="o">=</span> <span class="n">report</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"source_ip_address"</span><span class="p">]</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"ip_address"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"source_reverse_dns"</span><span class="p">]</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"reverse_dns"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"source_base_domain"</span><span class="p">]</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"base_domain"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"source_country"</span><span class="p">]</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"country"</span><span class="p">]</span>
|
|
<span class="k">del</span> <span class="n">row</span><span class="p">[</span><span class="s2">"source"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"subject"</span><span class="p">]</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"subject"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"auth_failure"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">","</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">"auth_failure"</span><span class="p">])</span>
|
|
<span class="n">authentication_mechanisms</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"authentication_mechanisms"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"authentication_mechanisms"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">","</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
|
|
<span class="n">authentication_mechanisms</span><span class="p">)</span>
|
|
<span class="k">del</span> <span class="n">row</span><span class="p">[</span><span class="s2">"sample"</span><span class="p">]</span>
|
|
<span class="k">del</span> <span class="n">row</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">]</span>
|
|
<span class="n">csv_writer</span><span class="o">.</span><span class="n">writerow</span><span class="p">(</span><span class="n">row</span><span class="p">)</span>
|
|
|
|
<span class="k">return</span> <span class="n">csv_file</span><span class="o">.</span><span class="n">getvalue</span><span class="p">()</span></div>
|
|
|
|
|
|
<div class="viewcode-block" id="parse_report_email"><a class="viewcode-back" href="../index.html#parsedmarc.parse_report_email">[docs]</a><span class="k">def</span> <span class="nf">parse_report_email</span><span class="p">(</span><span class="n">input_</span><span class="p">,</span> <span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">dns_timeout</span><span class="o">=</span><span class="mf">2.0</span><span class="p">,</span>
|
|
<span class="n">strip_attachment_payloads</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">parallel</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
|
|
<span class="sd">"""</span>
|
|
<span class="sd"> Parses a DMARC report from an email</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> input_: An emailed DMARC report in RFC 822 format, as bytes or a string</span>
|
|
<span class="sd"> nameservers (list): A list of one or more nameservers to use</span>
|
|
<span class="sd"> dns_timeout (float): Sets the DNS timeout in seconds</span>
|
|
<span class="sd"> strip_attachment_payloads (bool): Remove attachment payloads from</span>
|
|
<span class="sd"> forensic report results</span>
|
|
<span class="sd"> parallel (bool): Parallel processing</span>
|
|
|
|
<span class="sd"> Returns:</span>
|
|
<span class="sd"> OrderedDict:</span>
|
|
<span class="sd"> * ``report_type``: ``aggregate`` or ``forensic``</span>
|
|
<span class="sd"> * ``report``: The parsed report</span>
|
|
<span class="sd"> """</span>
|
|
<span class="n">result</span> <span class="o">=</span> <span class="kc">None</span>
|
|
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="n">is_outlook_msg</span><span class="p">(</span><span class="n">input_</span><span class="p">):</span>
|
|
<span class="n">input_</span> <span class="o">=</span> <span class="n">convert_outlook_msg</span><span class="p">(</span><span class="n">input_</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">input_</span><span class="p">)</span> <span class="o">==</span> <span class="nb">bytes</span><span class="p">:</span>
|
|
<span class="n">input_</span> <span class="o">=</span> <span class="n">input_</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="n">encoding</span><span class="o">=</span><span class="s2">"utf8"</span><span class="p">)</span>
|
|
<span class="n">msg</span> <span class="o">=</span> <span class="n">mailparser</span><span class="o">.</span><span class="n">parse_from_string</span><span class="p">(</span><span class="n">input_</span><span class="p">)</span>
|
|
<span class="n">msg_headers</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">msg</span><span class="o">.</span><span class="n">headers_json</span><span class="p">)</span>
|
|
<span class="n">date</span> <span class="o">=</span> <span class="n">email</span><span class="o">.</span><span class="n">utils</span><span class="o">.</span><span class="n">format_datetime</span><span class="p">(</span><span class="n">datetime</span><span class="o">.</span><span class="n">utcnow</span><span class="p">())</span>
|
|
<span class="k">if</span> <span class="s2">"Date"</span> <span class="ow">in</span> <span class="n">msg_headers</span><span class="p">:</span>
|
|
<span class="n">date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span>
|
|
<span class="n">msg_headers</span><span class="p">[</span><span class="s2">"Date"</span><span class="p">])</span>
|
|
<span class="n">msg</span> <span class="o">=</span> <span class="n">email</span><span class="o">.</span><span class="n">message_from_string</span><span class="p">(</span><span class="n">input_</span><span class="p">)</span>
|
|
|
|
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">InvalidDMARCReport</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
|
<span class="n">subject</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="n">feedback_report</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="n">sample</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="k">if</span> <span class="s2">"Subject"</span> <span class="ow">in</span> <span class="n">msg_headers</span><span class="p">:</span>
|
|
<span class="n">subject</span> <span class="o">=</span> <span class="n">msg_headers</span><span class="p">[</span><span class="s2">"Subject"</span><span class="p">]</span>
|
|
<span class="k">for</span> <span class="n">part</span> <span class="ow">in</span> <span class="n">msg</span><span class="o">.</span><span class="n">walk</span><span class="p">():</span>
|
|
<span class="n">content_type</span> <span class="o">=</span> <span class="n">part</span><span class="o">.</span><span class="n">get_content_type</span><span class="p">()</span>
|
|
<span class="n">payload</span> <span class="o">=</span> <span class="n">part</span><span class="o">.</span><span class="n">get_payload</span><span class="p">()</span>
|
|
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">payload</span><span class="p">)</span> <span class="o">!=</span> <span class="nb">list</span><span class="p">:</span>
|
|
<span class="n">payload</span> <span class="o">=</span> <span class="p">[</span><span class="n">payload</span><span class="p">]</span>
|
|
<span class="n">payload</span> <span class="o">=</span> <span class="n">payload</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span>
|
|
<span class="k">if</span> <span class="n">content_type</span> <span class="o">==</span> <span class="s2">"message/feedback-report"</span><span class="p">:</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="s2">"Feedback-Type"</span> <span class="ow">in</span> <span class="n">payload</span><span class="p">:</span>
|
|
<span class="n">feedback_report</span> <span class="o">=</span> <span class="n">payload</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">feedback_report</span> <span class="o">=</span> <span class="n">b64decode</span><span class="p">(</span><span class="n">payload</span><span class="p">)</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span>
|
|
<span class="n">feedback_report</span> <span class="o">=</span> <span class="n">feedback_report</span><span class="o">.</span><span class="n">lstrip</span><span class="p">(</span>
|
|
<span class="s2">"b'"</span><span class="p">)</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s2">"'"</span><span class="p">)</span>
|
|
<span class="n">feedback_report</span> <span class="o">=</span> <span class="n">feedback_report</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"</span><span class="se">\\</span><span class="s2">r"</span><span class="p">,</span> <span class="s2">""</span><span class="p">)</span>
|
|
<span class="n">feedback_report</span> <span class="o">=</span> <span class="n">feedback_report</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"</span><span class="se">\\</span><span class="s2">n"</span><span class="p">,</span> <span class="s2">"</span><span class="se">\n</span><span class="s2">"</span><span class="p">)</span>
|
|
<span class="k">except</span> <span class="p">(</span><span class="ne">ValueError</span><span class="p">,</span> <span class="ne">TypeError</span><span class="p">,</span> <span class="n">binascii</span><span class="o">.</span><span class="n">Error</span><span class="p">):</span>
|
|
<span class="n">feedback_report</span> <span class="o">=</span> <span class="n">payload</span>
|
|
|
|
<span class="k">elif</span> <span class="n">content_type</span> <span class="o">==</span> <span class="s2">"text/rfc822-headers"</span><span class="p">:</span>
|
|
<span class="n">sample</span> <span class="o">=</span> <span class="n">payload</span>
|
|
<span class="k">elif</span> <span class="n">content_type</span> <span class="o">==</span> <span class="s2">"message/rfc822"</span><span class="p">:</span>
|
|
<span class="n">sample</span> <span class="o">=</span> <span class="n">payload</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">payload</span> <span class="o">=</span> <span class="n">b64decode</span><span class="p">(</span><span class="n">payload</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">payload</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="n">MAGIC_ZIP</span><span class="p">)</span> <span class="ow">or</span> \
|
|
<span class="n">payload</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="n">MAGIC_GZIP</span><span class="p">)</span> <span class="ow">or</span> \
|
|
<span class="n">payload</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="n">MAGIC_XML</span><span class="p">):</span>
|
|
<span class="n">ns</span> <span class="o">=</span> <span class="n">nameservers</span>
|
|
<span class="n">aggregate_report</span> <span class="o">=</span> <span class="n">parse_aggregate_report_file</span><span class="p">(</span>
|
|
<span class="n">payload</span><span class="p">,</span>
|
|
<span class="n">nameservers</span><span class="o">=</span><span class="n">ns</span><span class="p">,</span>
|
|
<span class="n">dns_timeout</span><span class="o">=</span><span class="n">dns_timeout</span><span class="p">,</span>
|
|
<span class="n">parallel</span><span class="o">=</span><span class="n">parallel</span><span class="p">)</span>
|
|
<span class="n">result</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">([(</span><span class="s2">"report_type"</span><span class="p">,</span> <span class="s2">"aggregate"</span><span class="p">),</span>
|
|
<span class="p">(</span><span class="s2">"report"</span><span class="p">,</span> <span class="n">aggregate_report</span><span class="p">)])</span>
|
|
<span class="k">return</span> <span class="n">result</span>
|
|
|
|
<span class="k">except</span> <span class="p">(</span><span class="ne">TypeError</span><span class="p">,</span> <span class="ne">ValueError</span><span class="p">,</span> <span class="n">binascii</span><span class="o">.</span><span class="n">Error</span><span class="p">):</span>
|
|
<span class="k">pass</span>
|
|
|
|
<span class="k">except</span> <span class="n">InvalidAggregateReport</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
|
<span class="n">error</span> <span class="o">=</span> <span class="s1">'Message with subject "</span><span class="si">{0}</span><span class="s1">" '</span> \
|
|
<span class="s1">'is not a valid '</span> \
|
|
<span class="s1">'aggregate DMARC report: </span><span class="si">{1}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">subject</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
|
|
<span class="k">raise</span> <span class="n">InvalidAggregateReport</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
|
|
|
|
<span class="k">except</span> <span class="ne">FileNotFoundError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
|
<span class="n">error</span> <span class="o">=</span> <span class="s1">'Unable to parse message with '</span> \
|
|
<span class="s1">'subject "</span><span class="si">{0}</span><span class="s1">": </span><span class="si">{1}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">subject</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
|
|
<span class="k">raise</span> <span class="n">InvalidDMARCReport</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
|
|
|
|
<span class="k">if</span> <span class="n">feedback_report</span> <span class="ow">and</span> <span class="n">sample</span><span class="p">:</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">forensic_report</span> <span class="o">=</span> <span class="n">parse_forensic_report</span><span class="p">(</span>
|
|
<span class="n">feedback_report</span><span class="p">,</span>
|
|
<span class="n">sample</span><span class="p">,</span>
|
|
<span class="n">date</span><span class="p">,</span>
|
|
<span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
|
|
<span class="n">dns_timeout</span><span class="o">=</span><span class="n">dns_timeout</span><span class="p">,</span>
|
|
<span class="n">strip_attachment_payloads</span><span class="o">=</span><span class="n">strip_attachment_payloads</span><span class="p">,</span>
|
|
<span class="n">parallel</span><span class="o">=</span><span class="n">parallel</span><span class="p">)</span>
|
|
<span class="k">except</span> <span class="n">InvalidForensicReport</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
|
<span class="n">error</span> <span class="o">=</span> <span class="s1">'Message with subject "</span><span class="si">{0}</span><span class="s1">" '</span> \
|
|
<span class="s1">'is not a valid '</span> \
|
|
<span class="s1">'forensic DMARC report: </span><span class="si">{1}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">subject</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
|
|
<span class="k">raise</span> <span class="n">InvalidForensicReport</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
|
|
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">InvalidForensicReport</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
|
|
|
<span class="n">result</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">([(</span><span class="s2">"report_type"</span><span class="p">,</span> <span class="s2">"forensic"</span><span class="p">),</span>
|
|
<span class="p">(</span><span class="s2">"report"</span><span class="p">,</span> <span class="n">forensic_report</span><span class="p">)])</span>
|
|
<span class="k">return</span> <span class="n">result</span>
|
|
|
|
<span class="k">if</span> <span class="n">result</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">error</span> <span class="o">=</span> <span class="s1">'Message with subject "</span><span class="si">{0}</span><span class="s1">" is '</span> \
|
|
<span class="s1">'not a valid DMARC report'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">subject</span><span class="p">)</span>
|
|
<span class="k">raise</span> <span class="n">InvalidDMARCReport</span><span class="p">(</span><span class="n">error</span><span class="p">)</span></div>
|
|
|
|
|
|
<div class="viewcode-block" id="parse_report_file"><a class="viewcode-back" href="../index.html#parsedmarc.parse_report_file">[docs]</a><span class="k">def</span> <span class="nf">parse_report_file</span><span class="p">(</span><span class="n">input_</span><span class="p">,</span> <span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">dns_timeout</span><span class="o">=</span><span class="mf">2.0</span><span class="p">,</span>
|
|
<span class="n">strip_attachment_payloads</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">parallel</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
|
|
<span class="sd">"""Parses a DMARC aggregate or forensic file at the given path, a</span>
|
|
<span class="sd"> file-like object. or bytes</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> input_: A path to a file, a file like object, or bytes</span>
|
|
<span class="sd"> nameservers (list): A list of one or more nameservers to use</span>
|
|
<span class="sd"> (Cloudflare's public DNS resolvers by default)</span>
|
|
<span class="sd"> dns_timeout (float): Sets the DNS timeout in seconds</span>
|
|
<span class="sd"> strip_attachment_payloads (bool): Remove attachment payloads from</span>
|
|
<span class="sd"> forensic report results</span>
|
|
<span class="sd"> parallel (bool): Parallel processing</span>
|
|
|
|
<span class="sd"> Returns:</span>
|
|
<span class="sd"> OrderedDict: The parsed DMARC report</span>
|
|
<span class="sd"> """</span>
|
|
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">input_</span><span class="p">)</span> <span class="o">==</span> <span class="nb">str</span><span class="p">:</span>
|
|
<span class="n">file_object</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="n">input_</span><span class="p">,</span> <span class="s2">"rb"</span><span class="p">)</span>
|
|
<span class="k">elif</span> <span class="nb">type</span><span class="p">(</span><span class="n">input_</span><span class="p">)</span> <span class="o">==</span> <span class="nb">bytes</span><span class="p">:</span>
|
|
<span class="n">file_object</span> <span class="o">=</span> <span class="n">BytesIO</span><span class="p">(</span><span class="n">input_</span><span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">file_object</span> <span class="o">=</span> <span class="n">input_</span>
|
|
|
|
<span class="n">content</span> <span class="o">=</span> <span class="n">file_object</span><span class="o">.</span><span class="n">read</span><span class="p">()</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">report</span> <span class="o">=</span> <span class="n">parse_aggregate_report_file</span><span class="p">(</span><span class="n">content</span><span class="p">,</span> <span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
|
|
<span class="n">dns_timeout</span><span class="o">=</span><span class="n">dns_timeout</span><span class="p">,</span>
|
|
<span class="n">parallel</span><span class="o">=</span><span class="n">parallel</span><span class="p">)</span>
|
|
<span class="n">results</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">([(</span><span class="s2">"report_type"</span><span class="p">,</span> <span class="s2">"aggregate"</span><span class="p">),</span>
|
|
<span class="p">(</span><span class="s2">"report"</span><span class="p">,</span> <span class="n">report</span><span class="p">)])</span>
|
|
<span class="k">except</span> <span class="n">InvalidAggregateReport</span><span class="p">:</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">sa</span> <span class="o">=</span> <span class="n">strip_attachment_payloads</span>
|
|
<span class="n">results</span> <span class="o">=</span> <span class="n">parse_report_email</span><span class="p">(</span><span class="n">content</span><span class="p">,</span>
|
|
<span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
|
|
<span class="n">dns_timeout</span><span class="o">=</span><span class="n">dns_timeout</span><span class="p">,</span>
|
|
<span class="n">strip_attachment_payloads</span><span class="o">=</span><span class="n">sa</span><span class="p">,</span>
|
|
<span class="n">parallel</span><span class="o">=</span><span class="n">parallel</span><span class="p">)</span>
|
|
<span class="k">except</span> <span class="n">InvalidDMARCReport</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">InvalidDMARCReport</span><span class="p">(</span><span class="s2">"Not a valid aggregate or forensic "</span>
|
|
<span class="s2">"report"</span><span class="p">)</span>
|
|
<span class="k">return</span> <span class="n">results</span></div>
|
|
|
|
|
|
<div class="viewcode-block" id="get_imap_capabilities"><a class="viewcode-back" href="../index.html#parsedmarc.get_imap_capabilities">[docs]</a><span class="k">def</span> <span class="nf">get_imap_capabilities</span><span class="p">(</span><span class="n">server</span><span class="p">):</span>
|
|
<span class="sd">"""</span>
|
|
<span class="sd"> Returns a list of an IMAP server's capabilities</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> server (imapclient.IMAPClient): An instance of imapclient.IMAPClient</span>
|
|
|
|
<span class="sd"> Returns (list): A list of capabilities</span>
|
|
<span class="sd"> """</span>
|
|
|
|
<span class="n">capabilities</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="nb">str</span><span class="p">,</span> <span class="nb">list</span><span class="p">(</span><span class="n">server</span><span class="o">.</span><span class="n">capabilities</span><span class="p">())))</span>
|
|
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">capabilities</span><span class="p">)):</span>
|
|
<span class="n">capabilities</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">capabilities</span><span class="p">[</span><span class="n">i</span><span class="p">])</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"b'"</span><span class="p">,</span>
|
|
<span class="s2">""</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"'"</span><span class="p">,</span>
|
|
<span class="s2">""</span><span class="p">)</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"IMAP server supports: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">capabilities</span><span class="p">))</span>
|
|
|
|
<span class="k">return</span> <span class="n">capabilities</span></div>
|
|
|
|
|
|
<div class="viewcode-block" id="get_dmarc_reports_from_inbox"><a class="viewcode-back" href="../index.html#parsedmarc.get_dmarc_reports_from_inbox">[docs]</a><span class="k">def</span> <span class="nf">get_dmarc_reports_from_inbox</span><span class="p">(</span><span class="n">host</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">user</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">password</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">connection</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">port</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">ssl</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
|
<span class="n">ssl_context</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">move_supported</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">reports_folder</span><span class="o">=</span><span class="s2">"INBOX"</span><span class="p">,</span>
|
|
<span class="n">archive_folder</span><span class="o">=</span><span class="s2">"Archive"</span><span class="p">,</span>
|
|
<span class="n">delete</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
|
<span class="n">test</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
|
<span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">dns_timeout</span><span class="o">=</span><span class="mf">6.0</span><span class="p">,</span>
|
|
<span class="n">strip_attachment_payloads</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
|
<span class="n">results</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
|
<span class="sd">"""</span>
|
|
<span class="sd"> Fetches and parses DMARC reports from sn inbox</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> host: The mail server hostname or IP address</span>
|
|
<span class="sd"> user: The mail server user</span>
|
|
<span class="sd"> password: The mail server password</span>
|
|
<span class="sd"> connection: An IMAPCLient connection to reuse</span>
|
|
<span class="sd"> port: The mail server port</span>
|
|
<span class="sd"> ssl (bool): Use SSL/TLS</span>
|
|
<span class="sd"> ssl_context (SSLContext): A SSL context</span>
|
|
<span class="sd"> move_supported: Indicate if the IMAP server supports the MOVE command</span>
|
|
<span class="sd"> (autodetect if None)</span>
|
|
<span class="sd"> reports_folder: The IMAP folder where reports can be found</span>
|
|
<span class="sd"> archive_folder: The folder to move processed mail to</span>
|
|
<span class="sd"> delete (bool): Delete messages after processing them</span>
|
|
<span class="sd"> test (bool): Do not move or delete messages after processing them</span>
|
|
<span class="sd"> nameservers (list): A list of DNS nameservers to query</span>
|
|
<span class="sd"> dns_timeout (float): Set the DNS query timeout</span>
|
|
<span class="sd"> strip_attachment_payloads (bool): Remove attachment payloads from</span>
|
|
<span class="sd"> forensic report results</span>
|
|
<span class="sd"> results (dict): Results from the previous run</span>
|
|
|
|
<span class="sd"> Returns:</span>
|
|
<span class="sd"> OrderedDict: Lists of ``aggregate_reports`` and ``forensic_reports``</span>
|
|
<span class="sd"> """</span>
|
|
|
|
<span class="k">def</span> <span class="nf">chunks</span><span class="p">(</span><span class="n">l</span><span class="p">,</span> <span class="n">n</span><span class="p">):</span>
|
|
<span class="sd">"""Yield successive n-sized chunks from l."""</span>
|
|
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">l</span><span class="p">),</span> <span class="n">n</span><span class="p">):</span>
|
|
<span class="k">yield</span> <span class="n">l</span><span class="p">[</span><span class="n">i</span><span class="p">:</span><span class="n">i</span> <span class="o">+</span> <span class="n">n</span><span class="p">]</span>
|
|
|
|
<span class="k">if</span> <span class="n">delete</span> <span class="ow">and</span> <span class="n">test</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"delete and test options are mutually exclusive"</span><span class="p">)</span>
|
|
|
|
<span class="k">if</span> <span class="n">connection</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="p">(</span><span class="n">user</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">or</span> <span class="n">password</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">):</span>
|
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"Must supply a connection, or a username and "</span>
|
|
<span class="s2">"password"</span><span class="p">)</span>
|
|
|
|
<span class="n">aggregate_reports</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="n">forensic_reports</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="n">aggregate_report_msg_uids</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="n">forensic_report_msg_uids</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="n">aggregate_reports_folder</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">/Aggregate"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">archive_folder</span><span class="p">)</span>
|
|
<span class="n">forensic_reports_folder</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">/Forensic"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">archive_folder</span><span class="p">)</span>
|
|
<span class="n">invalid_reports_folder</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">/Invalid"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">archive_folder</span><span class="p">)</span>
|
|
|
|
<span class="k">if</span> <span class="n">results</span><span class="p">:</span>
|
|
<span class="n">aggregate_reports</span> <span class="o">=</span> <span class="n">results</span><span class="p">[</span><span class="s2">"aggregate_reports"</span><span class="p">]</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
|
<span class="n">forensic_reports</span> <span class="o">=</span> <span class="n">results</span><span class="p">[</span><span class="s2">"forensic_reports"</span><span class="p">]</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
|
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="n">connection</span><span class="p">:</span>
|
|
<span class="n">server</span> <span class="o">=</span> <span class="n">connection</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">ssl</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Connecting to IMAP over plain text"</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">ssl_context</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">ssl_context</span> <span class="o">=</span> <span class="n">create_default_context</span><span class="p">()</span>
|
|
<span class="n">server</span> <span class="o">=</span> <span class="n">imapclient</span><span class="o">.</span><span class="n">IMAPClient</span><span class="p">(</span><span class="n">host</span><span class="p">,</span>
|
|
<span class="n">port</span><span class="o">=</span><span class="n">port</span><span class="p">,</span>
|
|
<span class="n">ssl</span><span class="o">=</span><span class="n">ssl</span><span class="p">,</span>
|
|
<span class="n">ssl_context</span><span class="o">=</span><span class="n">ssl_context</span><span class="p">,</span>
|
|
<span class="n">use_uid</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">login</span><span class="p">(</span><span class="n">user</span><span class="p">,</span> <span class="n">password</span><span class="p">)</span>
|
|
|
|
<span class="k">if</span> <span class="n">move_supported</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">server_capabilities</span> <span class="o">=</span> <span class="n">get_imap_capabilities</span><span class="p">(</span><span class="n">server</span><span class="p">)</span>
|
|
<span class="n">move_supported</span> <span class="o">=</span> <span class="s2">"MOVE"</span> <span class="ow">in</span> <span class="n">server_capabilities</span>
|
|
|
|
<span class="k">def</span> <span class="nf">delete_messages</span><span class="p">(</span><span class="n">msg_uids</span><span class="p">):</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Deleting message UID(s) </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="s2">","</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
|
|
<span class="nb">str</span><span class="p">(</span><span class="n">uid</span><span class="p">)</span> <span class="k">for</span> <span class="n">uid</span> <span class="ow">in</span> <span class="n">msg_uids</span><span class="p">)))</span>
|
|
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">msg_uids</span><span class="p">)</span> <span class="o">==</span> <span class="nb">str</span> <span class="ow">or</span> <span class="nb">type</span><span class="p">(</span><span class="n">msg_uids</span><span class="p">)</span> <span class="o">==</span> <span class="nb">int</span><span class="p">:</span>
|
|
<span class="n">msg_uids</span> <span class="o">=</span> <span class="p">[</span><span class="nb">int</span><span class="p">(</span><span class="n">msg_uids</span><span class="p">)]</span>
|
|
|
|
<span class="n">server</span><span class="o">.</span><span class="n">delete_messages</span><span class="p">(</span><span class="n">msg_uids</span><span class="p">,</span> <span class="n">silent</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">expunge</span><span class="p">(</span><span class="n">msg_uids</span><span class="p">)</span>
|
|
|
|
<span class="k">def</span> <span class="nf">move_messages</span><span class="p">(</span><span class="n">msg_uids</span><span class="p">,</span> <span class="n">folder</span><span class="p">):</span>
|
|
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">msg_uids</span><span class="p">)</span> <span class="o">==</span> <span class="nb">str</span> <span class="ow">or</span> <span class="nb">type</span><span class="p">(</span><span class="n">msg_uids</span><span class="p">)</span> <span class="o">==</span> <span class="nb">int</span><span class="p">:</span>
|
|
<span class="n">msg_uids</span> <span class="o">=</span> <span class="p">[</span><span class="nb">int</span><span class="p">(</span><span class="n">msg_uids</span><span class="p">)]</span>
|
|
<span class="k">for</span> <span class="n">chunk</span> <span class="ow">in</span> <span class="n">chunks</span><span class="p">(</span><span class="n">msg_uids</span><span class="p">,</span> <span class="mi">100</span><span class="p">):</span>
|
|
<span class="k">if</span> <span class="n">move_supported</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Moving message UID(s) </span><span class="si">{0}</span><span class="s2"> to </span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
|
<span class="s2">","</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">uid</span><span class="p">)</span> <span class="k">for</span> <span class="n">uid</span> <span class="ow">in</span> <span class="n">chunk</span><span class="p">),</span> <span class="n">folder</span>
|
|
<span class="p">))</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">move</span><span class="p">(</span><span class="n">chunk</span><span class="p">,</span> <span class="n">folder</span><span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Copying message UID(s) </span><span class="si">{0}</span><span class="s2"> to </span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
|
<span class="s2">","</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">uid</span><span class="p">)</span> <span class="k">for</span> <span class="n">uid</span> <span class="ow">in</span> <span class="n">chunk</span><span class="p">),</span> <span class="n">folder</span>
|
|
<span class="p">))</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">copy</span><span class="p">(</span><span class="n">msg_uids</span><span class="p">,</span> <span class="n">folder</span><span class="p">)</span>
|
|
<span class="n">delete_messages</span><span class="p">(</span><span class="n">msg_uids</span><span class="p">)</span>
|
|
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">server</span><span class="o">.</span><span class="n">folder_exists</span><span class="p">(</span><span class="n">archive_folder</span><span class="p">):</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Creating IMAP folder: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">archive_folder</span><span class="p">))</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">create_folder</span><span class="p">(</span><span class="n">archive_folder</span><span class="p">)</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="c1"># Test subfolder creation</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">server</span><span class="o">.</span><span class="n">folder_exists</span><span class="p">(</span><span class="n">aggregate_reports_folder</span><span class="p">):</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">create_folder</span><span class="p">(</span><span class="n">aggregate_reports_folder</span><span class="p">)</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
|
<span class="s2">"Creating IMAP folder: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
|
<span class="n">aggregate_reports_folder</span><span class="p">))</span>
|
|
<span class="k">except</span> <span class="n">imapclient</span><span class="o">.</span><span class="n">exceptions</span><span class="o">.</span><span class="n">IMAPClientError</span><span class="p">:</span>
|
|
<span class="c1"># Only replace / with . when . doesn't work</span>
|
|
<span class="c1"># This usually indicates a dovecot IMAP server</span>
|
|
<span class="n">aggregate_reports_folder</span> <span class="o">=</span> <span class="n">aggregate_reports_folder</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"/"</span><span class="p">,</span>
|
|
<span class="s2">"."</span><span class="p">)</span>
|
|
<span class="n">forensic_reports_folder</span> <span class="o">=</span> <span class="n">forensic_reports_folder</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"/"</span><span class="p">,</span>
|
|
<span class="s2">"."</span><span class="p">)</span>
|
|
<span class="n">invalid_reports_folder</span> <span class="o">=</span> <span class="n">invalid_reports_folder</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"/"</span><span class="p">,</span>
|
|
<span class="s2">"."</span><span class="p">)</span>
|
|
<span class="n">subfolders</span> <span class="o">=</span> <span class="p">[</span><span class="n">aggregate_reports_folder</span><span class="p">,</span>
|
|
<span class="n">forensic_reports_folder</span><span class="p">,</span>
|
|
<span class="n">invalid_reports_folder</span><span class="p">]</span>
|
|
|
|
<span class="k">for</span> <span class="n">subfolder</span> <span class="ow">in</span> <span class="n">subfolders</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">server</span><span class="o">.</span><span class="n">folder_exists</span><span class="p">(</span><span class="n">subfolder</span><span class="p">):</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
|
<span class="s2">"Creating IMAP folder: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">subfolder</span><span class="p">))</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">create_folder</span><span class="p">(</span><span class="n">subfolder</span><span class="p">)</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">select_folder</span><span class="p">(</span><span class="n">reports_folder</span><span class="p">)</span>
|
|
<span class="n">messages</span> <span class="o">=</span> <span class="n">server</span><span class="o">.</span><span class="n">search</span><span class="p">()</span>
|
|
<span class="n">total_messages</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">messages</span><span class="p">)</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Found </span><span class="si">{0}</span><span class="s2"> messages in IMAP folder </span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
|
<span class="nb">len</span><span class="p">(</span><span class="n">messages</span><span class="p">),</span> <span class="n">reports_folder</span><span class="p">))</span>
|
|
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">messages</span><span class="p">)):</span>
|
|
<span class="n">msg_uid</span> <span class="o">=</span> <span class="n">messages</span><span class="p">[</span><span class="n">i</span><span class="p">]</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Processing message </span><span class="si">{0}</span><span class="s2"> of </span><span class="si">{1}</span><span class="s2">: UID </span><span class="si">{2}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
|
<span class="n">i</span><span class="o">+</span><span class="mi">1</span><span class="p">,</span>
|
|
<span class="n">total_messages</span><span class="p">,</span>
|
|
<span class="n">msg_uid</span>
|
|
<span class="p">))</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">raw_msg</span> <span class="o">=</span> <span class="n">server</span><span class="o">.</span><span class="n">fetch</span><span class="p">(</span><span class="n">msg_uid</span><span class="p">,</span>
|
|
<span class="p">[</span><span class="s2">"RFC822"</span><span class="p">])[</span><span class="n">msg_uid</span><span class="p">]</span>
|
|
<span class="n">msg_keys</span> <span class="o">=</span> <span class="p">[</span><span class="sa">b</span><span class="s1">'RFC822'</span><span class="p">,</span> <span class="sa">b</span><span class="s1">'BODY[NULL]'</span><span class="p">,</span> <span class="sa">b</span><span class="s1">'BODY[]'</span><span class="p">]</span>
|
|
<span class="n">msg_key</span> <span class="o">=</span> <span class="s1">''</span>
|
|
<span class="k">for</span> <span class="n">key</span> <span class="ow">in</span> <span class="n">msg_keys</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="n">key</span> <span class="ow">in</span> <span class="n">raw_msg</span><span class="o">.</span><span class="n">keys</span><span class="p">():</span>
|
|
<span class="n">msg_key</span> <span class="o">=</span> <span class="n">key</span>
|
|
<span class="k">break</span>
|
|
<span class="n">raw_msg</span> <span class="o">=</span> <span class="n">raw_msg</span><span class="p">[</span><span class="n">msg_key</span><span class="p">]</span>
|
|
|
|
<span class="k">except</span> <span class="p">(</span><span class="ne">ConnectionResetError</span><span class="p">,</span> <span class="n">socket</span><span class="o">.</span><span class="n">error</span><span class="p">,</span>
|
|
<span class="ne">TimeoutError</span><span class="p">,</span>
|
|
<span class="n">imapclient</span><span class="o">.</span><span class="n">exceptions</span><span class="o">.</span><span class="n">IMAPClientError</span><span class="p">)</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
|
<span class="n">error</span> <span class="o">=</span> <span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="o">.</span><span class="n">lstrip</span><span class="p">(</span><span class="s2">"b'"</span><span class="p">)</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s2">"'"</span><span class="p">)</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span>
|
|
<span class="s2">"."</span><span class="p">)</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"IMAP error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Reconnecting to IMAP"</span><span class="p">)</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">shutdown</span><span class="p">()</span>
|
|
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
|
<span class="s2">"Failed to log out: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">ssl</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Connecting to IMAP over plain text"</span><span class="p">)</span>
|
|
<span class="n">server</span> <span class="o">=</span> <span class="n">imapclient</span><span class="o">.</span><span class="n">IMAPClient</span><span class="p">(</span><span class="n">host</span><span class="p">,</span>
|
|
<span class="n">port</span><span class="o">=</span><span class="n">port</span><span class="p">,</span>
|
|
<span class="n">ssl</span><span class="o">=</span><span class="n">ssl</span><span class="p">,</span>
|
|
<span class="n">ssl_context</span><span class="o">=</span><span class="n">ssl_context</span><span class="p">,</span>
|
|
<span class="n">use_uid</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">login</span><span class="p">(</span><span class="n">user</span><span class="p">,</span> <span class="n">password</span><span class="p">)</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">select_folder</span><span class="p">(</span><span class="n">reports_folder</span><span class="p">)</span>
|
|
<span class="n">raw_msg</span> <span class="o">=</span> <span class="n">server</span><span class="o">.</span><span class="n">fetch</span><span class="p">(</span><span class="n">msg_uid</span><span class="p">,</span>
|
|
<span class="p">[</span><span class="s2">"RFC822"</span><span class="p">])[</span><span class="n">msg_uid</span><span class="p">][</span><span class="sa">b</span><span class="s2">"RFC822"</span><span class="p">]</span>
|
|
|
|
<span class="n">msg_content</span> <span class="o">=</span> <span class="n">raw_msg</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s2">"utf-8"</span><span class="p">,</span> <span class="n">errors</span><span class="o">=</span><span class="s2">"replace"</span><span class="p">)</span>
|
|
<span class="n">sa</span> <span class="o">=</span> <span class="n">strip_attachment_payloads</span>
|
|
<span class="n">parsed_email</span> <span class="o">=</span> <span class="n">parse_report_email</span><span class="p">(</span><span class="n">msg_content</span><span class="p">,</span>
|
|
<span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
|
|
<span class="n">dns_timeout</span><span class="o">=</span><span class="n">dns_timeout</span><span class="p">,</span>
|
|
<span class="n">strip_attachment_payloads</span><span class="o">=</span><span class="n">sa</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">"report_type"</span><span class="p">]</span> <span class="o">==</span> <span class="s2">"aggregate"</span><span class="p">:</span>
|
|
<span class="n">aggregate_reports</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">parsed_email</span><span class="p">[</span><span class="s2">"report"</span><span class="p">])</span>
|
|
<span class="n">aggregate_report_msg_uids</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">msg_uid</span><span class="p">)</span>
|
|
<span class="k">elif</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">"report_type"</span><span class="p">]</span> <span class="o">==</span> <span class="s2">"forensic"</span><span class="p">:</span>
|
|
<span class="n">forensic_reports</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">parsed_email</span><span class="p">[</span><span class="s2">"report"</span><span class="p">])</span>
|
|
<span class="n">forensic_report_msg_uids</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">msg_uid</span><span class="p">)</span>
|
|
<span class="k">except</span> <span class="n">InvalidDMARCReport</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">test</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="n">delete</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
|
<span class="s2">"Deleting message UID </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">msg_uid</span><span class="p">))</span>
|
|
<span class="n">delete_messages</span><span class="p">([</span><span class="n">msg_uid</span><span class="p">])</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
|
<span class="s2">"Moving message UID </span><span class="si">{0}</span><span class="s2"> to </span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
|
<span class="n">msg_uid</span><span class="p">,</span> <span class="n">invalid_reports_folder</span><span class="p">))</span>
|
|
<span class="n">move_messages</span><span class="p">([</span><span class="n">msg_uid</span><span class="p">],</span> <span class="n">invalid_reports_folder</span><span class="p">)</span>
|
|
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">test</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="n">delete</span><span class="p">:</span>
|
|
<span class="n">processed_messages</span> <span class="o">=</span> <span class="n">aggregate_report_msg_uids</span> <span class="o">+</span> \
|
|
<span class="n">forensic_report_msg_uids</span>
|
|
|
|
<span class="n">number_of_processed_msgs</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">processed_messages</span><span class="p">)</span>
|
|
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">number_of_processed_msgs</span><span class="p">):</span>
|
|
<span class="n">msg_uid</span> <span class="o">=</span> <span class="n">processed_messages</span><span class="p">[</span><span class="n">i</span><span class="p">]</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
|
<span class="s2">"Deleting message </span><span class="si">{0}</span><span class="s2"> of </span><span class="si">{1}</span><span class="s2">: UID </span><span class="si">{2}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
|
<span class="n">i</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="n">number_of_processed_msgs</span><span class="p">,</span> <span class="n">msg_uid</span><span class="p">))</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">delete_messages</span><span class="p">([</span><span class="n">msg_uid</span><span class="p">])</span>
|
|
|
|
<span class="k">except</span> <span class="n">imapclient</span><span class="o">.</span><span class="n">exceptions</span><span class="o">.</span><span class="n">IMAPClientError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
|
<span class="n">e</span> <span class="o">=</span> <span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="o">.</span><span class="n">lstrip</span><span class="p">(</span><span class="s2">"b'"</span><span class="p">)</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span>
|
|
<span class="s2">"'"</span><span class="p">)</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s2">"."</span><span class="p">)</span>
|
|
<span class="n">message</span> <span class="o">=</span> <span class="s2">"Error deleting message UID"</span>
|
|
<span class="n">e</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2"> </span><span class="si">{1}</span><span class="s2">: "</span> <span class="s2">"</span><span class="si">{2}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">message</span><span class="p">,</span> <span class="n">msg_uid</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"IMAP error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="p">))</span>
|
|
<span class="k">except</span> <span class="p">(</span><span class="ne">ConnectionResetError</span><span class="p">,</span> <span class="n">socket</span><span class="o">.</span><span class="n">error</span><span class="p">,</span>
|
|
<span class="ne">TimeoutError</span><span class="p">)</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"IMAP error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Reconnecting to IMAP"</span><span class="p">)</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">shutdown</span><span class="p">()</span>
|
|
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
|
<span class="s2">"Failed to log out: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">ssl</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Connecting to IMAP over plain text"</span><span class="p">)</span>
|
|
<span class="n">server</span> <span class="o">=</span> <span class="n">imapclient</span><span class="o">.</span><span class="n">IMAPClient</span><span class="p">(</span><span class="n">host</span><span class="p">,</span>
|
|
<span class="n">port</span><span class="o">=</span><span class="n">port</span><span class="p">,</span>
|
|
<span class="n">ssl</span><span class="o">=</span><span class="n">ssl</span><span class="p">,</span>
|
|
<span class="n">ssl_context</span><span class="o">=</span><span class="n">ssl_context</span><span class="p">,</span>
|
|
<span class="n">use_uid</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">login</span><span class="p">(</span><span class="n">user</span><span class="p">,</span> <span class="n">password</span><span class="p">)</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">select_folder</span><span class="p">(</span><span class="n">reports_folder</span><span class="p">)</span>
|
|
<span class="n">delete_messages</span><span class="p">([</span><span class="n">msg_uid</span><span class="p">])</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">aggregate_report_msg_uids</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
|
<span class="n">log_message</span> <span class="o">=</span> <span class="s2">"Moving aggregate report messages from"</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
|
<span class="s2">"</span><span class="si">{0}</span><span class="s2"> </span><span class="si">{1}</span><span class="s2"> to </span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
|
<span class="n">log_message</span><span class="p">,</span> <span class="n">reports_folder</span><span class="p">,</span>
|
|
<span class="n">aggregate_reports_folder</span><span class="p">))</span>
|
|
<span class="n">number_of_agg_report_msgs</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">aggregate_report_msg_uids</span><span class="p">)</span>
|
|
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">number_of_agg_report_msgs</span><span class="p">):</span>
|
|
<span class="n">msg_uid</span> <span class="o">=</span> <span class="n">aggregate_report_msg_uids</span><span class="p">[</span><span class="n">i</span><span class="p">]</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
|
<span class="s2">"Moving message </span><span class="si">{0}</span><span class="s2"> of </span><span class="si">{1}</span><span class="s2">: UID </span><span class="si">{2}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
|
<span class="n">i</span><span class="o">+</span><span class="mi">1</span><span class="p">,</span> <span class="n">number_of_agg_report_msgs</span><span class="p">,</span> <span class="n">msg_uid</span><span class="p">))</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">move_messages</span><span class="p">([</span><span class="n">msg_uid</span><span class="p">],</span>
|
|
<span class="n">aggregate_reports_folder</span><span class="p">)</span>
|
|
<span class="k">except</span> <span class="n">imapclient</span><span class="o">.</span><span class="n">exceptions</span><span class="o">.</span><span class="n">IMAPClientError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
|
<span class="n">e</span> <span class="o">=</span> <span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="o">.</span><span class="n">lstrip</span><span class="p">(</span><span class="s2">"b'"</span><span class="p">)</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span>
|
|
<span class="s2">"'"</span><span class="p">)</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s2">"."</span><span class="p">)</span>
|
|
<span class="n">message</span> <span class="o">=</span> <span class="s2">"Error moving message UID"</span>
|
|
<span class="n">e</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2"> </span><span class="si">{1}</span><span class="s2">: </span><span class="si">{2}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">message</span><span class="p">,</span> <span class="n">msg_uid</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"IMAP error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="p">))</span>
|
|
<span class="k">except</span> <span class="p">(</span><span class="ne">ConnectionResetError</span><span class="p">,</span> <span class="n">socket</span><span class="o">.</span><span class="n">error</span><span class="p">,</span>
|
|
<span class="ne">TimeoutError</span><span class="p">)</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"IMAP error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
|
<span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Reconnecting to IMAP"</span><span class="p">)</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">shutdown</span><span class="p">()</span>
|
|
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Failed to log out: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
|
<span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">ssl</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
|
<span class="s2">"Connecting to IMAP over plain text"</span><span class="p">)</span>
|
|
<span class="n">server</span> <span class="o">=</span> <span class="n">imapclient</span><span class="o">.</span><span class="n">IMAPClient</span><span class="p">(</span>
|
|
<span class="n">host</span><span class="p">,</span>
|
|
<span class="n">port</span><span class="o">=</span><span class="n">port</span><span class="p">,</span>
|
|
<span class="n">ssl</span><span class="o">=</span><span class="n">ssl</span><span class="p">,</span>
|
|
<span class="n">ssl_context</span><span class="o">=</span><span class="n">ssl_context</span><span class="p">,</span>
|
|
<span class="n">use_uid</span><span class="o">=</span><span class="kc">True</span>
|
|
<span class="p">)</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">login</span><span class="p">(</span><span class="n">user</span><span class="p">,</span> <span class="n">password</span><span class="p">)</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">select_folder</span><span class="p">(</span><span class="n">reports_folder</span><span class="p">)</span>
|
|
<span class="n">move_messages</span><span class="p">([</span><span class="n">msg_uid</span><span class="p">],</span>
|
|
<span class="n">aggregate_reports_folder</span><span class="p">)</span>
|
|
|
|
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">forensic_report_msg_uids</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
|
<span class="n">message</span> <span class="o">=</span> <span class="s2">"Moving forensic report messages from"</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
|
<span class="s2">"</span><span class="si">{0}</span><span class="s2"> </span><span class="si">{1}</span><span class="s2"> to </span><span class="si">{2}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">message</span><span class="p">,</span>
|
|
<span class="n">reports_folder</span><span class="p">,</span>
|
|
<span class="n">forensic_reports_folder</span><span class="p">))</span>
|
|
<span class="n">number_of_forensic_msgs</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">forensic_report_msg_uids</span><span class="p">)</span>
|
|
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">number_of_forensic_msgs</span><span class="p">):</span>
|
|
<span class="n">msg_uid</span> <span class="o">=</span> <span class="n">forensic_report_msg_uids</span><span class="p">[</span><span class="n">i</span><span class="p">]</span>
|
|
<span class="n">message</span> <span class="o">=</span> <span class="s2">"Moving message"</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"</span><span class="si">{0}</span><span class="s2"> </span><span class="si">{1}</span><span class="s2"> of </span><span class="si">{2}</span><span class="s2">: UID </span><span class="si">{2}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
|
<span class="n">message</span><span class="p">,</span>
|
|
<span class="n">i</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="n">number_of_forensic_msgs</span><span class="p">,</span> <span class="n">msg_uid</span><span class="p">))</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">move_messages</span><span class="p">([</span><span class="n">msg_uid</span><span class="p">],</span>
|
|
<span class="n">forensic_reports_folder</span><span class="p">)</span>
|
|
<span class="k">except</span> <span class="n">imapclient</span><span class="o">.</span><span class="n">exceptions</span><span class="o">.</span><span class="n">IMAPClientError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
|
<span class="n">e</span> <span class="o">=</span> <span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="o">.</span><span class="n">lstrip</span><span class="p">(</span><span class="s2">"b'"</span><span class="p">)</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span>
|
|
<span class="s2">"'"</span><span class="p">)</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s2">"."</span><span class="p">)</span>
|
|
<span class="n">e</span> <span class="o">=</span> <span class="s2">"Error moving message UID </span><span class="si">{0}</span><span class="s2">: </span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
|
<span class="n">msg_uid</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"IMAP error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="p">))</span>
|
|
<span class="k">except</span> <span class="p">(</span><span class="ne">ConnectionResetError</span><span class="p">,</span> <span class="ne">TimeoutError</span><span class="p">)</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"IMAP error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
|
<span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Reconnecting to IMAP"</span><span class="p">)</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">shutdown</span><span class="p">()</span>
|
|
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Failed to "</span>
|
|
<span class="s2">"disconnect: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
|
<span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">ssl</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
|
<span class="s2">"Connecting to IMAP over plain text"</span><span class="p">)</span>
|
|
<span class="n">server</span> <span class="o">=</span> <span class="n">imapclient</span><span class="o">.</span><span class="n">IMAPClient</span><span class="p">(</span>
|
|
<span class="n">host</span><span class="p">,</span>
|
|
<span class="n">port</span><span class="o">=</span><span class="n">port</span><span class="p">,</span>
|
|
<span class="n">ssl</span><span class="o">=</span><span class="n">ssl</span><span class="p">,</span>
|
|
<span class="n">ssl_context</span><span class="o">=</span><span class="n">ssl_context</span><span class="p">,</span>
|
|
<span class="n">use_uid</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">login</span><span class="p">(</span><span class="n">user</span><span class="p">,</span> <span class="n">password</span><span class="p">)</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">select_folder</span><span class="p">(</span><span class="n">reports_folder</span><span class="p">)</span>
|
|
<span class="n">move_messages</span><span class="p">([</span><span class="n">msg_uid</span><span class="p">],</span>
|
|
<span class="n">forensic_reports_folder</span><span class="p">)</span>
|
|
|
|
<span class="n">results</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">([(</span><span class="s2">"aggregate_reports"</span><span class="p">,</span> <span class="n">aggregate_reports</span><span class="p">),</span>
|
|
<span class="p">(</span><span class="s2">"forensic_reports"</span><span class="p">,</span> <span class="n">forensic_reports</span><span class="p">)])</span>
|
|
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">test</span> <span class="ow">and</span> <span class="n">total_messages</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
|
<span class="c1"># Process emails that came in during the last run</span>
|
|
<span class="n">results</span> <span class="o">=</span> <span class="n">get_dmarc_reports_from_inbox</span><span class="p">(</span>
|
|
<span class="n">host</span><span class="o">=</span><span class="n">host</span><span class="p">,</span>
|
|
<span class="n">user</span><span class="o">=</span><span class="n">user</span><span class="p">,</span>
|
|
<span class="n">password</span><span class="o">=</span><span class="n">password</span><span class="p">,</span>
|
|
<span class="n">connection</span><span class="o">=</span><span class="n">connection</span><span class="p">,</span>
|
|
<span class="n">port</span><span class="o">=</span><span class="n">port</span><span class="p">,</span>
|
|
<span class="n">ssl</span><span class="o">=</span><span class="n">ssl</span><span class="p">,</span>
|
|
<span class="n">ssl_context</span><span class="o">=</span><span class="n">ssl_context</span><span class="p">,</span>
|
|
<span class="n">move_supported</span><span class="o">=</span><span class="n">move_supported</span><span class="p">,</span>
|
|
<span class="n">reports_folder</span><span class="o">=</span><span class="n">reports_folder</span><span class="p">,</span>
|
|
<span class="n">archive_folder</span><span class="o">=</span><span class="n">archive_folder</span><span class="p">,</span>
|
|
<span class="n">delete</span><span class="o">=</span><span class="n">delete</span><span class="p">,</span>
|
|
<span class="n">test</span><span class="o">=</span><span class="n">test</span><span class="p">,</span>
|
|
<span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
|
|
<span class="n">dns_timeout</span><span class="o">=</span><span class="n">dns_timeout</span><span class="p">,</span>
|
|
<span class="n">strip_attachment_payloads</span><span class="o">=</span><span class="n">strip_attachment_payloads</span><span class="p">,</span>
|
|
<span class="n">results</span><span class="o">=</span><span class="n">results</span>
|
|
<span class="p">)</span>
|
|
|
|
<span class="k">return</span> <span class="n">results</span>
|
|
<span class="k">except</span> <span class="n">imapclient</span><span class="o">.</span><span class="n">exceptions</span><span class="o">.</span><span class="n">IMAPClientError</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
|
<span class="n">error</span> <span class="o">=</span> <span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="o">.</span><span class="n">lstrip</span><span class="p">(</span><span class="s2">"b'"</span><span class="p">)</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s2">"'"</span><span class="p">)</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s2">"."</span><span class="p">)</span>
|
|
<span class="c1"># Workaround for random Exchange/Office365 IMAP errors</span>
|
|
<span class="k">if</span> <span class="s2">"unexpected response"</span> <span class="ow">in</span> <span class="n">error</span> <span class="ow">or</span> <span class="s2">"BAD"</span> <span class="ow">in</span> <span class="n">error</span><span class="p">:</span>
|
|
<span class="n">sleep_minutes</span> <span class="o">=</span> <span class="mi">5</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
|
<span class="s2">"</span><span class="si">{0}</span><span class="s2">. "</span>
|
|
<span class="s2">"Waiting </span><span class="si">{1}</span><span class="s2"> minutes before trying again"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
|
<span class="n">error</span><span class="p">,</span>
|
|
<span class="n">sleep_minutes</span><span class="p">))</span>
|
|
<span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="n">sleep_minutes</span> <span class="o">*</span> <span class="mi">60</span><span class="p">)</span>
|
|
<span class="n">results</span> <span class="o">=</span> <span class="n">get_dmarc_reports_from_inbox</span><span class="p">(</span>
|
|
<span class="n">host</span><span class="o">=</span><span class="n">host</span><span class="p">,</span>
|
|
<span class="n">user</span><span class="o">=</span><span class="n">user</span><span class="p">,</span>
|
|
<span class="n">password</span><span class="o">=</span><span class="n">password</span><span class="p">,</span>
|
|
<span class="n">connection</span><span class="o">=</span><span class="n">connection</span><span class="p">,</span>
|
|
<span class="n">port</span><span class="o">=</span><span class="n">port</span><span class="p">,</span>
|
|
<span class="n">ssl</span><span class="o">=</span><span class="n">ssl</span><span class="p">,</span>
|
|
<span class="n">ssl_context</span><span class="o">=</span><span class="n">ssl_context</span><span class="p">,</span>
|
|
<span class="n">move_supported</span><span class="o">=</span><span class="n">move_supported</span><span class="p">,</span>
|
|
<span class="n">reports_folder</span><span class="o">=</span><span class="n">reports_folder</span><span class="p">,</span>
|
|
<span class="n">archive_folder</span><span class="o">=</span><span class="n">archive_folder</span><span class="p">,</span>
|
|
<span class="n">delete</span><span class="o">=</span><span class="n">delete</span><span class="p">,</span>
|
|
<span class="n">test</span><span class="o">=</span><span class="n">test</span><span class="p">,</span>
|
|
<span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
|
|
<span class="n">dns_timeout</span><span class="o">=</span><span class="n">dns_timeout</span><span class="p">,</span>
|
|
<span class="n">strip_attachment_payloads</span><span class="o">=</span><span class="n">strip_attachment_payloads</span><span class="p">,</span>
|
|
<span class="n">results</span><span class="o">=</span><span class="n">results</span>
|
|
<span class="p">)</span>
|
|
|
|
<span class="k">return</span> <span class="n">results</span>
|
|
|
|
<span class="k">raise</span> <span class="n">IMAPError</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
|
|
<span class="k">except</span> <span class="n">socket</span><span class="o">.</span><span class="n">gaierror</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">IMAPError</span><span class="p">(</span><span class="s2">"DNS resolution failed"</span><span class="p">)</span>
|
|
<span class="k">except</span> <span class="ne">ConnectionRefusedError</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">IMAPError</span><span class="p">(</span><span class="s2">"Connection refused"</span><span class="p">)</span>
|
|
<span class="k">except</span> <span class="ne">ConnectionResetError</span><span class="p">:</span>
|
|
<span class="n">sleep_minutes</span> <span class="o">=</span> <span class="mi">5</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
|
<span class="s2">"Connection reset. "</span>
|
|
<span class="s2">"Waiting </span><span class="si">{0}</span><span class="s2"> minutes before trying again"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">sleep_minutes</span><span class="p">))</span>
|
|
<span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="n">sleep_minutes</span> <span class="o">*</span> <span class="mi">60</span><span class="p">)</span>
|
|
<span class="n">results</span> <span class="o">=</span> <span class="n">get_dmarc_reports_from_inbox</span><span class="p">(</span>
|
|
<span class="n">host</span><span class="o">=</span><span class="n">host</span><span class="p">,</span>
|
|
<span class="n">user</span><span class="o">=</span><span class="n">user</span><span class="p">,</span>
|
|
<span class="n">password</span><span class="o">=</span><span class="n">password</span><span class="p">,</span>
|
|
<span class="n">connection</span><span class="o">=</span><span class="n">connection</span><span class="p">,</span>
|
|
<span class="n">port</span><span class="o">=</span><span class="n">port</span><span class="p">,</span>
|
|
<span class="n">ssl</span><span class="o">=</span><span class="n">ssl</span><span class="p">,</span>
|
|
<span class="n">ssl_context</span><span class="o">=</span><span class="n">ssl_context</span><span class="p">,</span>
|
|
<span class="n">move_supported</span><span class="o">=</span><span class="n">move_supported</span><span class="p">,</span>
|
|
<span class="n">reports_folder</span><span class="o">=</span><span class="n">reports_folder</span><span class="p">,</span>
|
|
<span class="n">archive_folder</span><span class="o">=</span><span class="n">archive_folder</span><span class="p">,</span>
|
|
<span class="n">delete</span><span class="o">=</span><span class="n">delete</span><span class="p">,</span>
|
|
<span class="n">test</span><span class="o">=</span><span class="n">test</span><span class="p">,</span>
|
|
<span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
|
|
<span class="n">dns_timeout</span><span class="o">=</span><span class="n">dns_timeout</span><span class="p">,</span>
|
|
<span class="n">strip_attachment_payloads</span><span class="o">=</span><span class="n">strip_attachment_payloads</span><span class="p">,</span>
|
|
<span class="n">results</span><span class="o">=</span><span class="n">results</span>
|
|
<span class="p">)</span>
|
|
|
|
<span class="k">return</span> <span class="n">results</span>
|
|
<span class="k">except</span> <span class="ne">ConnectionAbortedError</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">IMAPError</span><span class="p">(</span><span class="s2">"Connection aborted"</span><span class="p">)</span>
|
|
<span class="k">except</span> <span class="ne">TimeoutError</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">IMAPError</span><span class="p">(</span><span class="s2">"Connection timed out"</span><span class="p">)</span>
|
|
<span class="k">except</span> <span class="n">SSLError</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">IMAPError</span><span class="p">(</span><span class="s2">"SSL error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
|
<span class="k">except</span> <span class="n">CertificateError</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">IMAPError</span><span class="p">(</span><span class="s2">"Certificate error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
|
|
|
|
|
<div class="viewcode-block" id="save_output"><a class="viewcode-back" href="../index.html#parsedmarc.save_output">[docs]</a><span class="k">def</span> <span class="nf">save_output</span><span class="p">(</span><span class="n">results</span><span class="p">,</span> <span class="n">output_directory</span><span class="o">=</span><span class="s2">"output"</span><span class="p">):</span>
|
|
<span class="sd">"""</span>
|
|
<span class="sd"> Save report data in the given directory</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> results (OrderedDict): Parsing results</span>
|
|
<span class="sd"> output_directory: The patch to the directory to save in</span>
|
|
<span class="sd"> """</span>
|
|
|
|
<span class="n">aggregate_reports</span> <span class="o">=</span> <span class="n">results</span><span class="p">[</span><span class="s2">"aggregate_reports"</span><span class="p">]</span>
|
|
<span class="n">forensic_reports</span> <span class="o">=</span> <span class="n">results</span><span class="p">[</span><span class="s2">"forensic_reports"</span><span class="p">]</span>
|
|
|
|
<span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">output_directory</span><span class="p">):</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isdir</span><span class="p">(</span><span class="n">output_directory</span><span class="p">):</span>
|
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"</span><span class="si">{0}</span><span class="s2"> is not a directory"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">output_directory</span><span class="p">))</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">os</span><span class="o">.</span><span class="n">makedirs</span><span class="p">(</span><span class="n">output_directory</span><span class="p">)</span>
|
|
|
|
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="s2">"</span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">output_directory</span><span class="p">,</span> <span class="s2">"aggregate.json"</span><span class="p">)),</span>
|
|
<span class="s2">"w"</span><span class="p">,</span> <span class="n">newline</span><span class="o">=</span><span class="s2">"</span><span class="se">\n</span><span class="s2">"</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s2">"utf-8"</span><span class="p">)</span> <span class="k">as</span> <span class="n">agg_json</span><span class="p">:</span>
|
|
<span class="n">agg_json</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">aggregate_reports</span><span class="p">,</span> <span class="n">ensure_ascii</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
|
<span class="n">indent</span><span class="o">=</span><span class="mi">2</span><span class="p">))</span>
|
|
|
|
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="s2">"</span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">output_directory</span><span class="p">,</span> <span class="s2">"aggregate.csv"</span><span class="p">)),</span>
|
|
<span class="s2">"w"</span><span class="p">,</span> <span class="n">newline</span><span class="o">=</span><span class="s2">"</span><span class="se">\n</span><span class="s2">"</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s2">"utf-8"</span><span class="p">)</span> <span class="k">as</span> <span class="n">agg_csv</span><span class="p">:</span>
|
|
<span class="n">csv</span> <span class="o">=</span> <span class="n">parsed_aggregate_reports_to_csv</span><span class="p">(</span><span class="n">aggregate_reports</span><span class="p">)</span>
|
|
<span class="n">agg_csv</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">csv</span><span class="p">)</span>
|
|
|
|
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="s2">"</span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">output_directory</span><span class="p">,</span> <span class="s2">"forensic.json"</span><span class="p">)),</span>
|
|
<span class="s2">"w"</span><span class="p">,</span> <span class="n">newline</span><span class="o">=</span><span class="s2">"</span><span class="se">\n</span><span class="s2">"</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s2">"utf-8"</span><span class="p">)</span> <span class="k">as</span> <span class="n">for_json</span><span class="p">:</span>
|
|
<span class="n">for_json</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">forensic_reports</span><span class="p">,</span> <span class="n">ensure_ascii</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
|
<span class="n">indent</span><span class="o">=</span><span class="mi">2</span><span class="p">))</span>
|
|
|
|
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="s2">"</span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">output_directory</span><span class="p">,</span> <span class="s2">"forensic.csv"</span><span class="p">)),</span>
|
|
<span class="s2">"w"</span><span class="p">,</span> <span class="n">newline</span><span class="o">=</span><span class="s2">"</span><span class="se">\n</span><span class="s2">"</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s2">"utf-8"</span><span class="p">)</span> <span class="k">as</span> <span class="n">for_csv</span><span class="p">:</span>
|
|
<span class="n">csv</span> <span class="o">=</span> <span class="n">parsed_forensic_reports_to_csv</span><span class="p">(</span><span class="n">forensic_reports</span><span class="p">)</span>
|
|
<span class="n">for_csv</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">csv</span><span class="p">)</span>
|
|
|
|
<span class="n">samples_directory</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">output_directory</span><span class="p">,</span> <span class="s2">"samples"</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">samples_directory</span><span class="p">):</span>
|
|
<span class="n">os</span><span class="o">.</span><span class="n">makedirs</span><span class="p">(</span><span class="n">samples_directory</span><span class="p">)</span>
|
|
|
|
<span class="n">sample_filenames</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="k">for</span> <span class="n">forensic_report</span> <span class="ow">in</span> <span class="n">forensic_reports</span><span class="p">:</span>
|
|
<span class="n">sample</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"sample"</span><span class="p">]</span>
|
|
<span class="n">message_count</span> <span class="o">=</span> <span class="mi">0</span>
|
|
<span class="n">parsed_sample</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">]</span>
|
|
<span class="n">subject</span> <span class="o">=</span> <span class="n">parsed_sample</span><span class="p">[</span><span class="s2">"filename_safe_subject"</span><span class="p">]</span>
|
|
<span class="n">filename</span> <span class="o">=</span> <span class="n">subject</span>
|
|
|
|
<span class="k">while</span> <span class="n">filename</span> <span class="ow">in</span> <span class="n">sample_filenames</span><span class="p">:</span>
|
|
<span class="n">message_count</span> <span class="o">+=</span> <span class="mi">1</span>
|
|
<span class="n">filename</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2"> (</span><span class="si">{1}</span><span class="s2">)"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">subject</span><span class="p">,</span> <span class="n">message_count</span><span class="p">)</span>
|
|
|
|
<span class="n">sample_filenames</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">filename</span><span class="p">)</span>
|
|
|
|
<span class="n">filename</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">.eml"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">filename</span><span class="p">)</span>
|
|
<span class="n">path</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">samples_directory</span><span class="p">,</span> <span class="n">filename</span><span class="p">)</span>
|
|
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="s2">"w"</span><span class="p">,</span> <span class="n">newline</span><span class="o">=</span><span class="s2">"</span><span class="se">\n</span><span class="s2">"</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s2">"utf-8"</span><span class="p">)</span> <span class="k">as</span> <span class="n">sample_file</span><span class="p">:</span>
|
|
<span class="n">sample_file</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">sample</span><span class="p">)</span></div>
|
|
|
|
|
|
<div class="viewcode-block" id="get_report_zip"><a class="viewcode-back" href="../index.html#parsedmarc.get_report_zip">[docs]</a><span class="k">def</span> <span class="nf">get_report_zip</span><span class="p">(</span><span class="n">results</span><span class="p">):</span>
|
|
<span class="sd">"""</span>
|
|
<span class="sd"> Creates a zip file of parsed report output</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> results (OrderedDict): The parsed results</span>
|
|
|
|
<span class="sd"> Returns:</span>
|
|
<span class="sd"> bytes: zip file bytes</span>
|
|
<span class="sd"> """</span>
|
|
<span class="k">def</span> <span class="nf">add_subdir</span><span class="p">(</span><span class="n">root_path</span><span class="p">,</span> <span class="n">subdir</span><span class="p">):</span>
|
|
<span class="n">subdir_path</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">root_path</span><span class="p">,</span> <span class="n">subdir</span><span class="p">)</span>
|
|
<span class="k">for</span> <span class="n">subdir_root</span><span class="p">,</span> <span class="n">subdir_dirs</span><span class="p">,</span> <span class="n">subdir_files</span> <span class="ow">in</span> <span class="n">os</span><span class="o">.</span><span class="n">walk</span><span class="p">(</span><span class="n">subdir_path</span><span class="p">):</span>
|
|
<span class="k">for</span> <span class="n">subdir_file</span> <span class="ow">in</span> <span class="n">subdir_files</span><span class="p">:</span>
|
|
<span class="n">subdir_file_path</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">root_path</span><span class="p">,</span> <span class="n">subdir</span><span class="p">,</span> <span class="n">subdir_file</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isfile</span><span class="p">(</span><span class="n">subdir_file_path</span><span class="p">):</span>
|
|
<span class="n">rel_path</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">relpath</span><span class="p">(</span><span class="n">subdir_root</span><span class="p">,</span> <span class="n">subdir_file_path</span><span class="p">)</span>
|
|
<span class="n">subdir_arc_name</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">rel_path</span><span class="p">,</span> <span class="n">subdir_file</span><span class="p">)</span>
|
|
<span class="n">zip_file</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">subdir_file_path</span><span class="p">,</span> <span class="n">subdir_arc_name</span><span class="p">)</span>
|
|
<span class="k">for</span> <span class="n">subdir</span> <span class="ow">in</span> <span class="n">subdir_dirs</span><span class="p">:</span>
|
|
<span class="n">add_subdir</span><span class="p">(</span><span class="n">subdir_path</span><span class="p">,</span> <span class="n">subdir</span><span class="p">)</span>
|
|
|
|
<span class="n">storage</span> <span class="o">=</span> <span class="n">BytesIO</span><span class="p">()</span>
|
|
<span class="n">tmp_dir</span> <span class="o">=</span> <span class="n">tempfile</span><span class="o">.</span><span class="n">mkdtemp</span><span class="p">()</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">save_output</span><span class="p">(</span><span class="n">results</span><span class="p">,</span> <span class="n">tmp_dir</span><span class="p">)</span>
|
|
<span class="k">with</span> <span class="n">zipfile</span><span class="o">.</span><span class="n">ZipFile</span><span class="p">(</span><span class="n">storage</span><span class="p">,</span> <span class="s1">'w'</span><span class="p">,</span> <span class="n">zipfile</span><span class="o">.</span><span class="n">ZIP_DEFLATED</span><span class="p">)</span> <span class="k">as</span> <span class="n">zip_file</span><span class="p">:</span>
|
|
<span class="k">for</span> <span class="n">root</span><span class="p">,</span> <span class="n">dirs</span><span class="p">,</span> <span class="n">files</span> <span class="ow">in</span> <span class="n">os</span><span class="o">.</span><span class="n">walk</span><span class="p">(</span><span class="n">tmp_dir</span><span class="p">):</span>
|
|
<span class="k">for</span> <span class="n">file</span> <span class="ow">in</span> <span class="n">files</span><span class="p">:</span>
|
|
<span class="n">file_path</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">root</span><span class="p">,</span> <span class="n">file</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isfile</span><span class="p">(</span><span class="n">file_path</span><span class="p">):</span>
|
|
<span class="n">arcname</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">relpath</span><span class="p">(</span><span class="n">root</span><span class="p">,</span> <span class="n">tmp_dir</span><span class="p">),</span>
|
|
<span class="n">file</span><span class="p">)</span>
|
|
<span class="n">zip_file</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">file_path</span><span class="p">,</span> <span class="n">arcname</span><span class="p">)</span>
|
|
<span class="k">for</span> <span class="n">directory</span> <span class="ow">in</span> <span class="n">dirs</span><span class="p">:</span>
|
|
<span class="n">dir_path</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">root</span><span class="p">,</span> <span class="n">directory</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isdir</span><span class="p">(</span><span class="n">dir_path</span><span class="p">):</span>
|
|
<span class="n">zip_file</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">dir_path</span><span class="p">,</span> <span class="n">directory</span><span class="p">)</span>
|
|
<span class="n">add_subdir</span><span class="p">(</span><span class="n">root</span><span class="p">,</span> <span class="n">directory</span><span class="p">)</span>
|
|
<span class="k">finally</span><span class="p">:</span>
|
|
<span class="n">shutil</span><span class="o">.</span><span class="n">rmtree</span><span class="p">(</span><span class="n">tmp_dir</span><span class="p">)</span>
|
|
|
|
<span class="k">return</span> <span class="n">storage</span><span class="o">.</span><span class="n">getvalue</span><span class="p">()</span></div>
|
|
|
|
|
|
<div class="viewcode-block" id="email_results"><a class="viewcode-back" href="../index.html#parsedmarc.email_results">[docs]</a><span class="k">def</span> <span class="nf">email_results</span><span class="p">(</span><span class="n">results</span><span class="p">,</span> <span class="n">host</span><span class="p">,</span> <span class="n">mail_from</span><span class="p">,</span> <span class="n">mail_to</span><span class="p">,</span> <span class="n">port</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
|
|
<span class="n">ssl</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">user</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">password</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">subject</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">attachment_filename</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">message</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">ssl_context</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
|
<span class="sd">"""</span>
|
|
<span class="sd"> Emails parsing results as a zip file</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> results (OrderedDict): Parsing results</span>
|
|
<span class="sd"> host: Mail server hostname or IP address</span>
|
|
<span class="sd"> mail_from: The value of the message from header</span>
|
|
<span class="sd"> mail_to : A list of addresses to mail to</span>
|
|
<span class="sd"> port (int): Port to use</span>
|
|
<span class="sd"> ssl (bool): Require a SSL connection from the start</span>
|
|
<span class="sd"> user: An optional username</span>
|
|
<span class="sd"> password: An optional password</span>
|
|
<span class="sd"> subject: Overrides the default message subject</span>
|
|
<span class="sd"> attachment_filename: Override the default attachment filename</span>
|
|
<span class="sd"> message: Override the default plain text body</span>
|
|
<span class="sd"> ssl_context: SSL context options</span>
|
|
<span class="sd"> """</span>
|
|
<span class="n">logging</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Emailing report to: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="s2">","</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">mail_to</span><span class="p">)))</span>
|
|
<span class="n">date_string</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2">"</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">attachment_filename</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">attachment_filename</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s2">".zip"</span><span class="p">):</span>
|
|
<span class="n">attachment_filename</span> <span class="o">+=</span> <span class="s2">".zip"</span>
|
|
<span class="n">filename</span> <span class="o">=</span> <span class="n">attachment_filename</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">filename</span> <span class="o">=</span> <span class="s2">"DMARC-</span><span class="si">{0}</span><span class="s2">.zip"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">date_string</span><span class="p">)</span>
|
|
|
|
<span class="k">assert</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">mail_to</span><span class="p">,</span> <span class="nb">list</span><span class="p">)</span>
|
|
|
|
<span class="n">msg</span> <span class="o">=</span> <span class="n">MIMEMultipart</span><span class="p">()</span>
|
|
<span class="n">msg</span><span class="p">[</span><span class="s1">'From'</span><span class="p">]</span> <span class="o">=</span> <span class="n">mail_from</span>
|
|
<span class="n">msg</span><span class="p">[</span><span class="s1">'To'</span><span class="p">]</span> <span class="o">=</span> <span class="s2">", "</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">mail_to</span><span class="p">)</span>
|
|
<span class="n">msg</span><span class="p">[</span><span class="s1">'Date'</span><span class="p">]</span> <span class="o">=</span> <span class="n">email</span><span class="o">.</span><span class="n">utils</span><span class="o">.</span><span class="n">formatdate</span><span class="p">(</span><span class="n">localtime</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
|
<span class="n">msg</span><span class="p">[</span><span class="s1">'Subject'</span><span class="p">]</span> <span class="o">=</span> <span class="n">subject</span> <span class="ow">or</span> <span class="s2">"DMARC results for </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">date_string</span><span class="p">)</span>
|
|
<span class="n">text</span> <span class="o">=</span> <span class="n">message</span> <span class="ow">or</span> <span class="s2">"Please see the attached zip file</span><span class="se">\n</span><span class="s2">"</span>
|
|
|
|
<span class="n">msg</span><span class="o">.</span><span class="n">attach</span><span class="p">(</span><span class="n">MIMEText</span><span class="p">(</span><span class="n">text</span><span class="p">))</span>
|
|
|
|
<span class="n">zip_bytes</span> <span class="o">=</span> <span class="n">get_report_zip</span><span class="p">(</span><span class="n">results</span><span class="p">)</span>
|
|
<span class="n">part</span> <span class="o">=</span> <span class="n">MIMEApplication</span><span class="p">(</span><span class="n">zip_bytes</span><span class="p">,</span> <span class="n">Name</span><span class="o">=</span><span class="n">filename</span><span class="p">)</span>
|
|
|
|
<span class="n">part</span><span class="p">[</span><span class="s1">'Content-Disposition'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'attachment; filename="</span><span class="si">{0}</span><span class="s1">"'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">filename</span><span class="p">)</span>
|
|
<span class="n">msg</span><span class="o">.</span><span class="n">attach</span><span class="p">(</span><span class="n">part</span><span class="p">)</span>
|
|
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="n">ssl_context</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">ssl_context</span> <span class="o">=</span> <span class="n">create_default_context</span><span class="p">()</span>
|
|
<span class="k">if</span> <span class="n">ssl</span><span class="p">:</span>
|
|
<span class="n">server</span> <span class="o">=</span> <span class="n">smtplib</span><span class="o">.</span><span class="n">SMTP_SSL</span><span class="p">(</span><span class="n">host</span><span class="p">,</span> <span class="n">port</span><span class="o">=</span><span class="n">port</span><span class="p">,</span> <span class="n">context</span><span class="o">=</span><span class="n">ssl_context</span><span class="p">)</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="n">host</span><span class="p">,</span> <span class="n">port</span><span class="p">)</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">ehlo_or_helo_if_needed</span><span class="p">()</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">server</span> <span class="o">=</span> <span class="n">smtplib</span><span class="o">.</span><span class="n">SMTP</span><span class="p">(</span><span class="n">host</span><span class="p">,</span> <span class="n">port</span><span class="o">=</span><span class="n">port</span><span class="p">)</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="n">host</span><span class="p">,</span> <span class="n">port</span><span class="p">)</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">ehlo_or_helo_if_needed</span><span class="p">()</span>
|
|
<span class="k">if</span> <span class="n">server</span><span class="o">.</span><span class="n">has_extn</span><span class="p">(</span><span class="s2">"starttls"</span><span class="p">):</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">starttls</span><span class="p">(</span><span class="n">context</span><span class="o">=</span><span class="n">ssl_context</span><span class="p">)</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">ehlo</span><span class="p">()</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"SMTP server does not support STARTTLS. "</span>
|
|
<span class="s2">"Proceeding in plain text!"</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">user</span> <span class="ow">and</span> <span class="n">password</span><span class="p">:</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">login</span><span class="p">(</span><span class="n">user</span><span class="p">,</span> <span class="n">password</span><span class="p">)</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">sendmail</span><span class="p">(</span><span class="n">mail_from</span><span class="p">,</span> <span class="n">mail_to</span><span class="p">,</span> <span class="n">msg</span><span class="o">.</span><span class="n">as_string</span><span class="p">())</span>
|
|
<span class="k">except</span> <span class="n">smtplib</span><span class="o">.</span><span class="n">SMTPException</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
|
<span class="n">error</span> <span class="o">=</span> <span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="o">.</span><span class="n">lstrip</span><span class="p">(</span><span class="s2">"b'"</span><span class="p">)</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s2">"'"</span><span class="p">)</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s2">"."</span><span class="p">)</span>
|
|
<span class="k">raise</span> <span class="n">SMTPError</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
|
|
<span class="k">except</span> <span class="n">socket</span><span class="o">.</span><span class="n">gaierror</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">SMTPError</span><span class="p">(</span><span class="s2">"DNS resolution failed"</span><span class="p">)</span>
|
|
<span class="k">except</span> <span class="ne">ConnectionRefusedError</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">SMTPError</span><span class="p">(</span><span class="s2">"Connection refused"</span><span class="p">)</span>
|
|
<span class="k">except</span> <span class="ne">ConnectionResetError</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">SMTPError</span><span class="p">(</span><span class="s2">"Connection reset"</span><span class="p">)</span>
|
|
<span class="k">except</span> <span class="ne">ConnectionAbortedError</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">SMTPError</span><span class="p">(</span><span class="s2">"Connection aborted"</span><span class="p">)</span>
|
|
<span class="k">except</span> <span class="ne">TimeoutError</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">SMTPError</span><span class="p">(</span><span class="s2">"Connection timed out"</span><span class="p">)</span>
|
|
<span class="k">except</span> <span class="n">SSLError</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">SMTPError</span><span class="p">(</span><span class="s2">"SSL error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
|
<span class="k">except</span> <span class="n">CertificateError</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">SMTPError</span><span class="p">(</span><span class="s2">"Certificate error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
|
|
|
|
|
<div class="viewcode-block" id="watch_inbox"><a class="viewcode-back" href="../index.html#parsedmarc.watch_inbox">[docs]</a><span class="k">def</span> <span class="nf">watch_inbox</span><span class="p">(</span><span class="n">host</span><span class="p">,</span> <span class="n">username</span><span class="p">,</span> <span class="n">password</span><span class="p">,</span> <span class="n">callback</span><span class="p">,</span> <span class="n">port</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">ssl</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
|
<span class="n">ssl_context</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">reports_folder</span><span class="o">=</span><span class="s2">"INBOX"</span><span class="p">,</span>
|
|
<span class="n">archive_folder</span><span class="o">=</span><span class="s2">"Archive"</span><span class="p">,</span> <span class="n">delete</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">test</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">wait</span><span class="o">=</span><span class="mi">30</span><span class="p">,</span>
|
|
<span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">dns_timeout</span><span class="o">=</span><span class="mf">6.0</span><span class="p">,</span>
|
|
<span class="n">strip_attachment_payloads</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
|
|
<span class="sd">"""</span>
|
|
<span class="sd"> Use an IDLE IMAP connection to parse incoming emails, and pass the results</span>
|
|
<span class="sd"> to a callback function</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> host: The mail server hostname or IP address</span>
|
|
<span class="sd"> username: The mail server username</span>
|
|
<span class="sd"> password: The mail server password</span>
|
|
<span class="sd"> callback: The callback function to receive the parsing results</span>
|
|
<span class="sd"> port: The mail server port</span>
|
|
<span class="sd"> ssl (bool): Use SSL/TLS</span>
|
|
<span class="sd"> ssl_context (SSLContext): A SSL context</span>
|
|
<span class="sd"> reports_folder: The IMAP folder where reports can be found</span>
|
|
<span class="sd"> archive_folder: The folder to move processed mail to</span>
|
|
<span class="sd"> delete (bool): Delete messages after processing them</span>
|
|
<span class="sd"> test (bool): Do not move or delete messages after processing them</span>
|
|
<span class="sd"> wait (int): Number of seconds to wait for a IMAP IDLE response</span>
|
|
<span class="sd"> nameservers (list): A list of one or more nameservers to use</span>
|
|
<span class="sd"> (Cloudflare's public DNS resolvers by default)</span>
|
|
<span class="sd"> dns_timeout (float): Set the DNS query timeout</span>
|
|
<span class="sd"> strip_attachment_payloads (bool): Replace attachment payloads in</span>
|
|
<span class="sd"> forensic report samples with None</span>
|
|
<span class="sd"> """</span>
|
|
<span class="n">rf</span> <span class="o">=</span> <span class="n">reports_folder</span>
|
|
<span class="n">af</span> <span class="o">=</span> <span class="n">archive_folder</span>
|
|
<span class="n">ns</span> <span class="o">=</span> <span class="n">nameservers</span>
|
|
<span class="n">dt</span> <span class="o">=</span> <span class="n">dns_timeout</span>
|
|
<span class="k">if</span> <span class="n">ssl_context</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">ssl_context</span> <span class="o">=</span> <span class="n">create_default_context</span><span class="p">()</span>
|
|
<span class="n">server</span> <span class="o">=</span> <span class="n">imapclient</span><span class="o">.</span><span class="n">IMAPClient</span><span class="p">(</span><span class="n">host</span><span class="p">,</span> <span class="n">port</span><span class="o">=</span><span class="n">port</span><span class="p">,</span> <span class="n">ssl</span><span class="o">=</span><span class="n">ssl</span><span class="p">,</span>
|
|
<span class="n">ssl_context</span><span class="o">=</span><span class="n">ssl_context</span><span class="p">,</span>
|
|
<span class="n">use_uid</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
|
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">login</span><span class="p">(</span><span class="n">username</span><span class="p">,</span> <span class="n">password</span><span class="p">)</span>
|
|
<span class="n">imap_capabilities</span> <span class="o">=</span> <span class="n">get_imap_capabilities</span><span class="p">(</span><span class="n">server</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="s2">"IDLE"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">imap_capabilities</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">IMAPError</span><span class="p">(</span><span class="s2">"Cannot watch inbox: IMAP server does not support "</span>
|
|
<span class="s2">"the IDLE command"</span><span class="p">)</span>
|
|
|
|
<span class="n">ms</span> <span class="o">=</span> <span class="s2">"MOVE"</span> <span class="ow">in</span> <span class="n">imap_capabilities</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">select_folder</span><span class="p">(</span><span class="n">rf</span><span class="p">)</span>
|
|
<span class="n">idle_start_time</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">monotonic</span><span class="p">()</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">idle</span><span class="p">()</span>
|
|
|
|
<span class="k">except</span> <span class="n">imapclient</span><span class="o">.</span><span class="n">exceptions</span><span class="o">.</span><span class="n">IMAPClientError</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
|
<span class="n">error</span> <span class="o">=</span> <span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"b'"</span><span class="p">,</span> <span class="s2">""</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"'"</span><span class="p">,</span> <span class="s2">""</span><span class="p">)</span>
|
|
<span class="c1"># Workaround for random Exchange/Office365 IMAP errors</span>
|
|
<span class="k">if</span> <span class="s2">"unexpected response"</span> <span class="ow">in</span> <span class="n">error</span> <span class="ow">or</span> <span class="s2">"BAD"</span> <span class="ow">in</span> <span class="n">error</span><span class="p">:</span>
|
|
<span class="n">sleep_minutes</span> <span class="o">=</span> <span class="mi">5</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
|
<span class="s2">"</span><span class="si">{0}</span><span class="s2">. "</span>
|
|
<span class="s2">"Waiting </span><span class="si">{1}</span><span class="s2"> minutes before trying again"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
|
<span class="n">error</span><span class="p">,</span>
|
|
<span class="n">sleep_minutes</span><span class="p">))</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Reconnecting watcher"</span><span class="p">)</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">logout</span><span class="p">()</span>
|
|
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Failed to log out: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
|
<span class="n">server</span> <span class="o">=</span> <span class="n">imapclient</span><span class="o">.</span><span class="n">IMAPClient</span><span class="p">(</span><span class="n">host</span><span class="p">)</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">login</span><span class="p">(</span><span class="n">username</span><span class="p">,</span> <span class="n">password</span><span class="p">)</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">select_folder</span><span class="p">(</span><span class="n">rf</span><span class="p">)</span>
|
|
<span class="n">idle_start_time</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">monotonic</span><span class="p">()</span>
|
|
<span class="n">ms</span> <span class="o">=</span> <span class="s2">"MOVE"</span> <span class="ow">in</span> <span class="n">get_imap_capabilities</span><span class="p">(</span><span class="n">server</span><span class="p">)</span>
|
|
<span class="n">sa</span> <span class="o">=</span> <span class="n">strip_attachment_payloads</span>
|
|
<span class="n">res</span> <span class="o">=</span> <span class="n">get_dmarc_reports_from_inbox</span><span class="p">(</span><span class="n">connection</span><span class="o">=</span><span class="n">server</span><span class="p">,</span>
|
|
<span class="n">move_supported</span><span class="o">=</span><span class="n">ms</span><span class="p">,</span>
|
|
<span class="n">reports_folder</span><span class="o">=</span><span class="n">rf</span><span class="p">,</span>
|
|
<span class="n">archive_folder</span><span class="o">=</span><span class="n">af</span><span class="p">,</span>
|
|
<span class="n">delete</span><span class="o">=</span><span class="n">delete</span><span class="p">,</span>
|
|
<span class="n">test</span><span class="o">=</span><span class="n">test</span><span class="p">,</span>
|
|
<span class="n">nameservers</span><span class="o">=</span><span class="n">ns</span><span class="p">,</span>
|
|
<span class="n">dns_timeout</span><span class="o">=</span><span class="n">dt</span><span class="p">,</span>
|
|
<span class="n">strip_attachment_payloads</span><span class="o">=</span><span class="n">sa</span><span class="p">)</span>
|
|
<span class="n">callback</span><span class="p">(</span><span class="n">res</span><span class="p">)</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">idle</span><span class="p">()</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">IMAPError</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
|
|
<span class="k">except</span> <span class="n">socket</span><span class="o">.</span><span class="n">gaierror</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">IMAPError</span><span class="p">(</span><span class="s2">"DNS resolution failed"</span><span class="p">)</span>
|
|
<span class="k">except</span> <span class="ne">ConnectionRefusedError</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">IMAPError</span><span class="p">(</span><span class="s2">"Connection refused"</span><span class="p">)</span>
|
|
<span class="k">except</span> <span class="ne">ConnectionResetError</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"IMAP error: Connection reset"</span><span class="p">)</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Reconnecting watcher"</span><span class="p">)</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">shutdown</span><span class="p">()</span>
|
|
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Failed to disconnect: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
|
<span class="n">server</span> <span class="o">=</span> <span class="n">imapclient</span><span class="o">.</span><span class="n">IMAPClient</span><span class="p">(</span><span class="n">host</span><span class="p">)</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">login</span><span class="p">(</span><span class="n">username</span><span class="p">,</span> <span class="n">password</span><span class="p">)</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">select_folder</span><span class="p">(</span><span class="n">rf</span><span class="p">)</span>
|
|
<span class="n">idle_start_time</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">monotonic</span><span class="p">()</span>
|
|
<span class="n">ms</span> <span class="o">=</span> <span class="s2">"MOVE"</span> <span class="ow">in</span> <span class="n">get_imap_capabilities</span><span class="p">(</span><span class="n">server</span><span class="p">)</span>
|
|
<span class="n">res</span> <span class="o">=</span> <span class="n">get_dmarc_reports_from_inbox</span><span class="p">(</span><span class="n">connection</span><span class="o">=</span><span class="n">server</span><span class="p">,</span>
|
|
<span class="n">move_supported</span><span class="o">=</span><span class="n">ms</span><span class="p">,</span>
|
|
<span class="n">reports_folder</span><span class="o">=</span><span class="n">rf</span><span class="p">,</span>
|
|
<span class="n">archive_folder</span><span class="o">=</span><span class="n">af</span><span class="p">,</span>
|
|
<span class="n">delete</span><span class="o">=</span><span class="n">delete</span><span class="p">,</span>
|
|
<span class="n">test</span><span class="o">=</span><span class="n">test</span><span class="p">,</span>
|
|
<span class="n">nameservers</span><span class="o">=</span><span class="n">ns</span><span class="p">,</span>
|
|
<span class="n">dns_timeout</span><span class="o">=</span><span class="n">dt</span><span class="p">)</span>
|
|
<span class="n">callback</span><span class="p">(</span><span class="n">res</span><span class="p">)</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">idle</span><span class="p">()</span>
|
|
<span class="k">except</span> <span class="ne">KeyError</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"IMAP error: Server returned unexpected result"</span><span class="p">)</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Reconnecting watcher"</span><span class="p">)</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">logout</span><span class="p">()</span>
|
|
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Failed to log out: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
|
<span class="n">server</span> <span class="o">=</span> <span class="n">imapclient</span><span class="o">.</span><span class="n">IMAPClient</span><span class="p">(</span><span class="n">host</span><span class="p">)</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">login</span><span class="p">(</span><span class="n">username</span><span class="p">,</span> <span class="n">password</span><span class="p">)</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">select_folder</span><span class="p">(</span><span class="n">rf</span><span class="p">)</span>
|
|
<span class="n">idle_start_time</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">monotonic</span><span class="p">()</span>
|
|
<span class="n">ms</span> <span class="o">=</span> <span class="s2">"MOVE"</span> <span class="ow">in</span> <span class="n">get_imap_capabilities</span><span class="p">(</span><span class="n">server</span><span class="p">)</span>
|
|
<span class="n">res</span> <span class="o">=</span> <span class="n">get_dmarc_reports_from_inbox</span><span class="p">(</span><span class="n">connection</span><span class="o">=</span><span class="n">server</span><span class="p">,</span>
|
|
<span class="n">move_supported</span><span class="o">=</span><span class="n">ms</span><span class="p">,</span>
|
|
<span class="n">reports_folder</span><span class="o">=</span><span class="n">rf</span><span class="p">,</span>
|
|
<span class="n">archive_folder</span><span class="o">=</span><span class="n">af</span><span class="p">,</span>
|
|
<span class="n">delete</span><span class="o">=</span><span class="n">delete</span><span class="p">,</span>
|
|
<span class="n">test</span><span class="o">=</span><span class="n">test</span><span class="p">,</span>
|
|
<span class="n">nameservers</span><span class="o">=</span><span class="n">ns</span><span class="p">,</span>
|
|
<span class="n">dns_timeout</span><span class="o">=</span><span class="n">dt</span><span class="p">)</span>
|
|
<span class="n">callback</span><span class="p">(</span><span class="n">res</span><span class="p">)</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">idle</span><span class="p">()</span>
|
|
<span class="k">except</span> <span class="ne">ConnectionAbortedError</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">IMAPError</span><span class="p">(</span><span class="s2">"Connection aborted"</span><span class="p">)</span>
|
|
<span class="k">except</span> <span class="ne">TimeoutError</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">IMAPError</span><span class="p">(</span><span class="s2">"Connection timed out"</span><span class="p">)</span>
|
|
<span class="k">except</span> <span class="n">SSLError</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">IMAPError</span><span class="p">(</span><span class="s2">"SSL error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
|
<span class="k">except</span> <span class="n">CertificateError</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">IMAPError</span><span class="p">(</span><span class="s2">"Certificate error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
|
<span class="k">except</span> <span class="ne">BrokenPipeError</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"IMAP error: Broken pipe"</span><span class="p">)</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Reconnecting watcher"</span><span class="p">)</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">shutdown</span><span class="p">()</span>
|
|
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Failed to disconnect: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
|
<span class="n">server</span> <span class="o">=</span> <span class="n">imapclient</span><span class="o">.</span><span class="n">IMAPClient</span><span class="p">(</span><span class="n">host</span><span class="p">)</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">login</span><span class="p">(</span><span class="n">username</span><span class="p">,</span> <span class="n">password</span><span class="p">)</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">select_folder</span><span class="p">(</span><span class="n">rf</span><span class="p">)</span>
|
|
<span class="n">idle_start_time</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">monotonic</span><span class="p">()</span>
|
|
<span class="n">ms</span> <span class="o">=</span> <span class="s2">"MOVE"</span> <span class="ow">in</span> <span class="n">get_imap_capabilities</span><span class="p">(</span><span class="n">server</span><span class="p">)</span>
|
|
<span class="n">res</span> <span class="o">=</span> <span class="n">get_dmarc_reports_from_inbox</span><span class="p">(</span><span class="n">connection</span><span class="o">=</span><span class="n">server</span><span class="p">,</span>
|
|
<span class="n">move_supported</span><span class="o">=</span><span class="n">ms</span><span class="p">,</span>
|
|
<span class="n">reports_folder</span><span class="o">=</span><span class="n">rf</span><span class="p">,</span>
|
|
<span class="n">archive_folder</span><span class="o">=</span><span class="n">af</span><span class="p">,</span>
|
|
<span class="n">delete</span><span class="o">=</span><span class="n">delete</span><span class="p">,</span>
|
|
<span class="n">test</span><span class="o">=</span><span class="n">test</span><span class="p">,</span>
|
|
<span class="n">nameservers</span><span class="o">=</span><span class="n">ns</span><span class="p">,</span>
|
|
<span class="n">dns_timeout</span><span class="o">=</span><span class="n">dt</span><span class="p">)</span>
|
|
<span class="n">callback</span><span class="p">(</span><span class="n">res</span><span class="p">)</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">idle</span><span class="p">()</span>
|
|
|
|
<span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="c1"># Refresh the IDLE session every 5 minutes to stay connected</span>
|
|
<span class="k">if</span> <span class="n">time</span><span class="o">.</span><span class="n">monotonic</span><span class="p">()</span> <span class="o">-</span> <span class="n">idle_start_time</span> <span class="o">></span> <span class="mi">5</span> <span class="o">*</span> <span class="mi">60</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"IMAP: Refreshing IDLE session"</span><span class="p">)</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">idle_done</span><span class="p">()</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">idle</span><span class="p">()</span>
|
|
<span class="n">idle_start_time</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">monotonic</span><span class="p">()</span>
|
|
<span class="n">responses</span> <span class="o">=</span> <span class="n">server</span><span class="o">.</span><span class="n">idle_check</span><span class="p">(</span><span class="n">timeout</span><span class="o">=</span><span class="n">wait</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">responses</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">responses</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
|
|
<span class="c1"># Gmail/G-Suite does not generate anything in the responses</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">idle_done</span><span class="p">()</span>
|
|
<span class="n">res</span> <span class="o">=</span> <span class="n">get_dmarc_reports_from_inbox</span><span class="p">(</span><span class="n">connection</span><span class="o">=</span><span class="n">server</span><span class="p">,</span>
|
|
<span class="n">move_supported</span><span class="o">=</span><span class="n">ms</span><span class="p">,</span>
|
|
<span class="n">reports_folder</span><span class="o">=</span><span class="n">rf</span><span class="p">,</span>
|
|
<span class="n">archive_folder</span><span class="o">=</span><span class="n">af</span><span class="p">,</span>
|
|
<span class="n">delete</span><span class="o">=</span><span class="n">delete</span><span class="p">,</span>
|
|
<span class="n">test</span><span class="o">=</span><span class="n">test</span><span class="p">,</span>
|
|
<span class="n">nameservers</span><span class="o">=</span><span class="n">ns</span><span class="p">,</span>
|
|
<span class="n">dns_timeout</span><span class="o">=</span><span class="n">dt</span><span class="p">)</span>
|
|
<span class="n">callback</span><span class="p">(</span><span class="n">res</span><span class="p">)</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">idle</span><span class="p">()</span>
|
|
<span class="n">idle_start_time</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">monotonic</span><span class="p">()</span>
|
|
<span class="k">for</span> <span class="n">response</span> <span class="ow">in</span> <span class="n">responses</span><span class="p">:</span>
|
|
<span class="n">logging</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Received response: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">response</span><span class="p">))</span>
|
|
<span class="k">if</span> <span class="n">response</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">></span> <span class="mi">0</span> <span class="ow">and</span> <span class="n">response</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="o">==</span> <span class="sa">b</span><span class="s1">'RECENT'</span><span class="p">:</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">idle_done</span><span class="p">()</span>
|
|
<span class="n">res</span> <span class="o">=</span> <span class="n">get_dmarc_reports_from_inbox</span><span class="p">(</span><span class="n">connection</span><span class="o">=</span><span class="n">server</span><span class="p">,</span>
|
|
<span class="n">move_supported</span><span class="o">=</span><span class="n">ms</span><span class="p">,</span>
|
|
<span class="n">reports_folder</span><span class="o">=</span><span class="n">rf</span><span class="p">,</span>
|
|
<span class="n">archive_folder</span><span class="o">=</span><span class="n">af</span><span class="p">,</span>
|
|
<span class="n">delete</span><span class="o">=</span><span class="n">delete</span><span class="p">,</span>
|
|
<span class="n">test</span><span class="o">=</span><span class="n">test</span><span class="p">,</span>
|
|
<span class="n">nameservers</span><span class="o">=</span><span class="n">ns</span><span class="p">,</span>
|
|
<span class="n">dns_timeout</span><span class="o">=</span><span class="n">dt</span><span class="p">)</span>
|
|
<span class="n">callback</span><span class="p">(</span><span class="n">res</span><span class="p">)</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">idle</span><span class="p">()</span>
|
|
<span class="n">idle_start_time</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">monotonic</span><span class="p">()</span>
|
|
<span class="k">break</span>
|
|
<span class="k">except</span> <span class="n">imapclient</span><span class="o">.</span><span class="n">exceptions</span><span class="o">.</span><span class="n">IMAPClientError</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
|
<span class="n">error</span> <span class="o">=</span> <span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"b'"</span><span class="p">,</span> <span class="s2">""</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"'"</span><span class="p">,</span> <span class="s2">""</span><span class="p">)</span>
|
|
<span class="c1"># Workaround for random Exchange/Office365 IMAP errors</span>
|
|
<span class="k">if</span> <span class="s2">"unexpected response"</span> <span class="ow">in</span> <span class="n">error</span> <span class="ow">or</span> <span class="s2">"BAD"</span> <span class="ow">in</span> <span class="n">error</span><span class="p">:</span>
|
|
<span class="n">sleep_minutes</span> <span class="o">=</span> <span class="mi">5</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
|
<span class="s2">"</span><span class="si">{0}</span><span class="s2">. "</span>
|
|
<span class="s2">"Waiting </span><span class="si">{1}</span><span class="s2"> minutes before trying again"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
|
<span class="n">error</span><span class="p">,</span>
|
|
<span class="n">sleep_minutes</span><span class="p">))</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Reconnecting watcher"</span><span class="p">)</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">logout</span><span class="p">()</span>
|
|
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Failed to disconnect: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
|
<span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
|
<span class="n">server</span> <span class="o">=</span> <span class="n">imapclient</span><span class="o">.</span><span class="n">IMAPClient</span><span class="p">(</span><span class="n">host</span><span class="p">)</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">login</span><span class="p">(</span><span class="n">username</span><span class="p">,</span> <span class="n">password</span><span class="p">)</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">select_folder</span><span class="p">(</span><span class="n">rf</span><span class="p">)</span>
|
|
<span class="n">idle_start_time</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">monotonic</span><span class="p">()</span>
|
|
<span class="n">ms</span> <span class="o">=</span> <span class="s2">"MOVE"</span> <span class="ow">in</span> <span class="n">get_imap_capabilities</span><span class="p">(</span><span class="n">server</span><span class="p">)</span>
|
|
<span class="n">res</span> <span class="o">=</span> <span class="n">get_dmarc_reports_from_inbox</span><span class="p">(</span><span class="n">connection</span><span class="o">=</span><span class="n">server</span><span class="p">,</span>
|
|
<span class="n">move_supported</span><span class="o">=</span><span class="n">ms</span><span class="p">,</span>
|
|
<span class="n">reports_folder</span><span class="o">=</span><span class="n">rf</span><span class="p">,</span>
|
|
<span class="n">archive_folder</span><span class="o">=</span><span class="n">af</span><span class="p">,</span>
|
|
<span class="n">delete</span><span class="o">=</span><span class="n">delete</span><span class="p">,</span>
|
|
<span class="n">test</span><span class="o">=</span><span class="n">test</span><span class="p">,</span>
|
|
<span class="n">nameservers</span><span class="o">=</span><span class="n">ns</span><span class="p">,</span>
|
|
<span class="n">dns_timeout</span><span class="o">=</span><span class="n">dt</span><span class="p">)</span>
|
|
<span class="n">callback</span><span class="p">(</span><span class="n">res</span><span class="p">)</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">idle</span><span class="p">()</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">IMAPError</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
|
|
<span class="k">except</span> <span class="n">socket</span><span class="o">.</span><span class="n">gaierror</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">IMAPError</span><span class="p">(</span><span class="s2">"DNS resolution failed"</span><span class="p">)</span>
|
|
<span class="k">except</span> <span class="ne">ConnectionRefusedError</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">IMAPError</span><span class="p">(</span><span class="s2">"Connection refused"</span><span class="p">)</span>
|
|
<span class="k">except</span> <span class="p">(</span><span class="ne">KeyError</span><span class="p">,</span> <span class="n">socket</span><span class="o">.</span><span class="n">error</span><span class="p">,</span> <span class="ne">BrokenPipeError</span><span class="p">,</span> <span class="ne">ConnectionResetError</span><span class="p">):</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"IMAP error: Connection reset"</span><span class="p">)</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Reconnecting watcher"</span><span class="p">)</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">logout</span><span class="p">()</span>
|
|
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Failed to disconnect: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
|
<span class="n">server</span> <span class="o">=</span> <span class="n">imapclient</span><span class="o">.</span><span class="n">IMAPClient</span><span class="p">(</span><span class="n">host</span><span class="p">)</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">login</span><span class="p">(</span><span class="n">username</span><span class="p">,</span> <span class="n">password</span><span class="p">)</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">select_folder</span><span class="p">(</span><span class="n">rf</span><span class="p">)</span>
|
|
<span class="n">idle_start_time</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">monotonic</span><span class="p">()</span>
|
|
<span class="n">ms</span> <span class="o">=</span> <span class="s2">"MOVE"</span> <span class="ow">in</span> <span class="n">get_imap_capabilities</span><span class="p">(</span><span class="n">server</span><span class="p">)</span>
|
|
<span class="n">res</span> <span class="o">=</span> <span class="n">get_dmarc_reports_from_inbox</span><span class="p">(</span><span class="n">connection</span><span class="o">=</span><span class="n">server</span><span class="p">,</span>
|
|
<span class="n">move_supported</span><span class="o">=</span><span class="n">ms</span><span class="p">,</span>
|
|
<span class="n">reports_folder</span><span class="o">=</span><span class="n">rf</span><span class="p">,</span>
|
|
<span class="n">archive_folder</span><span class="o">=</span><span class="n">af</span><span class="p">,</span>
|
|
<span class="n">delete</span><span class="o">=</span><span class="n">delete</span><span class="p">,</span>
|
|
<span class="n">test</span><span class="o">=</span><span class="n">test</span><span class="p">,</span>
|
|
<span class="n">nameservers</span><span class="o">=</span><span class="n">ns</span><span class="p">,</span>
|
|
<span class="n">dns_timeout</span><span class="o">=</span><span class="n">dt</span><span class="p">)</span>
|
|
<span class="n">callback</span><span class="p">(</span><span class="n">res</span><span class="p">)</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">idle</span><span class="p">()</span>
|
|
<span class="k">except</span> <span class="ne">KeyError</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"IMAP error: Server returned unexpected result"</span><span class="p">)</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Reconnecting watcher"</span><span class="p">)</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">logout</span><span class="p">()</span>
|
|
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Failed to log out: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
|
<span class="n">server</span> <span class="o">=</span> <span class="n">imapclient</span><span class="o">.</span><span class="n">IMAPClient</span><span class="p">(</span><span class="n">host</span><span class="p">)</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">login</span><span class="p">(</span><span class="n">username</span><span class="p">,</span> <span class="n">password</span><span class="p">)</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">select_folder</span><span class="p">(</span><span class="n">rf</span><span class="p">)</span>
|
|
<span class="n">idle_start_time</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">monotonic</span><span class="p">()</span>
|
|
<span class="n">ms</span> <span class="o">=</span> <span class="s2">"MOVE"</span> <span class="ow">in</span> <span class="n">get_imap_capabilities</span><span class="p">(</span><span class="n">server</span><span class="p">)</span>
|
|
<span class="n">res</span> <span class="o">=</span> <span class="n">get_dmarc_reports_from_inbox</span><span class="p">(</span><span class="n">connection</span><span class="o">=</span><span class="n">server</span><span class="p">,</span>
|
|
<span class="n">move_supported</span><span class="o">=</span><span class="n">ms</span><span class="p">,</span>
|
|
<span class="n">reports_folder</span><span class="o">=</span><span class="n">rf</span><span class="p">,</span>
|
|
<span class="n">archive_folder</span><span class="o">=</span><span class="n">af</span><span class="p">,</span>
|
|
<span class="n">delete</span><span class="o">=</span><span class="n">delete</span><span class="p">,</span>
|
|
<span class="n">test</span><span class="o">=</span><span class="n">test</span><span class="p">,</span>
|
|
<span class="n">nameservers</span><span class="o">=</span><span class="n">ns</span><span class="p">,</span>
|
|
<span class="n">dns_timeout</span><span class="o">=</span><span class="n">dt</span><span class="p">)</span>
|
|
<span class="n">callback</span><span class="p">(</span><span class="n">res</span><span class="p">)</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">idle</span><span class="p">()</span>
|
|
<span class="k">except</span> <span class="ne">ConnectionAbortedError</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">IMAPError</span><span class="p">(</span><span class="s2">"Connection aborted"</span><span class="p">)</span>
|
|
<span class="k">except</span> <span class="ne">TimeoutError</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">IMAPError</span><span class="p">(</span><span class="s2">"Connection timed out"</span><span class="p">)</span>
|
|
<span class="k">except</span> <span class="n">SSLError</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">IMAPError</span><span class="p">(</span><span class="s2">"SSL error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
|
<span class="k">except</span> <span class="n">CertificateError</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">IMAPError</span><span class="p">(</span><span class="s2">"Certificate error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
|
<span class="k">except</span> <span class="ne">BrokenPipeError</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"IMAP error: Broken pipe"</span><span class="p">)</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Reconnecting watcher"</span><span class="p">)</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">shutdown</span><span class="p">()</span>
|
|
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Failed to disconnect: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
|
<span class="n">server</span> <span class="o">=</span> <span class="n">imapclient</span><span class="o">.</span><span class="n">IMAPClient</span><span class="p">(</span><span class="n">host</span><span class="p">)</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">login</span><span class="p">(</span><span class="n">username</span><span class="p">,</span> <span class="n">password</span><span class="p">)</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">select_folder</span><span class="p">(</span><span class="n">rf</span><span class="p">)</span>
|
|
<span class="n">idle_start_time</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">monotonic</span><span class="p">()</span>
|
|
<span class="n">res</span> <span class="o">=</span> <span class="n">get_dmarc_reports_from_inbox</span><span class="p">(</span><span class="n">connection</span><span class="o">=</span><span class="n">server</span><span class="p">,</span>
|
|
<span class="n">move_supported</span><span class="o">=</span><span class="n">ms</span><span class="p">,</span>
|
|
<span class="n">reports_folder</span><span class="o">=</span><span class="n">rf</span><span class="p">,</span>
|
|
<span class="n">archive_folder</span><span class="o">=</span><span class="n">af</span><span class="p">,</span>
|
|
<span class="n">delete</span><span class="o">=</span><span class="n">delete</span><span class="p">,</span>
|
|
<span class="n">test</span><span class="o">=</span><span class="n">test</span><span class="p">,</span>
|
|
<span class="n">nameservers</span><span class="o">=</span><span class="n">ns</span><span class="p">,</span>
|
|
<span class="n">dns_timeout</span><span class="o">=</span><span class="n">dt</span><span class="p">)</span>
|
|
<span class="n">callback</span><span class="p">(</span><span class="n">res</span><span class="p">)</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">idle</span><span class="p">()</span>
|
|
<span class="k">except</span> <span class="ne">KeyboardInterrupt</span><span class="p">:</span>
|
|
<span class="k">break</span>
|
|
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">idle_done</span><span class="p">()</span>
|
|
<span class="k">except</span> <span class="ne">BrokenPipeError</span><span class="p">:</span>
|
|
<span class="k">pass</span></div>
|
|
</pre></div>
|
|
|
|
</div>
|
|
|
|
</div>
|
|
<footer>
|
|
|
|
|
|
<hr/>
|
|
|
|
<div role="contentinfo">
|
|
<p>
|
|
© Copyright 2018, Sean Whalen
|
|
|
|
</p>
|
|
</div>
|
|
Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
|
|
|
|
</footer>
|
|
|
|
</div>
|
|
</div>
|
|
|
|
</section>
|
|
|
|
</div>
|
|
|
|
|
|
|
|
<script type="text/javascript">
|
|
jQuery(function () {
|
|
SphinxRtdTheme.Navigation.enable(true);
|
|
});
|
|
</script>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
</body>
|
|
</html> |