mirror of
https://github.com/domainaware/parsedmarc.git
synced 2026-03-16 19:45:58 +00:00
849 lines
101 KiB
HTML
849 lines
101 KiB
HTML
|
|
|
|
<!DOCTYPE html>
|
|
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
|
|
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
|
|
<head>
|
|
<meta charset="utf-8">
|
|
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
|
|
<title>parsedmarc — parsedmarc 1.0.0 documentation</title>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
|
|
|
|
|
|
|
|
|
|
|
|
<link rel="index" title="Index"
|
|
href="../genindex.html"/>
|
|
<link rel="search" title="Search" href="../search.html"/>
|
|
<link rel="top" title="parsedmarc 1.0.0 documentation" href="../index.html"/>
|
|
<link rel="up" title="Module code" href="index.html"/>
|
|
|
|
|
|
<script src="../_static/js/modernizr.min.js"></script>
|
|
|
|
</head>
|
|
|
|
<body class="wy-body-for-nav" role="document">
|
|
|
|
|
|
<div class="wy-grid-for-nav">
|
|
|
|
|
|
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
|
|
<div class="wy-side-scroll">
|
|
<div class="wy-side-nav-search">
|
|
|
|
|
|
|
|
<a href="../index.html" class="icon icon-home"> parsedmarc
|
|
|
|
|
|
|
|
</a>
|
|
|
|
|
|
|
|
|
|
<div class="version">
|
|
1.0.0
|
|
</div>
|
|
|
|
|
|
|
|
|
|
<div role="search">
|
|
<form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
|
|
<input type="text" name="q" placeholder="Search docs" />
|
|
<input type="hidden" name="check_keywords" value="yes" />
|
|
<input type="hidden" name="area" value="default" />
|
|
</form>
|
|
</div>
|
|
|
|
|
|
</div>
|
|
|
|
<div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<!-- Local TOC -->
|
|
<div class="local-toc"></div>
|
|
|
|
|
|
</div>
|
|
</div>
|
|
</nav>
|
|
|
|
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
|
|
|
|
|
|
<nav class="wy-nav-top" role="navigation" aria-label="top navigation">
|
|
|
|
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
|
|
<a href="../index.html">parsedmarc</a>
|
|
|
|
</nav>
|
|
|
|
|
|
|
|
<div class="wy-nav-content">
|
|
<div class="rst-content">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<div role="navigation" aria-label="breadcrumbs navigation">
|
|
|
|
<ul class="wy-breadcrumbs">
|
|
|
|
<li><a href="../index.html">Docs</a> »</li>
|
|
|
|
<li><a href="index.html">Module code</a> »</li>
|
|
|
|
<li>parsedmarc</li>
|
|
|
|
|
|
<li class="wy-breadcrumbs-aside">
|
|
|
|
|
|
|
|
</li>
|
|
|
|
</ul>
|
|
|
|
|
|
<hr/>
|
|
</div>
|
|
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
|
|
<div itemprop="articleBody">
|
|
|
|
<h1>Source code for parsedmarc</h1><div class="highlight"><pre>
|
|
<span></span><span class="ch">#!/usr/bin/env python</span>
|
|
<span class="c1"># -*- coding: utf-8 -*-</span>
|
|
|
|
<span class="sd">"""A Python module and CLI for parsing aggregate DMARC reports"""</span>
|
|
|
|
<span class="kn">from</span> <span class="nn">__future__</span> <span class="k">import</span> <span class="n">unicode_literals</span><span class="p">,</span> <span class="n">print_function</span><span class="p">,</span> <span class="n">absolute_import</span>
|
|
|
|
<span class="kn">import</span> <span class="nn">logging</span>
|
|
<span class="kn">from</span> <span class="nn">sys</span> <span class="k">import</span> <span class="n">version_info</span>
|
|
<span class="kn">from</span> <span class="nn">os</span> <span class="k">import</span> <span class="n">path</span><span class="p">,</span> <span class="n">stat</span>
|
|
<span class="kn">import</span> <span class="nn">json</span>
|
|
<span class="kn">from</span> <span class="nn">datetime</span> <span class="k">import</span> <span class="n">datetime</span>
|
|
<span class="kn">from</span> <span class="nn">collections</span> <span class="k">import</span> <span class="n">OrderedDict</span>
|
|
<span class="kn">from</span> <span class="nn">datetime</span> <span class="k">import</span> <span class="n">timedelta</span>
|
|
<span class="kn">from</span> <span class="nn">io</span> <span class="k">import</span> <span class="n">BytesIO</span><span class="p">,</span> <span class="n">StringIO</span>
|
|
<span class="kn">from</span> <span class="nn">gzip</span> <span class="k">import</span> <span class="n">GzipFile</span>
|
|
<span class="kn">import</span> <span class="nn">tarfile</span>
|
|
<span class="kn">from</span> <span class="nn">zipfile</span> <span class="k">import</span> <span class="n">ZipFile</span>
|
|
<span class="kn">from</span> <span class="nn">csv</span> <span class="k">import</span> <span class="n">DictWriter</span>
|
|
<span class="kn">import</span> <span class="nn">shutil</span>
|
|
<span class="kn">from</span> <span class="nn">argparse</span> <span class="k">import</span> <span class="n">ArgumentParser</span>
|
|
<span class="kn">from</span> <span class="nn">glob</span> <span class="k">import</span> <span class="n">glob</span>
|
|
|
|
<span class="kn">import</span> <span class="nn">publicsuffix</span>
|
|
<span class="kn">import</span> <span class="nn">xmltodict</span>
|
|
<span class="kn">import</span> <span class="nn">dns.reversename</span>
|
|
<span class="kn">import</span> <span class="nn">dns.resolver</span>
|
|
<span class="kn">import</span> <span class="nn">dns.exception</span>
|
|
<span class="kn">from</span> <span class="nn">requests</span> <span class="k">import</span> <span class="n">get</span>
|
|
<span class="kn">import</span> <span class="nn">geoip2.database</span>
|
|
<span class="kn">import</span> <span class="nn">geoip2.errors</span>
|
|
|
|
<span class="n">__version__</span> <span class="o">=</span> <span class="s2">"1.0.0"</span>
|
|
|
|
<span class="n">logger</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="vm">__name__</span><span class="p">)</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">setLevel</span><span class="p">(</span><span class="n">logging</span><span class="o">.</span><span class="n">WARNING</span><span class="p">)</span>
|
|
|
|
|
|
<span class="c1"># Python 2 comparability hack</span>
|
|
<span class="k">if</span> <span class="n">version_info</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">>=</span> <span class="mi">3</span><span class="p">:</span>
|
|
<span class="n">unicode</span> <span class="o">=</span> <span class="nb">str</span>
|
|
|
|
|
|
<div class="viewcode-block" id="InvalidAggregateReport"><a class="viewcode-back" href="../index.html#parsedmarc.InvalidAggregateReport">[docs]</a><span class="k">class</span> <span class="nc">InvalidAggregateReport</span><span class="p">(</span><span class="ne">Exception</span><span class="p">):</span>
|
|
<span class="sd">"""Raised when an invalid DMARC aggregate report is encountered"""</span></div>
|
|
|
|
|
|
<span class="k">def</span> <span class="nf">_get_base_domain</span><span class="p">(</span><span class="n">domain</span><span class="p">):</span>
|
|
<span class="sd">"""</span>
|
|
<span class="sd"> Gets the base domain name for the given domain</span>
|
|
|
|
<span class="sd"> .. note::</span>
|
|
<span class="sd"> Results are based on a list of public domain suffixes at</span>
|
|
<span class="sd"> https://publicsuffix.org/list/public_suffix_list.dat.</span>
|
|
|
|
<span class="sd"> This file is saved to the current working directory,</span>
|
|
<span class="sd"> where it is used as a cache file for 24 hours.</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> domain (str): A domain or subdomain</span>
|
|
|
|
<span class="sd"> Returns:</span>
|
|
<span class="sd"> str: The base domain of the given domain</span>
|
|
|
|
<span class="sd"> """</span>
|
|
<span class="n">psl_path</span> <span class="o">=</span> <span class="s2">"public_suffix_list.dat"</span>
|
|
|
|
<span class="k">def</span> <span class="nf">download_psl</span><span class="p">():</span>
|
|
<span class="n">fresh_psl</span> <span class="o">=</span> <span class="n">publicsuffix</span><span class="o">.</span><span class="n">fetch</span><span class="p">()</span>
|
|
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">psl_path</span><span class="p">,</span> <span class="s2">"w"</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s2">"utf-8"</span><span class="p">)</span> <span class="k">as</span> <span class="n">fresh_psl_file</span><span class="p">:</span>
|
|
<span class="n">fresh_psl_file</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">fresh_psl</span><span class="o">.</span><span class="n">read</span><span class="p">())</span>
|
|
|
|
<span class="k">return</span> <span class="n">publicsuffix</span><span class="o">.</span><span class="n">PublicSuffixList</span><span class="p">(</span><span class="n">fresh_psl</span><span class="p">)</span>
|
|
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">psl_path</span><span class="p">):</span>
|
|
<span class="n">psl</span> <span class="o">=</span> <span class="n">download_psl</span><span class="p">()</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">psl_age</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span> <span class="o">-</span> <span class="n">datetime</span><span class="o">.</span><span class="n">fromtimestamp</span><span class="p">(</span>
|
|
<span class="n">stat</span><span class="p">(</span><span class="n">psl_path</span><span class="p">)</span><span class="o">.</span><span class="n">st_mtime</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">psl_age</span> <span class="o">></span> <span class="n">timedelta</span><span class="p">(</span><span class="n">hours</span><span class="o">=</span><span class="mi">24</span><span class="p">):</span>
|
|
<span class="n">psl</span> <span class="o">=</span> <span class="n">download_psl</span><span class="p">()</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">psl_path</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s2">"utf-8"</span><span class="p">)</span> <span class="k">as</span> <span class="n">psl_file</span><span class="p">:</span>
|
|
<span class="n">psl</span> <span class="o">=</span> <span class="n">publicsuffix</span><span class="o">.</span><span class="n">PublicSuffixList</span><span class="p">(</span><span class="n">psl_file</span><span class="p">)</span>
|
|
|
|
<span class="k">return</span> <span class="n">psl</span><span class="o">.</span><span class="n">get_public_suffix</span><span class="p">(</span><span class="n">domain</span><span class="p">)</span>
|
|
|
|
|
|
<span class="k">def</span> <span class="nf">_query_dns</span><span class="p">(</span><span class="n">domain</span><span class="p">,</span> <span class="n">record_type</span><span class="p">,</span> <span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mf">6.0</span><span class="p">):</span>
|
|
<span class="sd">"""</span>
|
|
<span class="sd"> Queries DNS</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> domain (str): The domain or subdomain to query about</span>
|
|
<span class="sd"> record_type (str): The record type to query for</span>
|
|
<span class="sd"> nameservers (list): A list of one or more nameservers to use</span>
|
|
<span class="sd"> timeout (float): Sets the DNS timeout in seconds</span>
|
|
|
|
<span class="sd"> Returns:</span>
|
|
<span class="sd"> list: A list of answers</span>
|
|
<span class="sd"> """</span>
|
|
<span class="n">resolver</span> <span class="o">=</span> <span class="n">dns</span><span class="o">.</span><span class="n">resolver</span><span class="o">.</span><span class="n">Resolver</span><span class="p">()</span>
|
|
<span class="n">timeout</span> <span class="o">=</span> <span class="nb">float</span><span class="p">(</span><span class="n">timeout</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">nameservers</span><span class="p">:</span>
|
|
<span class="n">resolver</span><span class="o">.</span><span class="n">nameservers</span> <span class="o">=</span> <span class="n">nameservers</span>
|
|
<span class="n">resolver</span><span class="o">.</span><span class="n">timeout</span> <span class="o">=</span> <span class="n">timeout</span>
|
|
<span class="n">resolver</span><span class="o">.</span><span class="n">lifetime</span> <span class="o">=</span> <span class="n">timeout</span>
|
|
<span class="k">return</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span>
|
|
<span class="k">lambda</span> <span class="n">r</span><span class="p">:</span> <span class="n">r</span><span class="o">.</span><span class="n">to_text</span><span class="p">()</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">' "'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'"'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s2">"."</span><span class="p">),</span>
|
|
<span class="n">resolver</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">domain</span><span class="p">,</span> <span class="n">record_type</span><span class="p">,</span> <span class="n">tcp</span><span class="o">=</span><span class="kc">True</span><span class="p">)))</span>
|
|
|
|
|
|
<span class="k">def</span> <span class="nf">_get_reverse_dns</span><span class="p">(</span><span class="n">ip_address</span><span class="p">,</span> <span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mf">6.0</span><span class="p">):</span>
|
|
<span class="sd">"""</span>
|
|
<span class="sd"> Resolves an IP address to a hostname using a reverse DNS query</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> ip_address (str): The IP address to resolve</span>
|
|
<span class="sd"> nameservers (list): A list of nameservers to query</span>
|
|
<span class="sd"> timeout (float): Sets the DNS query timeout in seconds</span>
|
|
|
|
<span class="sd"> Returns:</span>
|
|
|
|
<span class="sd"> """</span>
|
|
<span class="n">hostname</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">address</span> <span class="o">=</span> <span class="n">dns</span><span class="o">.</span><span class="n">reversename</span><span class="o">.</span><span class="n">from_address</span><span class="p">(</span><span class="n">ip_address</span><span class="p">)</span>
|
|
<span class="n">hostname</span> <span class="o">=</span> <span class="n">_query_dns</span><span class="p">(</span><span class="n">address</span><span class="p">,</span> <span class="s2">"PTR"</span><span class="p">,</span>
|
|
<span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
|
|
<span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
|
|
|
|
<span class="k">except</span> <span class="n">dns</span><span class="o">.</span><span class="n">exception</span><span class="o">.</span><span class="n">DNSException</span><span class="p">:</span>
|
|
<span class="k">pass</span>
|
|
|
|
<span class="k">return</span> <span class="n">hostname</span>
|
|
|
|
|
|
<span class="k">def</span> <span class="nf">_timestamp_to_datetime</span><span class="p">(</span><span class="n">timestamp</span><span class="p">):</span>
|
|
<span class="sd">"""</span>
|
|
<span class="sd"> Converts a UNIX/DMARC timestamp to a Python ``DateTime`` object</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> timestamp: The timestamp</span>
|
|
|
|
<span class="sd"> Returns:</span>
|
|
<span class="sd"> DateTime: The converted timestamp as a Python ``DateTime`` object</span>
|
|
<span class="sd"> """</span>
|
|
<span class="k">return</span> <span class="n">datetime</span><span class="o">.</span><span class="n">fromtimestamp</span><span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="n">timestamp</span><span class="p">))</span>
|
|
|
|
|
|
<span class="k">def</span> <span class="nf">_timestamp_to_human</span><span class="p">(</span><span class="n">timestamp</span><span class="p">):</span>
|
|
<span class="sd">"""</span>
|
|
<span class="sd"> Converts a UNIX/DMARC timestamp to a human-readable string</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> timestamp: The timestamp</span>
|
|
|
|
<span class="sd"> Returns:</span>
|
|
<span class="sd"> str: The converted timestamp in ``YYYY-MM-DD HH:MM:SS`` format</span>
|
|
<span class="sd"> """</span>
|
|
<span class="k">return</span> <span class="n">_timestamp_to_datetime</span><span class="p">(</span><span class="n">timestamp</span><span class="p">)</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%S"</span><span class="p">)</span>
|
|
|
|
|
|
<span class="k">def</span> <span class="nf">_human_timestamp_to_datetime</span><span class="p">(</span><span class="n">human_timestamp</span><span class="p">):</span>
|
|
<span class="sd">"""</span>
|
|
<span class="sd"> Converts a human-readable timestamp into a Python ``DateTime`` object</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> human_timestamp (str): A timestamp in `YYYY-MM-DD HH:MM:SS`` format</span>
|
|
|
|
<span class="sd"> Returns:</span>
|
|
<span class="sd"> DateTime: The converted timestamp</span>
|
|
<span class="sd"> """</span>
|
|
<span class="k">return</span> <span class="n">datetime</span><span class="o">.</span><span class="n">strptime</span><span class="p">(</span><span class="n">human_timestamp</span><span class="p">,</span> <span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%S"</span><span class="p">)</span>
|
|
|
|
|
|
<span class="k">def</span> <span class="nf">_get_ip_address_country</span><span class="p">(</span><span class="n">ip_address</span><span class="p">):</span>
|
|
<span class="sd">"""</span>
|
|
<span class="sd"> Uses the MaxMind Geolite2 Country database to return the ISO code for the</span>
|
|
<span class="sd"> country associated with the given IPv4 or IPv6 address</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> ip_address (str): The IP address to query for</span>
|
|
|
|
<span class="sd"> Returns:</span>
|
|
<span class="sd"> str: And ISO country code associated with the given IP address</span>
|
|
<span class="sd"> """</span>
|
|
<span class="n">db_filename</span> <span class="o">=</span> <span class="s2">"GeoLite2-Country.mmdb"</span>
|
|
|
|
<span class="k">def</span> <span class="nf">download_country_database</span><span class="p">():</span>
|
|
<span class="sd">"""Downloads the MaxMind Geolite2 Country database to the current</span>
|
|
<span class="sd"> working directory"""</span>
|
|
<span class="n">url</span> <span class="o">=</span> <span class="s2">"https://geolite.maxmind.com/download/geoip/database/"</span> \
|
|
<span class="s2">"GeoLite2-Country.tar.gz"</span>
|
|
<span class="n">tar_file</span> <span class="o">=</span> <span class="n">tarfile</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">fileobj</span><span class="o">=</span><span class="n">BytesIO</span><span class="p">(</span><span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="p">)</span><span class="o">.</span><span class="n">content</span><span class="p">),</span> <span class="n">mode</span><span class="o">=</span><span class="s2">"r:gz"</span><span class="p">)</span>
|
|
<span class="n">tar_dir</span> <span class="o">=</span> <span class="n">tar_file</span><span class="o">.</span><span class="n">getnames</span><span class="p">()[</span><span class="mi">0</span><span class="p">]</span>
|
|
<span class="n">tar_path</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">/</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">tar_dir</span><span class="p">,</span> <span class="n">db_filename</span><span class="p">)</span>
|
|
<span class="n">tar_file</span><span class="o">.</span><span class="n">extract</span><span class="p">(</span><span class="n">tar_path</span><span class="p">)</span>
|
|
<span class="n">shutil</span><span class="o">.</span><span class="n">move</span><span class="p">(</span><span class="n">tar_path</span><span class="p">,</span> <span class="s2">"."</span><span class="p">)</span>
|
|
<span class="n">shutil</span><span class="o">.</span><span class="n">rmtree</span><span class="p">(</span><span class="n">tar_dir</span><span class="p">)</span>
|
|
|
|
<span class="n">system_paths</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"/usr/local/share/GeoIP/GeoLite2-Country.mmdb"</span><span class="p">,</span>
|
|
<span class="s2">"/usr/share/GeoIP/GeoLite2-Country.mmdb"</span><span class="p">]</span>
|
|
<span class="n">db_path</span> <span class="o">=</span> <span class="s2">""</span>
|
|
|
|
<span class="k">for</span> <span class="n">system_path</span> <span class="ow">in</span> <span class="n">system_paths</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">system_path</span><span class="p">):</span>
|
|
<span class="n">db_path</span> <span class="o">=</span> <span class="n">system_path</span>
|
|
<span class="k">break</span>
|
|
|
|
<span class="k">if</span> <span class="n">db_path</span> <span class="o">==</span> <span class="s2">""</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">db_filename</span><span class="p">):</span>
|
|
<span class="n">download_country_database</span><span class="p">()</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">db_age</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span> <span class="o">-</span> <span class="n">datetime</span><span class="o">.</span><span class="n">fromtimestamp</span><span class="p">(</span>
|
|
<span class="n">stat</span><span class="p">(</span><span class="n">db_filename</span><span class="p">)</span><span class="o">.</span><span class="n">st_mtime</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">db_age</span> <span class="o">></span> <span class="n">timedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="mi">60</span><span class="p">):</span>
|
|
<span class="n">shutil</span><span class="o">.</span><span class="n">rmtree</span><span class="p">(</span><span class="n">db_path</span><span class="p">)</span>
|
|
<span class="n">download_country_database</span><span class="p">()</span>
|
|
<span class="n">db_path</span> <span class="o">=</span> <span class="n">db_filename</span>
|
|
|
|
<span class="n">db_reader</span> <span class="o">=</span> <span class="n">geoip2</span><span class="o">.</span><span class="n">database</span><span class="o">.</span><span class="n">Reader</span><span class="p">(</span><span class="n">db_path</span><span class="p">)</span>
|
|
|
|
<span class="n">country</span> <span class="o">=</span> <span class="kc">None</span>
|
|
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">country</span> <span class="o">=</span> <span class="n">db_reader</span><span class="o">.</span><span class="n">country</span><span class="p">(</span><span class="n">ip_address</span><span class="p">)</span><span class="o">.</span><span class="n">country</span><span class="o">.</span><span class="n">iso_code</span>
|
|
<span class="k">except</span> <span class="n">geoip2</span><span class="o">.</span><span class="n">errors</span><span class="o">.</span><span class="n">AddressNotFoundError</span><span class="p">:</span>
|
|
<span class="k">pass</span>
|
|
|
|
<span class="k">return</span> <span class="n">country</span>
|
|
|
|
|
|
<span class="k">def</span> <span class="nf">_parse_report_record</span><span class="p">(</span><span class="n">record</span><span class="p">,</span> <span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mf">6.0</span><span class="p">):</span>
|
|
<span class="sd">"""</span>
|
|
<span class="sd"> Converts a record from a DMARC aggregate report into a more consistent</span>
|
|
<span class="sd"> format</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> record (OrderedDict): The record to convert</span>
|
|
<span class="sd"> nameservers (list): A list of one or more nameservers to use</span>
|
|
<span class="sd"> timeout (float): Sets the DNS timeout in seconds</span>
|
|
|
|
<span class="sd"> Returns:</span>
|
|
<span class="sd"> OrderedDict: The converted record</span>
|
|
<span class="sd"> """</span>
|
|
<span class="n">record</span> <span class="o">=</span> <span class="n">record</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
|
<span class="n">new_record</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">()</span>
|
|
<span class="n">new_record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">]</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">()</span>
|
|
<span class="n">new_record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"ip_address"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"row"</span><span class="p">][</span><span class="s2">"source_ip"</span><span class="p">]</span>
|
|
<span class="n">reverse_dns</span> <span class="o">=</span> <span class="n">_get_reverse_dns</span><span class="p">(</span><span class="n">new_record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"ip_address"</span><span class="p">],</span>
|
|
<span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
|
|
<span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">)</span>
|
|
<span class="n">country</span> <span class="o">=</span> <span class="n">_get_ip_address_country</span><span class="p">(</span><span class="n">new_record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"ip_address"</span><span class="p">])</span>
|
|
<span class="n">new_record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"country"</span><span class="p">]</span> <span class="o">=</span> <span class="n">country</span>
|
|
<span class="n">new_record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"reverse_dns"</span><span class="p">]</span> <span class="o">=</span> <span class="n">reverse_dns</span>
|
|
<span class="n">new_record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"base_domain"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="k">if</span> <span class="n">new_record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"reverse_dns"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">base_domain</span> <span class="o">=</span> <span class="n">_get_base_domain</span><span class="p">(</span><span class="n">new_record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"reverse_dns"</span><span class="p">])</span>
|
|
<span class="n">new_record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"base_domain"</span><span class="p">]</span> <span class="o">=</span> <span class="n">base_domain</span>
|
|
<span class="n">new_record</span><span class="p">[</span><span class="s2">"count"</span><span class="p">]</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">record</span><span class="p">[</span><span class="s2">"row"</span><span class="p">][</span><span class="s2">"count"</span><span class="p">])</span>
|
|
<span class="n">policy_evaluated</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"row"</span><span class="p">][</span><span class="s2">"policy_evaluated"</span><span class="p">]</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
|
<span class="n">new_policy_evaluated</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">([(</span><span class="s2">"disposition"</span><span class="p">,</span> <span class="s2">"none"</span><span class="p">),</span>
|
|
<span class="p">(</span><span class="s2">"dkim"</span><span class="p">,</span> <span class="s2">"fail"</span><span class="p">),</span>
|
|
<span class="p">(</span><span class="s2">"spf"</span><span class="p">,</span> <span class="s2">"fail"</span><span class="p">),</span>
|
|
<span class="p">(</span><span class="s2">"policy_override_reasons"</span><span class="p">,</span> <span class="p">[])</span>
|
|
<span class="p">])</span>
|
|
<span class="k">if</span> <span class="s2">"disposition"</span> <span class="ow">in</span> <span class="n">policy_evaluated</span><span class="p">:</span>
|
|
<span class="n">new_policy_evaluated</span><span class="p">[</span><span class="s2">"disposition"</span><span class="p">]</span> <span class="o">=</span> <span class="n">policy_evaluated</span><span class="p">[</span><span class="s2">"disposition"</span><span class="p">]</span>
|
|
<span class="k">if</span> <span class="s2">"dkim"</span> <span class="ow">in</span> <span class="n">policy_evaluated</span><span class="p">:</span>
|
|
<span class="n">new_policy_evaluated</span><span class="p">[</span><span class="s2">"dkim"</span><span class="p">]</span> <span class="o">=</span> <span class="n">policy_evaluated</span><span class="p">[</span><span class="s2">"dkim"</span><span class="p">]</span>
|
|
<span class="k">if</span> <span class="s2">"spf"</span> <span class="ow">in</span> <span class="n">policy_evaluated</span><span class="p">:</span>
|
|
<span class="n">new_policy_evaluated</span><span class="p">[</span><span class="s2">"spf"</span><span class="p">]</span> <span class="o">=</span> <span class="n">policy_evaluated</span><span class="p">[</span><span class="s2">"spf"</span><span class="p">]</span>
|
|
<span class="n">reasons</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="k">if</span> <span class="s2">"reason"</span> <span class="ow">in</span> <span class="n">policy_evaluated</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">policy_evaluated</span><span class="p">[</span><span class="s2">"reason"</span><span class="p">])</span> <span class="o">==</span> <span class="nb">list</span><span class="p">:</span>
|
|
<span class="n">reasons</span> <span class="o">=</span> <span class="n">policy_evaluated</span><span class="p">[</span><span class="s2">"reason"</span><span class="p">]</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">reasons</span> <span class="o">=</span> <span class="p">[</span><span class="n">policy_evaluated</span><span class="p">[</span><span class="s2">"reason"</span><span class="p">]]</span>
|
|
<span class="k">for</span> <span class="n">reason</span> <span class="ow">in</span> <span class="n">reasons</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="s2">"comment"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">reason</span><span class="p">:</span>
|
|
<span class="n">reason</span><span class="p">[</span><span class="s2">"comment"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"none"</span>
|
|
<span class="n">reasons</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">reason</span><span class="p">)</span>
|
|
<span class="n">new_policy_evaluated</span><span class="p">[</span><span class="s2">"policy_override_reasons"</span><span class="p">]</span> <span class="o">=</span> <span class="n">reasons</span>
|
|
<span class="n">new_record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">]</span> <span class="o">=</span> <span class="n">new_policy_evaluated</span>
|
|
<span class="n">new_record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">]</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
|
<span class="n">new_record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">]</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">([(</span><span class="s2">"dkim"</span><span class="p">,</span> <span class="p">[]),</span> <span class="p">(</span><span class="s2">"spf"</span><span class="p">,</span> <span class="p">[])])</span>
|
|
<span class="n">auth_results</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">]</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
|
<span class="k">if</span> <span class="s2">"dkim"</span> <span class="ow">in</span> <span class="n">auth_results</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">auth_results</span><span class="p">[</span><span class="s2">"dkim"</span><span class="p">])</span> <span class="o">!=</span> <span class="nb">list</span><span class="p">:</span>
|
|
<span class="n">auth_results</span><span class="p">[</span><span class="s2">"dkim"</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="n">auth_results</span><span class="p">[</span><span class="s2">"dkim"</span><span class="p">]]</span>
|
|
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">auth_results</span><span class="p">[</span><span class="s2">"dkim"</span><span class="p">]:</span>
|
|
<span class="k">if</span> <span class="s2">"domain"</span> <span class="ow">in</span> <span class="n">result</span> <span class="ow">and</span> <span class="n">result</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">new_result</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">([(</span><span class="s2">"domain"</span><span class="p">,</span> <span class="n">result</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">])])</span>
|
|
<span class="k">if</span> <span class="s2">"selector"</span> <span class="ow">in</span> <span class="n">result</span> <span class="ow">and</span> <span class="n">result</span><span class="p">[</span><span class="s2">"selector"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">new_result</span><span class="p">[</span><span class="s2">"selector"</span><span class="p">]</span> <span class="o">=</span> <span class="n">result</span><span class="p">[</span><span class="s2">"selector"</span><span class="p">]</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">new_result</span><span class="p">[</span><span class="s2">"selector"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"none"</span>
|
|
<span class="k">if</span> <span class="s2">"result"</span> <span class="ow">in</span> <span class="n">result</span> <span class="ow">and</span> <span class="n">result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">new_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">]</span> <span class="o">=</span> <span class="n">result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">]</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">new_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"none"</span>
|
|
<span class="n">new_record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">][</span><span class="s2">"dkim"</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">new_result</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">auth_results</span><span class="p">[</span><span class="s2">"spf"</span><span class="p">])</span> <span class="o">!=</span> <span class="nb">list</span><span class="p">:</span>
|
|
<span class="n">auth_results</span><span class="p">[</span><span class="s2">"spf"</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="n">auth_results</span><span class="p">[</span><span class="s2">"spf"</span><span class="p">]]</span>
|
|
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">auth_results</span><span class="p">[</span><span class="s2">"spf"</span><span class="p">]:</span>
|
|
<span class="n">new_result</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">([(</span><span class="s2">"domain"</span><span class="p">,</span> <span class="n">result</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">])])</span>
|
|
<span class="k">if</span> <span class="s2">"scope"</span> <span class="ow">in</span> <span class="n">result</span> <span class="ow">and</span> <span class="n">result</span><span class="p">[</span><span class="s2">"scope"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">new_result</span><span class="p">[</span><span class="s2">"scope"</span><span class="p">]</span> <span class="o">=</span> <span class="n">result</span><span class="p">[</span><span class="s2">"scope"</span><span class="p">]</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">new_result</span><span class="p">[</span><span class="s2">"scope"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"mfrom"</span>
|
|
<span class="k">if</span> <span class="s2">"result"</span> <span class="ow">in</span> <span class="n">result</span> <span class="ow">and</span> <span class="n">result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">new_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">]</span> <span class="o">=</span> <span class="n">result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">]</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">new_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"none"</span>
|
|
<span class="n">new_record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">new_result</span><span class="p">)</span>
|
|
|
|
<span class="k">if</span> <span class="s2">"envelope_from"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">new_record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">]:</span>
|
|
<span class="n">envelope_from</span> <span class="o">=</span> <span class="n">new_record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">][</span><span class="o">-</span><span class="mi">1</span><span class="p">][</span><span class="s2">"domain"</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
|
|
<span class="n">new_record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"envelope_from"</span><span class="p">]</span> <span class="o">=</span> <span class="n">envelope_from</span>
|
|
|
|
<span class="k">elif</span> <span class="n">new_record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"envelope_from"</span><span class="p">]</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">envelope_from</span> <span class="o">=</span> <span class="n">new_record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">][</span><span class="o">-</span><span class="mi">1</span><span class="p">][</span><span class="s2">"domain"</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
|
|
<span class="n">new_record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"envelope_from"</span><span class="p">]</span> <span class="o">=</span> <span class="n">envelope_from</span>
|
|
|
|
<span class="n">envelope_to</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="k">if</span> <span class="s2">"envelope_to"</span> <span class="ow">in</span> <span class="n">new_record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">]:</span>
|
|
<span class="n">envelope_to</span> <span class="o">=</span> <span class="n">new_record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"envelope_to"</span><span class="p">]</span>
|
|
<span class="k">del</span> <span class="n">new_record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"envelope_to"</span><span class="p">]</span>
|
|
|
|
<span class="n">new_record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"envelope_to"</span><span class="p">]</span> <span class="o">=</span> <span class="n">envelope_to</span>
|
|
|
|
<span class="k">return</span> <span class="n">new_record</span>
|
|
|
|
|
|
<div class="viewcode-block" id="parse_aggregate_report_xml"><a class="viewcode-back" href="../index.html#parsedmarc.parse_aggregate_report_xml">[docs]</a><span class="k">def</span> <span class="nf">parse_aggregate_report_xml</span><span class="p">(</span><span class="n">xml</span><span class="p">,</span> <span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mf">6.0</span><span class="p">):</span>
|
|
<span class="sd">"""Parses a DMARC XML report string and returns a consistent OrderedDict</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> xml (str): A string of DMARC aggregate report XML</span>
|
|
<span class="sd"> nameservers (list): A list of one or more nameservers to use</span>
|
|
<span class="sd"> timeout (float): Sets the DNS timeout in seconds</span>
|
|
|
|
<span class="sd"> Returns:</span>
|
|
<span class="sd"> OrderedDict: The parsed aggregate DMARC report</span>
|
|
<span class="sd"> """</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">report</span> <span class="o">=</span> <span class="n">xmltodict</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">xml</span><span class="p">)[</span><span class="s2">"feedback"</span><span class="p">]</span>
|
|
<span class="n">report_metadata</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">]</span>
|
|
<span class="n">schema</span> <span class="o">=</span> <span class="s2">"draft"</span>
|
|
<span class="k">if</span> <span class="s2">"version"</span> <span class="ow">in</span> <span class="n">report</span><span class="p">:</span>
|
|
<span class="n">schema</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"version"</span><span class="p">]</span>
|
|
<span class="n">new_report</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">([(</span><span class="s2">"xml_schema"</span><span class="p">,</span> <span class="n">schema</span><span class="p">)])</span>
|
|
<span class="n">new_report_metadata</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">()</span>
|
|
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"org_name"</span><span class="p">]</span> <span class="o">=</span> <span class="n">report_metadata</span><span class="p">[</span><span class="s2">"org_name"</span><span class="p">]</span>
|
|
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"org_email"</span><span class="p">]</span> <span class="o">=</span> <span class="n">report_metadata</span><span class="p">[</span><span class="s2">"email"</span><span class="p">]</span>
|
|
<span class="n">extra</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="k">if</span> <span class="s2">"extra_contact_info"</span> <span class="ow">in</span> <span class="n">report_metadata</span><span class="p">:</span>
|
|
<span class="n">extra</span> <span class="o">=</span> <span class="n">report_metadata</span><span class="p">[</span><span class="s2">"extra_contact_info"</span><span class="p">]</span>
|
|
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"org_extra_contact_info"</span><span class="p">]</span> <span class="o">=</span> <span class="n">extra</span>
|
|
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"report_id"</span><span class="p">]</span> <span class="o">=</span> <span class="n">report_metadata</span><span class="p">[</span><span class="s2">"report_id"</span><span class="p">]</span>
|
|
<span class="n">date_range</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">][</span><span class="s2">"date_range"</span><span class="p">]</span>
|
|
<span class="n">date_range</span><span class="p">[</span><span class="s2">"begin"</span><span class="p">]</span> <span class="o">=</span> <span class="n">_timestamp_to_human</span><span class="p">(</span><span class="n">date_range</span><span class="p">[</span><span class="s2">"begin"</span><span class="p">])</span>
|
|
<span class="n">date_range</span><span class="p">[</span><span class="s2">"end"</span><span class="p">]</span> <span class="o">=</span> <span class="n">_timestamp_to_human</span><span class="p">(</span><span class="n">date_range</span><span class="p">[</span><span class="s2">"end"</span><span class="p">])</span>
|
|
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">]</span> <span class="o">=</span> <span class="n">date_range</span><span class="p">[</span><span class="s2">"begin"</span><span class="p">]</span>
|
|
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">]</span> <span class="o">=</span> <span class="n">date_range</span><span class="p">[</span><span class="s2">"end"</span><span class="p">]</span>
|
|
<span class="n">errors</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="k">if</span> <span class="s2">"error"</span> <span class="ow">in</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">]:</span>
|
|
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">][</span><span class="s2">"error"</span><span class="p">])</span> <span class="o">!=</span> <span class="nb">list</span><span class="p">:</span>
|
|
<span class="n">errors</span> <span class="o">=</span> <span class="p">[</span><span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">][</span><span class="s2">"error"</span><span class="p">]]</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">errors</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">][</span><span class="s2">"error"</span><span class="p">]</span>
|
|
<span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">"errors"</span><span class="p">]</span> <span class="o">=</span> <span class="n">errors</span>
|
|
<span class="n">new_report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">]</span> <span class="o">=</span> <span class="n">new_report_metadata</span>
|
|
<span class="n">records</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="n">policy_published</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">]</span>
|
|
<span class="n">new_policy_published</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">()</span>
|
|
<span class="n">new_policy_published</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">]</span> <span class="o">=</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">]</span>
|
|
<span class="n">adkim</span> <span class="o">=</span> <span class="s2">"r"</span>
|
|
<span class="k">if</span> <span class="s2">"adkim"</span> <span class="ow">in</span> <span class="n">policy_published</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"adkim"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">adkim</span> <span class="o">=</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"adkim"</span><span class="p">]</span>
|
|
<span class="n">new_policy_published</span><span class="p">[</span><span class="s2">"adkim"</span><span class="p">]</span> <span class="o">=</span> <span class="n">adkim</span>
|
|
<span class="n">aspf</span> <span class="o">=</span> <span class="s2">"r"</span>
|
|
<span class="k">if</span> <span class="s2">"aspf"</span> <span class="ow">in</span> <span class="n">policy_published</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"aspf"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">aspf</span> <span class="o">=</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"aspf"</span><span class="p">]</span>
|
|
<span class="n">new_policy_published</span><span class="p">[</span><span class="s2">"aspf"</span><span class="p">]</span> <span class="o">=</span> <span class="n">aspf</span>
|
|
<span class="n">new_policy_published</span><span class="p">[</span><span class="s2">"p"</span><span class="p">]</span> <span class="o">=</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"p"</span><span class="p">]</span>
|
|
<span class="n">sp</span> <span class="o">=</span> <span class="n">new_policy_published</span><span class="p">[</span><span class="s2">"p"</span><span class="p">]</span>
|
|
<span class="k">if</span> <span class="s2">"sp"</span> <span class="ow">in</span> <span class="n">policy_published</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"sp"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">sp</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"sp"</span><span class="p">]</span>
|
|
<span class="n">new_policy_published</span><span class="p">[</span><span class="s2">"sp"</span><span class="p">]</span> <span class="o">=</span> <span class="n">sp</span>
|
|
<span class="n">pct</span> <span class="o">=</span> <span class="s2">"100"</span>
|
|
<span class="k">if</span> <span class="s2">"pct"</span> <span class="ow">in</span> <span class="n">policy_published</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"pct"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">pct</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"pct"</span><span class="p">]</span>
|
|
<span class="n">new_policy_published</span><span class="p">[</span><span class="s2">"pct"</span><span class="p">]</span> <span class="o">=</span> <span class="n">pct</span>
|
|
<span class="n">fo</span> <span class="o">=</span> <span class="s2">"0"</span>
|
|
<span class="k">if</span> <span class="s2">"fo"</span> <span class="ow">in</span> <span class="n">policy_published</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="n">policy_published</span><span class="p">[</span><span class="s2">"fo"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">fo</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"fo"</span><span class="p">]</span>
|
|
<span class="n">new_policy_published</span><span class="p">[</span><span class="s2">"fo"</span><span class="p">]</span> <span class="o">=</span> <span class="n">fo</span>
|
|
<span class="n">new_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">]</span> <span class="o">=</span> <span class="n">new_policy_published</span>
|
|
|
|
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">"record"</span><span class="p">])</span> <span class="o">==</span> <span class="nb">list</span><span class="p">:</span>
|
|
<span class="k">for</span> <span class="n">record</span> <span class="ow">in</span> <span class="n">report</span><span class="p">[</span><span class="s2">"record"</span><span class="p">]:</span>
|
|
<span class="n">records</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_parse_report_record</span><span class="p">(</span><span class="n">record</span><span class="p">,</span>
|
|
<span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
|
|
<span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">))</span>
|
|
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">records</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_parse_report_record</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">"record"</span><span class="p">]))</span>
|
|
|
|
<span class="n">new_report</span><span class="p">[</span><span class="s2">"records"</span><span class="p">]</span> <span class="o">=</span> <span class="n">records</span>
|
|
|
|
<span class="k">return</span> <span class="n">new_report</span>
|
|
|
|
<span class="k">except</span> <span class="ne">KeyError</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">InvalidAggregateReport</span><span class="p">(</span><span class="s2">"Missing field: "</span>
|
|
<span class="s2">"</span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
|
|
|
|
|
<div class="viewcode-block" id="parse_aggregate_report_file"><a class="viewcode-back" href="../index.html#parsedmarc.parse_aggregate_report_file">[docs]</a><span class="k">def</span> <span class="nf">parse_aggregate_report_file</span><span class="p">(</span><span class="n">_input</span><span class="p">,</span> <span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mf">6.0</span><span class="p">):</span>
|
|
<span class="sd">"""Parses a file at the given path, a file-like object. or bytes as a</span>
|
|
<span class="sd"> aggregate DMARC report</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> _input: A path to a file, a file like object, or bytes</span>
|
|
<span class="sd"> nameservers (list): A list of one or more nameservers to use</span>
|
|
<span class="sd"> timeout (float): Sets the DNS timeout in seconds</span>
|
|
|
|
<span class="sd"> Returns:</span>
|
|
<span class="sd"> OrderedDict: The parsed DMARC aggregate report</span>
|
|
<span class="sd"> """</span>
|
|
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">_input</span><span class="p">)</span> <span class="o">==</span> <span class="nb">str</span> <span class="ow">or</span> <span class="nb">type</span><span class="p">(</span><span class="n">_input</span><span class="p">)</span> <span class="o">==</span> <span class="n">unicode</span><span class="p">:</span>
|
|
<span class="n">file_object</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="n">_input</span><span class="p">,</span> <span class="s2">"rb"</span><span class="p">)</span>
|
|
<span class="k">elif</span> <span class="nb">type</span><span class="p">(</span><span class="n">_input</span><span class="p">)</span> <span class="o">==</span> <span class="nb">bytes</span><span class="p">:</span>
|
|
<span class="n">file_object</span> <span class="o">=</span> <span class="n">BytesIO</span><span class="p">(</span><span class="n">_input</span><span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">file_object</span> <span class="o">=</span> <span class="n">_input</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">header</span> <span class="o">=</span> <span class="n">file_object</span><span class="o">.</span><span class="n">read</span><span class="p">(</span><span class="mi">6</span><span class="p">)</span>
|
|
<span class="n">file_object</span><span class="o">.</span><span class="n">seek</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">header</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="sa">b</span><span class="s2">"</span><span class="se">\x50\x4B\x03\x04</span><span class="s2">"</span><span class="p">):</span>
|
|
<span class="n">_zip</span> <span class="o">=</span> <span class="n">ZipFile</span><span class="p">(</span><span class="n">file_object</span><span class="p">)</span>
|
|
<span class="n">xml</span> <span class="o">=</span> <span class="n">_zip</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">_zip</span><span class="o">.</span><span class="n">namelist</span><span class="p">()[</span><span class="mi">0</span><span class="p">])</span><span class="o">.</span><span class="n">read</span><span class="p">()</span><span class="o">.</span><span class="n">decode</span><span class="p">()</span>
|
|
<span class="k">elif</span> <span class="n">header</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="sa">b</span><span class="s2">"</span><span class="se">\x1F\x8B</span><span class="s2">"</span><span class="p">):</span>
|
|
<span class="n">xml</span> <span class="o">=</span> <span class="n">GzipFile</span><span class="p">(</span><span class="n">fileobj</span><span class="o">=</span><span class="n">file_object</span><span class="p">)</span><span class="o">.</span><span class="n">read</span><span class="p">()</span><span class="o">.</span><span class="n">decode</span><span class="p">()</span>
|
|
<span class="k">elif</span> <span class="n">header</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="sa">b</span><span class="s2">"</span><span class="se">\x3c\x3f\x78\x6d\x6c\x20</span><span class="s2">"</span><span class="p">):</span>
|
|
<span class="n">xml</span> <span class="o">=</span> <span class="n">file_object</span><span class="o">.</span><span class="n">read</span><span class="p">()</span><span class="o">.</span><span class="n">decode</span><span class="p">()</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">file_object</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
|
|
<span class="k">raise</span> <span class="n">InvalidAggregateReport</span><span class="p">(</span><span class="s2">"Not a valid zip, gzip, or xml file"</span><span class="p">)</span>
|
|
|
|
<span class="n">file_object</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
|
|
<span class="k">except</span> <span class="ne">UnicodeDecodeError</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">InvalidAggregateReport</span><span class="p">(</span><span class="s2">"File objects must be opened in binary "</span>
|
|
<span class="s2">"(rb) mode"</span><span class="p">)</span>
|
|
|
|
<span class="k">return</span> <span class="n">parse_aggregate_report_xml</span><span class="p">(</span><span class="n">xml</span><span class="p">,</span>
|
|
<span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
|
|
<span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">)</span></div>
|
|
|
|
|
|
<div class="viewcode-block" id="parsed_aggregate_report_to_csv"><a class="viewcode-back" href="../index.html#parsedmarc.parsed_aggregate_report_to_csv">[docs]</a><span class="k">def</span> <span class="nf">parsed_aggregate_report_to_csv</span><span class="p">(</span><span class="n">_input</span><span class="p">):</span>
|
|
<span class="sd">"""</span>
|
|
<span class="sd"> Converts one or more parsed aggregate reports to flat CSV format, including</span>
|
|
<span class="sd"> headers</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> _input: A parsed aggregate report or list of parsed aggregate reports</span>
|
|
|
|
<span class="sd"> Returns:</span>
|
|
<span class="sd"> str: Parsed aggregate report data in flat CSV format, including headers</span>
|
|
<span class="sd"> """</span>
|
|
<span class="n">fields</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"xml_schema"</span><span class="p">,</span> <span class="s2">"org_name"</span><span class="p">,</span> <span class="s2">"org_email"</span><span class="p">,</span>
|
|
<span class="s2">"org_extra_contact_info"</span><span class="p">,</span> <span class="s2">"report_id"</span><span class="p">,</span> <span class="s2">"begin_date"</span><span class="p">,</span> <span class="s2">"end_date"</span><span class="p">,</span>
|
|
<span class="s2">"errors"</span><span class="p">,</span> <span class="s2">"domain"</span><span class="p">,</span> <span class="s2">"adkim"</span><span class="p">,</span> <span class="s2">"aspf"</span><span class="p">,</span> <span class="s2">"p"</span><span class="p">,</span> <span class="s2">"sp"</span><span class="p">,</span> <span class="s2">"pct"</span><span class="p">,</span> <span class="s2">"fo"</span><span class="p">,</span>
|
|
<span class="s2">"source_ip_address"</span><span class="p">,</span> <span class="s2">"source_country"</span><span class="p">,</span> <span class="s2">"source_reverse_dns"</span><span class="p">,</span>
|
|
<span class="s2">"source_base_domain"</span><span class="p">,</span> <span class="s2">"count"</span><span class="p">,</span> <span class="s2">"disposition"</span><span class="p">,</span> <span class="s2">"dkim_alignment"</span><span class="p">,</span>
|
|
<span class="s2">"spf_alignment"</span><span class="p">,</span> <span class="s2">"policy_override_reasons"</span><span class="p">,</span>
|
|
<span class="s2">"policy_override_comments"</span><span class="p">,</span> <span class="s2">"envelope_from"</span><span class="p">,</span> <span class="s2">"header_from"</span><span class="p">,</span>
|
|
<span class="s2">"envelope_to"</span><span class="p">,</span> <span class="s2">"dkim_domains"</span><span class="p">,</span> <span class="s2">"dkim_selectors"</span><span class="p">,</span> <span class="s2">"dkim_results"</span><span class="p">,</span>
|
|
<span class="s2">"spf_domains"</span><span class="p">,</span> <span class="s2">"spf_scopes"</span><span class="p">,</span> <span class="s2">"spf_results"</span><span class="p">]</span>
|
|
|
|
<span class="n">csv_file_object</span> <span class="o">=</span> <span class="n">StringIO</span><span class="p">()</span>
|
|
<span class="n">writer</span> <span class="o">=</span> <span class="n">DictWriter</span><span class="p">(</span><span class="n">csv_file_object</span><span class="p">,</span> <span class="n">fields</span><span class="p">)</span>
|
|
<span class="n">writer</span><span class="o">.</span><span class="n">writeheader</span><span class="p">()</span>
|
|
|
|
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">_input</span><span class="p">)</span> <span class="o">==</span> <span class="n">OrderedDict</span><span class="p">:</span>
|
|
<span class="n">_input</span> <span class="o">=</span> <span class="p">[</span><span class="n">_input</span><span class="p">]</span>
|
|
|
|
<span class="k">for</span> <span class="n">report</span> <span class="ow">in</span> <span class="n">_input</span><span class="p">:</span>
|
|
<span class="n">xml_schema</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"xml_schema"</span><span class="p">]</span>
|
|
<span class="n">org_name</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">][</span><span class="s2">"org_name"</span><span class="p">]</span>
|
|
<span class="n">org_email</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">][</span><span class="s2">"org_email"</span><span class="p">]</span>
|
|
<span class="n">org_extra_contact</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">][</span><span class="s2">"org_extra_contact_info"</span><span class="p">]</span>
|
|
<span class="n">report_id</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">][</span><span class="s2">"report_id"</span><span class="p">]</span>
|
|
<span class="n">begin_date</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">][</span><span class="s2">"begin_date"</span><span class="p">]</span>
|
|
<span class="n">end_date</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">][</span><span class="s2">"end_date"</span><span class="p">]</span>
|
|
<span class="n">errors</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">][</span><span class="s2">"errors"</span><span class="p">]</span>
|
|
<span class="n">domain</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"domain"</span><span class="p">]</span>
|
|
<span class="n">adkim</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"adkim"</span><span class="p">]</span>
|
|
<span class="n">aspf</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"aspf"</span><span class="p">]</span>
|
|
<span class="n">p</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"p"</span><span class="p">]</span>
|
|
<span class="n">sp</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"sp"</span><span class="p">]</span>
|
|
<span class="n">pct</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"pct"</span><span class="p">]</span>
|
|
<span class="n">fo</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"fo"</span><span class="p">]</span>
|
|
|
|
<span class="n">report_dict</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="n">xml_schema</span><span class="o">=</span><span class="n">xml_schema</span><span class="p">,</span> <span class="n">org_name</span><span class="o">=</span><span class="n">org_name</span><span class="p">,</span>
|
|
<span class="n">org_email</span><span class="o">=</span><span class="n">org_email</span><span class="p">,</span>
|
|
<span class="n">org_extra_contact_info</span><span class="o">=</span><span class="n">org_extra_contact</span><span class="p">,</span>
|
|
<span class="n">report_id</span><span class="o">=</span><span class="n">report_id</span><span class="p">,</span> <span class="n">begin_date</span><span class="o">=</span><span class="n">begin_date</span><span class="p">,</span>
|
|
<span class="n">end_date</span><span class="o">=</span><span class="n">end_date</span><span class="p">,</span> <span class="n">errors</span><span class="o">=</span><span class="n">errors</span><span class="p">,</span> <span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span>
|
|
<span class="n">adkim</span><span class="o">=</span><span class="n">adkim</span><span class="p">,</span> <span class="n">aspf</span><span class="o">=</span><span class="n">aspf</span><span class="p">,</span> <span class="n">p</span><span class="o">=</span><span class="n">p</span><span class="p">,</span> <span class="n">sp</span><span class="o">=</span><span class="n">sp</span><span class="p">,</span> <span class="n">pct</span><span class="o">=</span><span class="n">pct</span><span class="p">,</span> <span class="n">fo</span><span class="o">=</span><span class="n">fo</span><span class="p">)</span>
|
|
|
|
<span class="k">for</span> <span class="n">record</span> <span class="ow">in</span> <span class="n">report</span><span class="p">[</span><span class="s2">"records"</span><span class="p">]:</span>
|
|
<span class="n">row</span> <span class="o">=</span> <span class="n">report_dict</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"source_ip_address"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"ip_address"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"source_country"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"country"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"source_reverse_dns"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"reverse_dns"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"source_base_domain"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"base_domain"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"count"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"count"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"disposition"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"disposition"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"spf_alignment"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"dkim_alignment"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"dkim"</span><span class="p">]</span>
|
|
<span class="n">policy_override_reasons</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">r</span><span class="p">:</span> <span class="n">r</span><span class="p">[</span><span class="s2">"type"</span><span class="p">],</span>
|
|
<span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">]</span>
|
|
<span class="p">[</span><span class="s2">"policy_override_reasons"</span><span class="p">]))</span>
|
|
<span class="n">policy_override_comments</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">r</span><span class="p">:</span> <span class="n">r</span><span class="p">[</span><span class="s2">"comment"</span><span class="p">],</span>
|
|
<span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">]</span>
|
|
<span class="p">[</span><span class="s2">"policy_override_reasons"</span><span class="p">]))</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"policy_override_reasons"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">","</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
|
|
<span class="n">policy_override_reasons</span><span class="p">)</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"policy_override_comments"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"|"</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
|
|
<span class="n">policy_override_comments</span><span class="p">)</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"envelope_from"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"envelope_from"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"header_from"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"header_from"</span><span class="p">]</span>
|
|
<span class="n">envelope_to</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"envelope_to"</span><span class="p">]</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"envelope_to"</span><span class="p">]</span> <span class="o">=</span> <span class="n">envelope_to</span>
|
|
<span class="n">dkim_domains</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="n">dkim_selectors</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="n">dkim_results</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="k">for</span> <span class="n">dkim_result</span> <span class="ow">in</span> <span class="n">record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">][</span><span class="s2">"dkim"</span><span class="p">]:</span>
|
|
<span class="n">dkim_domains</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">])</span>
|
|
<span class="k">if</span> <span class="s2">"selector"</span> <span class="ow">in</span> <span class="n">dkim_result</span><span class="p">:</span>
|
|
<span class="n">dkim_selectors</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">"selector"</span><span class="p">])</span>
|
|
<span class="n">dkim_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">])</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"dkim_domains"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">","</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">dkim_domains</span><span class="p">)</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"dkim_selectors"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">","</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">dkim_selectors</span><span class="p">)</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"dkim_results"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">","</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">dkim_results</span><span class="p">)</span>
|
|
<span class="n">spf_domains</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="n">spf_scopes</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="n">spf_results</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="k">for</span> <span class="n">spf_result</span> <span class="ow">in</span> <span class="n">record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">]:</span>
|
|
<span class="n">spf_domains</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">])</span>
|
|
<span class="n">spf_scopes</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">"scope"</span><span class="p">])</span>
|
|
<span class="n">spf_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">])</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"spf_domains"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">","</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">spf_domains</span><span class="p">)</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"spf_scopes"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">","</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">spf_scopes</span><span class="p">)</span>
|
|
<span class="n">row</span><span class="p">[</span><span class="s2">"spf_results"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">","</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">spf_results</span><span class="p">)</span>
|
|
|
|
<span class="n">writer</span><span class="o">.</span><span class="n">writerow</span><span class="p">(</span><span class="n">row</span><span class="p">)</span>
|
|
<span class="n">csv_file_object</span><span class="o">.</span><span class="n">flush</span><span class="p">()</span>
|
|
|
|
<span class="k">return</span> <span class="n">csv_file_object</span><span class="o">.</span><span class="n">getvalue</span><span class="p">()</span></div>
|
|
|
|
|
|
<span class="k">def</span> <span class="nf">_main</span><span class="p">():</span>
|
|
<span class="sd">"""Called when the module in executed"""</span>
|
|
<span class="n">arg_parser</span> <span class="o">=</span> <span class="n">ArgumentParser</span><span class="p">(</span><span class="n">description</span><span class="o">=</span><span class="s2">"Parses aggregate DMARC reports"</span><span class="p">)</span>
|
|
<span class="n">arg_parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span><span class="s2">"file_path"</span><span class="p">,</span> <span class="n">nargs</span><span class="o">=</span><span class="s2">"+"</span><span class="p">,</span>
|
|
<span class="n">help</span><span class="o">=</span><span class="s2">"one or more paths of aggregate report "</span>
|
|
<span class="s2">"files (compressed or uncompressed)"</span><span class="p">)</span>
|
|
<span class="n">arg_parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span><span class="s2">"-f"</span><span class="p">,</span> <span class="s2">"--format"</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="s2">"json"</span><span class="p">,</span>
|
|
<span class="n">help</span><span class="o">=</span><span class="s2">"specify JSON or CSV output format"</span><span class="p">)</span>
|
|
<span class="n">arg_parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span><span class="s2">"-o"</span><span class="p">,</span> <span class="s2">"--output"</span><span class="p">,</span>
|
|
<span class="n">help</span><span class="o">=</span><span class="s2">"output to a file path rather than "</span>
|
|
<span class="s2">"printing to the screen"</span><span class="p">)</span>
|
|
<span class="n">arg_parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span><span class="s2">"-n"</span><span class="p">,</span> <span class="s2">"--nameserver"</span><span class="p">,</span> <span class="n">nargs</span><span class="o">=</span><span class="s2">"+"</span><span class="p">,</span>
|
|
<span class="n">help</span><span class="o">=</span><span class="s2">"nameservers to query"</span><span class="p">)</span>
|
|
<span class="n">arg_parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span><span class="s2">"-t"</span><span class="p">,</span> <span class="s2">"--timeout"</span><span class="p">,</span>
|
|
<span class="n">help</span><span class="o">=</span><span class="s2">"number of seconds to wait for an answer "</span>
|
|
<span class="s2">"from DNS (default 6.0)"</span><span class="p">,</span>
|
|
<span class="nb">type</span><span class="o">=</span><span class="nb">float</span><span class="p">,</span>
|
|
<span class="n">default</span><span class="o">=</span><span class="mf">6.0</span><span class="p">)</span>
|
|
<span class="n">arg_parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span><span class="s2">"-v"</span><span class="p">,</span> <span class="s2">"--version"</span><span class="p">,</span> <span class="n">action</span><span class="o">=</span><span class="s2">"version"</span><span class="p">,</span>
|
|
<span class="n">version</span><span class="o">=</span><span class="n">__version__</span><span class="p">)</span>
|
|
|
|
<span class="n">args</span> <span class="o">=</span> <span class="n">arg_parser</span><span class="o">.</span><span class="n">parse_args</span><span class="p">()</span>
|
|
<span class="n">file_paths</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="k">for</span> <span class="n">file_path</span> <span class="ow">in</span> <span class="n">args</span><span class="o">.</span><span class="n">file_path</span><span class="p">:</span>
|
|
<span class="n">file_paths</span> <span class="o">+=</span> <span class="n">glob</span><span class="p">(</span><span class="n">file_path</span><span class="p">)</span>
|
|
<span class="n">file_paths</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">set</span><span class="p">(</span><span class="n">file_paths</span><span class="p">))</span>
|
|
|
|
<span class="n">parsed_reports</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="k">for</span> <span class="n">file_path</span> <span class="ow">in</span> <span class="n">file_paths</span><span class="p">:</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">report</span> <span class="o">=</span> <span class="n">parse_aggregate_report_file</span><span class="p">(</span><span class="n">file_path</span><span class="p">,</span>
|
|
<span class="n">nameservers</span><span class="o">=</span><span class="n">args</span><span class="o">.</span><span class="n">nameserver</span><span class="p">,</span>
|
|
<span class="n">timeout</span><span class="o">=</span><span class="n">args</span><span class="o">.</span><span class="n">timeout</span><span class="p">)</span>
|
|
<span class="n">parsed_reports</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">report</span><span class="p">)</span>
|
|
<span class="k">except</span> <span class="n">InvalidAggregateReport</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Unable to parse </span><span class="si">{0}</span><span class="s2">: </span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">file_path</span><span class="p">,</span>
|
|
<span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
|
<span class="n">output</span> <span class="o">=</span> <span class="s2">""</span>
|
|
<span class="k">if</span> <span class="n">args</span><span class="o">.</span><span class="n">format</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s2">"json"</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">parsed_reports</span><span class="p">)</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
|
|
<span class="n">parsed_reports</span> <span class="o">=</span> <span class="n">parsed_reports</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
|
|
<span class="n">output</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">parsed_reports</span><span class="p">,</span>
|
|
<span class="n">ensure_ascii</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
|
<span class="n">indent</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
|
|
<span class="k">elif</span> <span class="n">args</span><span class="o">.</span><span class="n">format</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s2">"csv"</span><span class="p">:</span>
|
|
<span class="n">output</span> <span class="o">=</span> <span class="n">parsed_aggregate_report_to_csv</span><span class="p">(</span><span class="n">parsed_reports</span><span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Invalid output format: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">args</span><span class="o">.</span><span class="n">format</span><span class="p">))</span>
|
|
<span class="n">exit</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span>
|
|
|
|
<span class="k">if</span> <span class="n">args</span><span class="o">.</span><span class="n">output</span><span class="p">:</span>
|
|
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">args</span><span class="o">.</span><span class="n">output</span><span class="p">,</span> <span class="s2">"w"</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s2">"utf-8"</span><span class="p">,</span> <span class="n">newline</span><span class="o">=</span><span class="s2">"</span><span class="se">\n</span><span class="s2">"</span><span class="p">)</span> <span class="k">as</span> <span class="n">file</span><span class="p">:</span>
|
|
<span class="n">file</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">output</span><span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="nb">print</span><span class="p">(</span><span class="n">output</span><span class="p">)</span>
|
|
|
|
|
|
<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
|
|
<span class="n">_main</span><span class="p">()</span>
|
|
</pre></div>
|
|
|
|
</div>
|
|
<div class="articleComments">
|
|
|
|
</div>
|
|
</div>
|
|
<footer>
|
|
|
|
|
|
<hr/>
|
|
|
|
<div role="contentinfo">
|
|
<p>
|
|
© Copyright 2018, Sean Whalen.
|
|
|
|
</p>
|
|
</div>
|
|
Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
|
|
|
|
</footer>
|
|
|
|
</div>
|
|
</div>
|
|
|
|
</section>
|
|
|
|
</div>
|
|
|
|
|
|
|
|
|
|
|
|
<script type="text/javascript">
|
|
var DOCUMENTATION_OPTIONS = {
|
|
URL_ROOT:'../',
|
|
VERSION:'1.0.0',
|
|
COLLAPSE_INDEX:false,
|
|
FILE_SUFFIX:'.html',
|
|
HAS_SOURCE: true,
|
|
SOURCELINK_SUFFIX: '.txt'
|
|
};
|
|
</script>
|
|
<script type="text/javascript" src="../_static/jquery.js"></script>
|
|
<script type="text/javascript" src="../_static/underscore.js"></script>
|
|
<script type="text/javascript" src="../_static/doctools.js"></script>
|
|
|
|
|
|
|
|
|
|
|
|
<script type="text/javascript" src="../_static/js/theme.js"></script>
|
|
|
|
|
|
|
|
|
|
<script type="text/javascript">
|
|
jQuery(function () {
|
|
SphinxRtdTheme.StickyNav.enable();
|
|
});
|
|
</script>
|
|
|
|
|
|
</body>
|
|
</html> |