mirror of
https://github.com/domainaware/parsedmarc.git
synced 2026-03-25 16:02:47 +00:00
992 lines
134 KiB
HTML
992 lines
134 KiB
HTML
|
|
|
|
<!DOCTYPE html>
|
|
<html class="writer-html5" lang="en" data-content_root="../../">
|
|
<head>
|
|
<meta charset="utf-8" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
<title>parsedmarc.elastic — parsedmarc 9.0.2 documentation</title>
|
|
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=b86133f3" />
|
|
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=e59714d7" />
|
|
|
|
|
|
<script src="../../_static/jquery.js?v=5d32c60e"></script>
|
|
<script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
|
<script src="../../_static/documentation_options.js?v=9893da7f"></script>
|
|
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
|
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
|
<script src="../../_static/js/theme.js"></script>
|
|
<link rel="index" title="Index" href="../../genindex.html" />
|
|
<link rel="search" title="Search" href="../../search.html" />
|
|
</head>
|
|
|
|
<body class="wy-body-for-nav">
|
|
<div class="wy-grid-for-nav">
|
|
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
|
|
<div class="wy-side-scroll">
|
|
<div class="wy-side-nav-search" >
|
|
|
|
|
|
|
|
<a href="../../index.html" class="icon icon-home">
|
|
parsedmarc
|
|
</a>
|
|
<div role="search">
|
|
<form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
|
|
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
|
<input type="hidden" name="check_keywords" value="yes" />
|
|
<input type="hidden" name="area" value="default" />
|
|
</form>
|
|
</div>
|
|
</div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
|
|
<p class="caption" role="heading"><span class="caption-text">Contents</span></p>
|
|
<ul>
|
|
<li class="toctree-l1"><a class="reference internal" href="../../installation.html">Installation</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../../usage.html">Using parsedmarc</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../../output.html">Sample outputs</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../../elasticsearch.html">Elasticsearch and Kibana</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../../opensearch.html">OpenSearch and Grafana</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../../kibana.html">Using the Kibana dashboards</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../../splunk.html">Splunk</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../../davmail.html">Accessing an inbox using OWA/EWS</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../../dmarc.html">Understanding DMARC</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../../contributing.html">Contributing to parsedmarc</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../../api.html">API reference</a></li>
|
|
</ul>
|
|
|
|
</div>
|
|
</div>
|
|
</nav>
|
|
|
|
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu" >
|
|
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
|
|
<a href="../../index.html">parsedmarc</a>
|
|
</nav>
|
|
|
|
<div class="wy-nav-content">
|
|
<div class="rst-content">
|
|
<div role="navigation" aria-label="Page navigation">
|
|
<ul class="wy-breadcrumbs">
|
|
<li><a href="../../index.html" class="icon icon-home" aria-label="Home"></a></li>
|
|
<li class="breadcrumb-item"><a href="../index.html">Module code</a></li>
|
|
<li class="breadcrumb-item"><a href="../parsedmarc.html">parsedmarc</a></li>
|
|
<li class="breadcrumb-item active">parsedmarc.elastic</li>
|
|
<li class="wy-breadcrumbs-aside">
|
|
</li>
|
|
</ul>
|
|
<hr/>
|
|
</div>
|
|
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
|
|
<div itemprop="articleBody">
|
|
|
|
<h1>Source code for parsedmarc.elastic</h1><div class="highlight"><pre>
|
|
<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
|
|
|
|
<span class="kn">from</span><span class="w"> </span><span class="nn">__future__</span><span class="w"> </span><span class="kn">import</span> <span class="n">annotations</span>
|
|
|
|
<span class="kn">from</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="kn">import</span> <span class="n">Optional</span><span class="p">,</span> <span class="n">Union</span><span class="p">,</span> <span class="n">Any</span>
|
|
|
|
<span class="kn">from</span><span class="w"> </span><span class="nn">collections</span><span class="w"> </span><span class="kn">import</span> <span class="n">OrderedDict</span>
|
|
|
|
<span class="kn">from</span><span class="w"> </span><span class="nn">elasticsearch_dsl.search</span><span class="w"> </span><span class="kn">import</span> <span class="n">Q</span>
|
|
<span class="kn">from</span><span class="w"> </span><span class="nn">elasticsearch_dsl</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
|
|
<span class="n">connections</span><span class="p">,</span>
|
|
<span class="n">Object</span><span class="p">,</span>
|
|
<span class="n">Document</span><span class="p">,</span>
|
|
<span class="n">Index</span><span class="p">,</span>
|
|
<span class="n">Nested</span><span class="p">,</span>
|
|
<span class="n">InnerDoc</span><span class="p">,</span>
|
|
<span class="n">Integer</span><span class="p">,</span>
|
|
<span class="n">Text</span><span class="p">,</span>
|
|
<span class="n">Boolean</span><span class="p">,</span>
|
|
<span class="n">Ip</span><span class="p">,</span>
|
|
<span class="n">Date</span><span class="p">,</span>
|
|
<span class="n">Search</span><span class="p">,</span>
|
|
<span class="p">)</span>
|
|
<span class="kn">from</span><span class="w"> </span><span class="nn">elasticsearch.helpers</span><span class="w"> </span><span class="kn">import</span> <span class="n">reindex</span>
|
|
|
|
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.log</span><span class="w"> </span><span class="kn">import</span> <span class="n">logger</span>
|
|
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">human_timestamp_to_datetime</span>
|
|
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc</span><span class="w"> </span><span class="kn">import</span> <span class="n">InvalidForensicReport</span>
|
|
|
|
|
|
<div class="viewcode-block" id="ElasticsearchError">
|
|
<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.ElasticsearchError">[docs]</a>
|
|
<span class="k">class</span><span class="w"> </span><span class="nc">ElasticsearchError</span><span class="p">(</span><span class="ne">Exception</span><span class="p">):</span>
|
|
<span class="w"> </span><span class="sd">"""Raised when an Elasticsearch error occurs"""</span></div>
|
|
|
|
|
|
|
|
<span class="k">class</span><span class="w"> </span><span class="nc">_PolicyOverride</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
|
<span class="nb">type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">comment</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
|
|
|
|
<span class="k">class</span><span class="w"> </span><span class="nc">_PublishedPolicy</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
|
<span class="n">domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">adkim</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">aspf</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">p</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">sp</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">pct</span> <span class="o">=</span> <span class="n">Integer</span><span class="p">()</span>
|
|
<span class="n">fo</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
|
|
|
|
<span class="k">class</span><span class="w"> </span><span class="nc">_DKIMResult</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
|
<span class="n">domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">selector</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">result</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
|
|
|
|
<span class="k">class</span><span class="w"> </span><span class="nc">_SPFResult</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
|
<span class="n">domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">scope</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">results</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
|
|
|
|
<span class="k">class</span><span class="w"> </span><span class="nc">_AggregateReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
|
<span class="k">class</span><span class="w"> </span><span class="nc">Index</span><span class="p">:</span>
|
|
<span class="n">name</span> <span class="o">=</span> <span class="s2">"dmarc_aggregate"</span>
|
|
|
|
<span class="n">xml_schema</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">org_name</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">org_email</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">org_extra_contact_info</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">report_id</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">date_range</span> <span class="o">=</span> <span class="n">Date</span><span class="p">()</span>
|
|
<span class="n">date_begin</span> <span class="o">=</span> <span class="n">Date</span><span class="p">()</span>
|
|
<span class="n">date_end</span> <span class="o">=</span> <span class="n">Date</span><span class="p">()</span>
|
|
<span class="n">normalized_timespan</span> <span class="o">=</span> <span class="n">Boolean</span><span class="p">()</span>
|
|
<span class="n">original_timespan_seconds</span> <span class="o">=</span> <span class="n">Integer</span>
|
|
<span class="n">errors</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">published_policy</span> <span class="o">=</span> <span class="n">Object</span><span class="p">(</span><span class="n">_PublishedPolicy</span><span class="p">)</span>
|
|
<span class="n">source_ip_address</span> <span class="o">=</span> <span class="n">Ip</span><span class="p">()</span>
|
|
<span class="n">source_country</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">source_reverse_dns</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">source_base_domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">source_type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">source_name</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">message_count</span> <span class="o">=</span> <span class="n">Integer</span>
|
|
<span class="n">disposition</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">dkim_aligned</span> <span class="o">=</span> <span class="n">Boolean</span><span class="p">()</span>
|
|
<span class="n">spf_aligned</span> <span class="o">=</span> <span class="n">Boolean</span><span class="p">()</span>
|
|
<span class="n">passed_dmarc</span> <span class="o">=</span> <span class="n">Boolean</span><span class="p">()</span>
|
|
<span class="n">policy_overrides</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_PolicyOverride</span><span class="p">)</span>
|
|
<span class="n">header_from</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">envelope_from</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">envelope_to</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">dkim_results</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_DKIMResult</span><span class="p">)</span>
|
|
<span class="n">spf_results</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SPFResult</span><span class="p">)</span>
|
|
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">add_policy_override</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">type_</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">comment</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">policy_overrides</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_PolicyOverride</span><span class="p">(</span><span class="nb">type</span><span class="o">=</span><span class="n">type_</span><span class="p">,</span> <span class="n">comment</span><span class="o">=</span><span class="n">comment</span><span class="p">))</span>
|
|
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">add_dkim_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">domain</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">selector</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">result</span><span class="p">:</span> <span class="n">_DKIMResult</span><span class="p">):</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">dkim_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
|
<span class="n">_DKIMResult</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span> <span class="n">selector</span><span class="o">=</span><span class="n">selector</span><span class="p">,</span> <span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">)</span>
|
|
<span class="p">)</span>
|
|
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">add_spf_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">domain</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">scope</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">result</span><span class="p">:</span> <span class="n">_SPFResult</span><span class="p">):</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">spf_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_SPFResult</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span> <span class="n">scope</span><span class="o">=</span><span class="n">scope</span><span class="p">,</span> <span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">))</span>
|
|
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">save</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">passed_dmarc</span> <span class="o">=</span> <span class="kc">False</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">passed_dmarc</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">spf_aligned</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">dkim_aligned</span>
|
|
|
|
<span class="k">return</span> <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
|
|
|
|
|
|
<span class="k">class</span><span class="w"> </span><span class="nc">_EmailAddressDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
|
<span class="n">display_name</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">address</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
|
|
|
|
<span class="k">class</span><span class="w"> </span><span class="nc">_EmailAttachmentDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
|
<span class="n">filename</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">content_type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">sha256</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
|
|
|
|
<span class="k">class</span><span class="w"> </span><span class="nc">_ForensicSampleDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
|
<span class="n">raw</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">headers</span> <span class="o">=</span> <span class="n">Object</span><span class="p">()</span>
|
|
<span class="n">headers_only</span> <span class="o">=</span> <span class="n">Boolean</span><span class="p">()</span>
|
|
<span class="n">to</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">)</span>
|
|
<span class="n">subject</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">filename_safe_subject</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">_from</span> <span class="o">=</span> <span class="n">Object</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">)</span>
|
|
<span class="n">date</span> <span class="o">=</span> <span class="n">Date</span><span class="p">()</span>
|
|
<span class="n">reply_to</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">)</span>
|
|
<span class="n">cc</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">)</span>
|
|
<span class="n">bcc</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">)</span>
|
|
<span class="n">body</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">attachments</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_EmailAttachmentDoc</span><span class="p">)</span>
|
|
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">add_to</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">address</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">to</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
|
|
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">add_reply_to</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">address</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">reply_to</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
|
<span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">)</span>
|
|
<span class="p">)</span>
|
|
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">add_cc</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">address</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">cc</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
|
|
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">add_bcc</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">address</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">bcc</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
|
|
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">add_attachment</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">filename</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">content_type</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">sha256</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">attachments</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
|
<span class="n">_EmailAttachmentDoc</span><span class="p">(</span>
|
|
<span class="n">filename</span><span class="o">=</span><span class="n">filename</span><span class="p">,</span> <span class="n">content_type</span><span class="o">=</span><span class="n">content_type</span><span class="p">,</span> <span class="n">sha256</span><span class="o">=</span><span class="n">sha256</span>
|
|
<span class="p">)</span>
|
|
<span class="p">)</span>
|
|
|
|
|
|
<span class="k">class</span><span class="w"> </span><span class="nc">_ForensicReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
|
<span class="k">class</span><span class="w"> </span><span class="nc">Index</span><span class="p">:</span>
|
|
<span class="n">name</span> <span class="o">=</span> <span class="s2">"dmarc_forensic"</span>
|
|
|
|
<span class="n">feedback_type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">user_agent</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">version</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">original_mail_from</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">arrival_date</span> <span class="o">=</span> <span class="n">Date</span><span class="p">()</span>
|
|
<span class="n">domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">original_envelope_id</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">authentication_results</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">delivery_results</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">source_ip_address</span> <span class="o">=</span> <span class="n">Ip</span><span class="p">()</span>
|
|
<span class="n">source_country</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">source_reverse_dns</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">source_authentication_mechanisms</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">source_auth_failures</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">dkim_domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">original_rcpt_to</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">sample</span> <span class="o">=</span> <span class="n">Object</span><span class="p">(</span><span class="n">_ForensicSampleDoc</span><span class="p">)</span>
|
|
|
|
|
|
<span class="k">class</span><span class="w"> </span><span class="nc">_SMTPTLSFailureDetailsDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
|
<span class="n">result_type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">sending_mta_ip</span> <span class="o">=</span> <span class="n">Ip</span><span class="p">()</span>
|
|
<span class="n">receiving_mx_helo</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">receiving_ip</span> <span class="o">=</span> <span class="n">Ip</span><span class="p">()</span>
|
|
<span class="n">failed_session_count</span> <span class="o">=</span> <span class="n">Integer</span><span class="p">()</span>
|
|
<span class="n">additional_information_uri</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">failure_reason_code</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
|
|
|
|
<span class="k">class</span><span class="w"> </span><span class="nc">_SMTPTLSPolicyDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
|
<span class="n">policy_domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">policy_type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">policy_strings</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">mx_host_patterns</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">successful_session_count</span> <span class="o">=</span> <span class="n">Integer</span><span class="p">()</span>
|
|
<span class="n">failed_session_count</span> <span class="o">=</span> <span class="n">Integer</span><span class="p">()</span>
|
|
<span class="n">failure_details</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SMTPTLSFailureDetailsDoc</span><span class="p">)</span>
|
|
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">add_failure_details</span><span class="p">(</span>
|
|
<span class="bp">self</span><span class="p">,</span>
|
|
<span class="n">result_type</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
|
<span class="n">ip_address</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
|
<span class="n">receiving_ip</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
|
<span class="n">receiving_mx_helo</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
|
<span class="n">failed_session_count</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
|
|
<span class="n">sending_mta_ip</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">receiving_mx_hostname</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">additional_information_uri</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">failure_reason_code</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="kc">None</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="p">):</span>
|
|
<span class="n">_details</span> <span class="o">=</span> <span class="n">_SMTPTLSFailureDetailsDoc</span><span class="p">(</span>
|
|
<span class="n">result_type</span><span class="o">=</span><span class="n">result_type</span><span class="p">,</span>
|
|
<span class="n">ip_address</span><span class="o">=</span><span class="n">ip_address</span><span class="p">,</span>
|
|
<span class="n">sending_mta_ip</span><span class="o">=</span><span class="n">sending_mta_ip</span><span class="p">,</span>
|
|
<span class="n">receiving_mx_hostname</span><span class="o">=</span><span class="n">receiving_mx_hostname</span><span class="p">,</span>
|
|
<span class="n">receiving_mx_helo</span><span class="o">=</span><span class="n">receiving_mx_helo</span><span class="p">,</span>
|
|
<span class="n">receiving_ip</span><span class="o">=</span><span class="n">receiving_ip</span><span class="p">,</span>
|
|
<span class="n">failed_session_count</span><span class="o">=</span><span class="n">failed_session_count</span><span class="p">,</span>
|
|
<span class="n">additional_information</span><span class="o">=</span><span class="n">additional_information_uri</span><span class="p">,</span>
|
|
<span class="n">failure_reason_code</span><span class="o">=</span><span class="n">failure_reason_code</span><span class="p">,</span>
|
|
<span class="p">)</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">failure_details</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_details</span><span class="p">)</span>
|
|
|
|
|
|
<span class="k">class</span><span class="w"> </span><span class="nc">_SMTPTLSReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
|
<span class="k">class</span><span class="w"> </span><span class="nc">Index</span><span class="p">:</span>
|
|
<span class="n">name</span> <span class="o">=</span> <span class="s2">"smtp_tls"</span>
|
|
|
|
<span class="n">organization_name</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">date_range</span> <span class="o">=</span> <span class="n">Date</span><span class="p">()</span>
|
|
<span class="n">date_begin</span> <span class="o">=</span> <span class="n">Date</span><span class="p">()</span>
|
|
<span class="n">date_end</span> <span class="o">=</span> <span class="n">Date</span><span class="p">()</span>
|
|
<span class="n">contact_info</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">report_id</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
|
<span class="n">policies</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SMTPTLSPolicyDoc</span><span class="p">)</span>
|
|
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">add_policy</span><span class="p">(</span>
|
|
<span class="bp">self</span><span class="p">,</span>
|
|
<span class="n">policy_type</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
|
<span class="n">policy_domain</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
|
<span class="n">successful_session_count</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
|
|
<span class="n">failed_session_count</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
|
|
<span class="o">*</span><span class="p">,</span>
|
|
<span class="n">policy_string</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">mx_host_patterns</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">failure_details</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="p">):</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">policies</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
|
<span class="n">policy_type</span><span class="o">=</span><span class="n">policy_type</span><span class="p">,</span>
|
|
<span class="n">policy_domain</span><span class="o">=</span><span class="n">policy_domain</span><span class="p">,</span>
|
|
<span class="n">successful_session_count</span><span class="o">=</span><span class="n">successful_session_count</span><span class="p">,</span>
|
|
<span class="n">failed_session_count</span><span class="o">=</span><span class="n">failed_session_count</span><span class="p">,</span>
|
|
<span class="n">policy_string</span><span class="o">=</span><span class="n">policy_string</span><span class="p">,</span>
|
|
<span class="n">mx_host_patterns</span><span class="o">=</span><span class="n">mx_host_patterns</span><span class="p">,</span>
|
|
<span class="n">failure_details</span><span class="o">=</span><span class="n">failure_details</span><span class="p">,</span>
|
|
<span class="p">)</span>
|
|
|
|
|
|
<div class="viewcode-block" id="AlreadySaved">
|
|
<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.AlreadySaved">[docs]</a>
|
|
<span class="k">class</span><span class="w"> </span><span class="nc">AlreadySaved</span><span class="p">(</span><span class="ne">ValueError</span><span class="p">):</span>
|
|
<span class="w"> </span><span class="sd">"""Raised when a report to be saved matches an existing report"""</span></div>
|
|
|
|
|
|
|
|
<div class="viewcode-block" id="set_hosts">
|
|
<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.set_hosts">[docs]</a>
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">set_hosts</span><span class="p">(</span>
|
|
<span class="n">hosts</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]],</span>
|
|
<span class="o">*</span><span class="p">,</span>
|
|
<span class="n">use_ssl</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
|
<span class="n">ssl_cert_path</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">username</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">password</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">api_key</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">timeout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="mf">60.0</span><span class="p">,</span>
|
|
<span class="p">):</span>
|
|
<span class="w"> </span><span class="sd">"""</span>
|
|
<span class="sd"> Sets the Elasticsearch hosts to use</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> hosts (str | list[str]): A single hostname or URL, or list of hostnames or URLs</span>
|
|
<span class="sd"> use_ssl (bool): Use an HTTPS connection to the server</span>
|
|
<span class="sd"> ssl_cert_path (str): Path to the certificate chain</span>
|
|
<span class="sd"> username (str): The username to use for authentication</span>
|
|
<span class="sd"> password (str): The password to use for authentication</span>
|
|
<span class="sd"> api_key (str): The Base64 encoded API key to use for authentication</span>
|
|
<span class="sd"> timeout (float): Timeout in seconds</span>
|
|
<span class="sd"> """</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">hosts</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
|
|
<span class="n">hosts</span> <span class="o">=</span> <span class="p">[</span><span class="n">hosts</span><span class="p">]</span>
|
|
<span class="n">conn_params</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"hosts"</span><span class="p">:</span> <span class="n">hosts</span><span class="p">,</span> <span class="s2">"timeout"</span><span class="p">:</span> <span class="n">timeout</span><span class="p">}</span>
|
|
<span class="k">if</span> <span class="n">use_ssl</span><span class="p">:</span>
|
|
<span class="n">conn_params</span><span class="p">[</span><span class="s2">"use_ssl"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
|
|
<span class="k">if</span> <span class="n">ssl_cert_path</span><span class="p">:</span>
|
|
<span class="n">conn_params</span><span class="p">[</span><span class="s2">"verify_certs"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
|
|
<span class="n">conn_params</span><span class="p">[</span><span class="s2">"ca_certs"</span><span class="p">]</span> <span class="o">=</span> <span class="n">ssl_cert_path</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">conn_params</span><span class="p">[</span><span class="s2">"verify_certs"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span>
|
|
<span class="k">if</span> <span class="n">username</span><span class="p">:</span>
|
|
<span class="n">conn_params</span><span class="p">[</span><span class="s2">"http_auth"</span><span class="p">]</span> <span class="o">=</span> <span class="n">username</span> <span class="o">+</span> <span class="s2">":"</span> <span class="o">+</span> <span class="n">password</span>
|
|
<span class="k">if</span> <span class="n">api_key</span><span class="p">:</span>
|
|
<span class="n">conn_params</span><span class="p">[</span><span class="s2">"api_key"</span><span class="p">]</span> <span class="o">=</span> <span class="n">api_key</span>
|
|
<span class="n">connections</span><span class="o">.</span><span class="n">create_connection</span><span class="p">(</span><span class="o">**</span><span class="n">conn_params</span><span class="p">)</span></div>
|
|
|
|
|
|
|
|
<div class="viewcode-block" id="create_indexes">
|
|
<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.create_indexes">[docs]</a>
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">create_indexes</span><span class="p">(</span><span class="n">names</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="n">settings</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">):</span>
|
|
<span class="w"> </span><span class="sd">"""</span>
|
|
<span class="sd"> Create Elasticsearch indexes</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> names (list): A list of index names</span>
|
|
<span class="sd"> settings (dict): Index settings</span>
|
|
|
|
<span class="sd"> """</span>
|
|
<span class="k">for</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">names</span><span class="p">:</span>
|
|
<span class="n">index</span> <span class="o">=</span> <span class="n">Index</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">index</span><span class="o">.</span><span class="n">exists</span><span class="p">():</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Creating Elasticsearch index: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">name</span><span class="p">))</span>
|
|
<span class="k">if</span> <span class="n">settings</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">index</span><span class="o">.</span><span class="n">settings</span><span class="p">(</span><span class="n">number_of_shards</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">number_of_replicas</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">index</span><span class="o">.</span><span class="n">settings</span><span class="p">(</span><span class="o">**</span><span class="n">settings</span><span class="p">)</span>
|
|
<span class="n">index</span><span class="o">.</span><span class="n">create</span><span class="p">()</span>
|
|
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="s2">"Elasticsearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
|
|
|
|
|
|
|
<div class="viewcode-block" id="migrate_indexes">
|
|
<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.migrate_indexes">[docs]</a>
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">migrate_indexes</span><span class="p">(</span>
|
|
<span class="n">aggregate_indexes</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">forensic_indexes</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="p">):</span>
|
|
<span class="w"> </span><span class="sd">"""</span>
|
|
<span class="sd"> Updates index mappings</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> aggregate_indexes (list): A list of aggregate index names</span>
|
|
<span class="sd"> forensic_indexes (list): A list of forensic index names</span>
|
|
<span class="sd"> """</span>
|
|
<span class="n">version</span> <span class="o">=</span> <span class="mi">2</span>
|
|
<span class="k">if</span> <span class="n">aggregate_indexes</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">aggregate_indexes</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="k">if</span> <span class="n">forensic_indexes</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">forensic_indexes</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="k">for</span> <span class="n">aggregate_index_name</span> <span class="ow">in</span> <span class="n">aggregate_indexes</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">Index</span><span class="p">(</span><span class="n">aggregate_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">exists</span><span class="p">():</span>
|
|
<span class="k">continue</span>
|
|
<span class="n">aggregate_index</span> <span class="o">=</span> <span class="n">Index</span><span class="p">(</span><span class="n">aggregate_index_name</span><span class="p">)</span>
|
|
<span class="n">doc</span> <span class="o">=</span> <span class="s2">"doc"</span>
|
|
<span class="n">fo_field</span> <span class="o">=</span> <span class="s2">"published_policy.fo"</span>
|
|
<span class="n">fo</span> <span class="o">=</span> <span class="s2">"fo"</span>
|
|
<span class="n">fo_mapping</span> <span class="o">=</span> <span class="n">aggregate_index</span><span class="o">.</span><span class="n">get_field_mapping</span><span class="p">(</span><span class="n">fields</span><span class="o">=</span><span class="p">[</span><span class="n">fo_field</span><span class="p">])</span>
|
|
<span class="n">fo_mapping</span> <span class="o">=</span> <span class="n">fo_mapping</span><span class="p">[</span><span class="nb">list</span><span class="p">(</span><span class="n">fo_mapping</span><span class="o">.</span><span class="n">keys</span><span class="p">())[</span><span class="mi">0</span><span class="p">]][</span><span class="s2">"mappings"</span><span class="p">]</span>
|
|
<span class="k">if</span> <span class="n">doc</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">fo_mapping</span><span class="p">:</span>
|
|
<span class="k">continue</span>
|
|
|
|
<span class="n">fo_mapping</span> <span class="o">=</span> <span class="n">fo_mapping</span><span class="p">[</span><span class="n">doc</span><span class="p">][</span><span class="n">fo_field</span><span class="p">][</span><span class="s2">"mapping"</span><span class="p">][</span><span class="n">fo</span><span class="p">]</span>
|
|
<span class="n">fo_type</span> <span class="o">=</span> <span class="n">fo_mapping</span><span class="p">[</span><span class="s2">"type"</span><span class="p">]</span>
|
|
<span class="k">if</span> <span class="n">fo_type</span> <span class="o">==</span> <span class="s2">"long"</span><span class="p">:</span>
|
|
<span class="n">new_index_name</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">-v</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">aggregate_index_name</span><span class="p">,</span> <span class="n">version</span><span class="p">)</span>
|
|
<span class="n">body</span> <span class="o">=</span> <span class="p">{</span>
|
|
<span class="s2">"properties"</span><span class="p">:</span> <span class="p">{</span>
|
|
<span class="s2">"published_policy.fo"</span><span class="p">:</span> <span class="p">{</span>
|
|
<span class="s2">"type"</span><span class="p">:</span> <span class="s2">"text"</span><span class="p">,</span>
|
|
<span class="s2">"fields"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"keyword"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"type"</span><span class="p">:</span> <span class="s2">"keyword"</span><span class="p">,</span> <span class="s2">"ignore_above"</span><span class="p">:</span> <span class="mi">256</span><span class="p">}},</span>
|
|
<span class="p">}</span>
|
|
<span class="p">}</span>
|
|
<span class="p">}</span>
|
|
<span class="n">Index</span><span class="p">(</span><span class="n">new_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">create</span><span class="p">()</span>
|
|
<span class="n">Index</span><span class="p">(</span><span class="n">new_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">put_mapping</span><span class="p">(</span><span class="n">doc_type</span><span class="o">=</span><span class="n">doc</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="n">body</span><span class="p">)</span>
|
|
<span class="n">reindex</span><span class="p">(</span><span class="n">connections</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(),</span> <span class="n">aggregate_index_name</span><span class="p">,</span> <span class="n">new_index_name</span><span class="p">)</span>
|
|
<span class="n">Index</span><span class="p">(</span><span class="n">aggregate_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">delete</span><span class="p">()</span>
|
|
|
|
<span class="k">for</span> <span class="n">forensic_index</span> <span class="ow">in</span> <span class="n">forensic_indexes</span><span class="p">:</span>
|
|
<span class="k">pass</span></div>
|
|
|
|
|
|
|
|
<div class="viewcode-block" id="save_aggregate_report_to_elasticsearch">
|
|
<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.save_aggregate_report_to_elasticsearch">[docs]</a>
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">save_aggregate_report_to_elasticsearch</span><span class="p">(</span>
|
|
<span class="n">aggregate_report</span><span class="p">:</span> <span class="n">OrderedDict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">],</span>
|
|
<span class="n">index_suffix</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">index_prefix</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">monthly_indexes</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
|
<span class="n">number_of_shards</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
|
|
<span class="n">number_of_replicas</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="mi">0</span><span class="p">,</span>
|
|
<span class="p">):</span>
|
|
<span class="w"> </span><span class="sd">"""</span>
|
|
<span class="sd"> Saves a parsed DMARC aggregate report to Elasticsearch</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> aggregate_report (OrderedDict): A parsed forensic report</span>
|
|
<span class="sd"> index_suffix (str): The suffix of the name of the index to save to</span>
|
|
<span class="sd"> index_prefix (str): The prefix of the name of the index to save to</span>
|
|
<span class="sd"> monthly_indexes (bool): Use monthly indexes instead of daily indexes</span>
|
|
<span class="sd"> number_of_shards (int): The number of shards to use in the index</span>
|
|
<span class="sd"> number_of_replicas (int): The number of replicas to use in the index</span>
|
|
|
|
<span class="sd"> Raises:</span>
|
|
<span class="sd"> AlreadySaved</span>
|
|
<span class="sd"> """</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Saving aggregate report to Elasticsearch"</span><span class="p">)</span>
|
|
<span class="n">aggregate_report</span> <span class="o">=</span> <span class="n">aggregate_report</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
|
<span class="n">metadata</span> <span class="o">=</span> <span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">]</span>
|
|
<span class="n">org_name</span> <span class="o">=</span> <span class="n">metadata</span><span class="p">[</span><span class="s2">"org_name"</span><span class="p">]</span>
|
|
<span class="n">report_id</span> <span class="o">=</span> <span class="n">metadata</span><span class="p">[</span><span class="s2">"report_id"</span><span class="p">]</span>
|
|
<span class="n">domain</span> <span class="o">=</span> <span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"domain"</span><span class="p">]</span>
|
|
<span class="n">begin_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">metadata</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
|
<span class="n">end_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">metadata</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
|
|
|
<span class="k">if</span> <span class="n">monthly_indexes</span><span class="p">:</span>
|
|
<span class="n">index_date</span> <span class="o">=</span> <span class="n">begin_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m"</span><span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">index_date</span> <span class="o">=</span> <span class="n">begin_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2">"</span><span class="p">)</span>
|
|
|
|
<span class="n">org_name_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match_phrase</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">org_name</span><span class="o">=</span><span class="n">org_name</span><span class="p">)))</span>
|
|
<span class="n">report_id_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match_phrase</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">report_id</span><span class="o">=</span><span class="n">report_id</span><span class="p">)))</span>
|
|
<span class="n">domain_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match_phrase</span><span class="o">=</span><span class="p">{</span><span class="s2">"published_policy.domain"</span><span class="p">:</span> <span class="n">domain</span><span class="p">}))</span>
|
|
<span class="n">begin_date_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">date_begin</span><span class="o">=</span><span class="n">begin_date</span><span class="p">)))</span>
|
|
<span class="n">end_date_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">date_end</span><span class="o">=</span><span class="n">end_date</span><span class="p">)))</span>
|
|
|
|
<span class="k">if</span> <span class="n">index_suffix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"dmarc_aggregate_</span><span class="si">{0}</span><span class="s2">*"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_suffix</span><span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"dmarc_aggregate*"</span>
|
|
<span class="k">if</span> <span class="n">index_prefix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">search_index</span><span class="p">)</span>
|
|
<span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="n">search_index</span><span class="p">)</span>
|
|
<span class="n">query</span> <span class="o">=</span> <span class="n">org_name_query</span> <span class="o">&</span> <span class="n">report_id_query</span> <span class="o">&</span> <span class="n">domain_query</span>
|
|
<span class="n">query</span> <span class="o">=</span> <span class="n">query</span> <span class="o">&</span> <span class="n">begin_date_query</span> <span class="o">&</span> <span class="n">end_date_query</span>
|
|
<span class="n">search</span><span class="o">.</span><span class="n">query</span> <span class="o">=</span> <span class="n">query</span>
|
|
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">existing</span> <span class="o">=</span> <span class="n">search</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
|
|
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error_</span><span class="p">:</span>
|
|
<span class="n">begin_date_human</span> <span class="o">=</span> <span class="n">begin_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%SZ"</span><span class="p">)</span>
|
|
<span class="n">end_date_human</span> <span class="o">=</span> <span class="n">end_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%SZ"</span><span class="p">)</span>
|
|
|
|
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span>
|
|
<span class="s2">"Elasticsearch's search for existing report </span><span class="se">\</span>
|
|
<span class="s2"> error: </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error_</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
|
<span class="p">)</span>
|
|
|
|
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">existing</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">AlreadySaved</span><span class="p">(</span>
|
|
<span class="s2">"An aggregate report ID </span><span class="si">{0}</span><span class="s2"> from </span><span class="si">{1}</span><span class="s2"> about </span><span class="si">{2}</span><span class="s2"> "</span>
|
|
<span class="s2">"with a date range of </span><span class="si">{3}</span><span class="s2"> UTC to </span><span class="si">{4}</span><span class="s2"> UTC already "</span>
|
|
<span class="s2">"exists in "</span>
|
|
<span class="s2">"Elasticsearch"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
|
<span class="n">report_id</span><span class="p">,</span> <span class="n">org_name</span><span class="p">,</span> <span class="n">domain</span><span class="p">,</span> <span class="n">begin_date_human</span><span class="p">,</span> <span class="n">end_date_human</span>
|
|
<span class="p">)</span>
|
|
<span class="p">)</span>
|
|
<span class="n">published_policy</span> <span class="o">=</span> <span class="n">_PublishedPolicy</span><span class="p">(</span>
|
|
<span class="n">domain</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"domain"</span><span class="p">],</span>
|
|
<span class="n">adkim</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"adkim"</span><span class="p">],</span>
|
|
<span class="n">aspf</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"aspf"</span><span class="p">],</span>
|
|
<span class="n">p</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"p"</span><span class="p">],</span>
|
|
<span class="n">sp</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"sp"</span><span class="p">],</span>
|
|
<span class="n">pct</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"pct"</span><span class="p">],</span>
|
|
<span class="n">fo</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"fo"</span><span class="p">],</span>
|
|
<span class="p">)</span>
|
|
|
|
<span class="k">for</span> <span class="n">record</span> <span class="ow">in</span> <span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"records"</span><span class="p">]:</span>
|
|
<span class="n">begin_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">record</span><span class="p">[</span><span class="s2">"interval_begin"</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
|
<span class="n">end_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">record</span><span class="p">[</span><span class="s2">"interval_end"</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
|
<span class="n">normalized_timespan</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"normalized_timespan"</span><span class="p">]</span>
|
|
|
|
<span class="k">if</span> <span class="n">monthly_indexes</span><span class="p">:</span>
|
|
<span class="n">index_date</span> <span class="o">=</span> <span class="n">begin_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m"</span><span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">index_date</span> <span class="o">=</span> <span class="n">begin_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2">"</span><span class="p">)</span>
|
|
<span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">]</span> <span class="o">=</span> <span class="n">begin_date</span>
|
|
<span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">]</span> <span class="o">=</span> <span class="n">end_date</span>
|
|
<span class="n">date_range</span> <span class="o">=</span> <span class="p">[</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">],</span> <span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">]]</span>
|
|
<span class="n">agg_doc</span> <span class="o">=</span> <span class="n">_AggregateReportDoc</span><span class="p">(</span>
|
|
<span class="n">xml_schema</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"xml_schema"</span><span class="p">],</span>
|
|
<span class="n">org_name</span><span class="o">=</span><span class="n">metadata</span><span class="p">[</span><span class="s2">"org_name"</span><span class="p">],</span>
|
|
<span class="n">org_email</span><span class="o">=</span><span class="n">metadata</span><span class="p">[</span><span class="s2">"org_email"</span><span class="p">],</span>
|
|
<span class="n">org_extra_contact_info</span><span class="o">=</span><span class="n">metadata</span><span class="p">[</span><span class="s2">"org_extra_contact_info"</span><span class="p">],</span>
|
|
<span class="n">report_id</span><span class="o">=</span><span class="n">metadata</span><span class="p">[</span><span class="s2">"report_id"</span><span class="p">],</span>
|
|
<span class="n">date_range</span><span class="o">=</span><span class="n">date_range</span><span class="p">,</span>
|
|
<span class="n">date_begin</span><span class="o">=</span><span class="n">begin_date</span><span class="p">,</span>
|
|
<span class="n">date_end</span><span class="o">=</span><span class="n">end_date</span><span class="p">,</span>
|
|
<span class="n">normalized_timespan</span><span class="o">=</span><span class="n">normalized_timespan</span><span class="p">,</span>
|
|
<span class="n">errors</span><span class="o">=</span><span class="n">metadata</span><span class="p">[</span><span class="s2">"errors"</span><span class="p">],</span>
|
|
<span class="n">published_policy</span><span class="o">=</span><span class="n">published_policy</span><span class="p">,</span>
|
|
<span class="n">source_ip_address</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"ip_address"</span><span class="p">],</span>
|
|
<span class="n">source_country</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"country"</span><span class="p">],</span>
|
|
<span class="n">source_reverse_dns</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"reverse_dns"</span><span class="p">],</span>
|
|
<span class="n">source_base_domain</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"base_domain"</span><span class="p">],</span>
|
|
<span class="n">source_type</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"type"</span><span class="p">],</span>
|
|
<span class="n">source_name</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"name"</span><span class="p">],</span>
|
|
<span class="n">message_count</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"count"</span><span class="p">],</span>
|
|
<span class="n">disposition</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"disposition"</span><span class="p">],</span>
|
|
<span class="n">dkim_aligned</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"dkim"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
|
|
<span class="ow">and</span> <span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"dkim"</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s2">"pass"</span><span class="p">,</span>
|
|
<span class="n">spf_aligned</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
|
|
<span class="ow">and</span> <span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s2">"pass"</span><span class="p">,</span>
|
|
<span class="n">header_from</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"header_from"</span><span class="p">],</span>
|
|
<span class="n">envelope_from</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"envelope_from"</span><span class="p">],</span>
|
|
<span class="n">envelope_to</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"envelope_to"</span><span class="p">],</span>
|
|
<span class="p">)</span>
|
|
|
|
<span class="k">for</span> <span class="n">override</span> <span class="ow">in</span> <span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"policy_override_reasons"</span><span class="p">]:</span>
|
|
<span class="n">agg_doc</span><span class="o">.</span><span class="n">add_policy_override</span><span class="p">(</span>
|
|
<span class="n">type_</span><span class="o">=</span><span class="n">override</span><span class="p">[</span><span class="s2">"type"</span><span class="p">],</span> <span class="n">comment</span><span class="o">=</span><span class="n">override</span><span class="p">[</span><span class="s2">"comment"</span><span class="p">]</span>
|
|
<span class="p">)</span>
|
|
|
|
<span class="k">for</span> <span class="n">dkim_result</span> <span class="ow">in</span> <span class="n">record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">][</span><span class="s2">"dkim"</span><span class="p">]:</span>
|
|
<span class="n">agg_doc</span><span class="o">.</span><span class="n">add_dkim_result</span><span class="p">(</span>
|
|
<span class="n">domain</span><span class="o">=</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">],</span>
|
|
<span class="n">selector</span><span class="o">=</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">"selector"</span><span class="p">],</span>
|
|
<span class="n">result</span><span class="o">=</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">],</span>
|
|
<span class="p">)</span>
|
|
|
|
<span class="k">for</span> <span class="n">spf_result</span> <span class="ow">in</span> <span class="n">record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">]:</span>
|
|
<span class="n">agg_doc</span><span class="o">.</span><span class="n">add_spf_result</span><span class="p">(</span>
|
|
<span class="n">domain</span><span class="o">=</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">],</span>
|
|
<span class="n">scope</span><span class="o">=</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">"scope"</span><span class="p">],</span>
|
|
<span class="n">result</span><span class="o">=</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">],</span>
|
|
<span class="p">)</span>
|
|
|
|
<span class="n">index</span> <span class="o">=</span> <span class="s2">"dmarc_aggregate"</span>
|
|
<span class="k">if</span> <span class="n">index_suffix</span><span class="p">:</span>
|
|
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">_</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_suffix</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">index_prefix</span><span class="p">:</span>
|
|
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">index</span><span class="p">)</span>
|
|
|
|
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">-</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_date</span><span class="p">)</span>
|
|
<span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
|
|
<span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span> <span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span>
|
|
<span class="p">)</span>
|
|
<span class="n">create_indexes</span><span class="p">([</span><span class="n">index</span><span class="p">],</span> <span class="n">index_settings</span><span class="p">)</span>
|
|
<span class="n">agg_doc</span><span class="o">.</span><span class="n">meta</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">index</span>
|
|
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">agg_doc</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
|
|
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="s2">"Elasticsearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
|
|
|
|
|
|
|
<div class="viewcode-block" id="save_forensic_report_to_elasticsearch">
|
|
<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.save_forensic_report_to_elasticsearch">[docs]</a>
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">save_forensic_report_to_elasticsearch</span><span class="p">(</span>
|
|
<span class="n">forensic_report</span><span class="p">:</span> <span class="n">OrderedDict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">],</span>
|
|
<span class="n">index_suffix</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">any</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">index_prefix</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">monthly_indexes</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
|
<span class="n">number_of_shards</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
|
|
<span class="n">number_of_replicas</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">0</span><span class="p">,</span>
|
|
<span class="p">):</span>
|
|
<span class="w"> </span><span class="sd">"""</span>
|
|
<span class="sd"> Saves a parsed DMARC forensic report to Elasticsearch</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> forensic_report (OrderedDict): A parsed forensic report</span>
|
|
<span class="sd"> index_suffix (str): The suffix of the name of the index to save to</span>
|
|
<span class="sd"> index_prefix (str): The prefix of the name of the index to save to</span>
|
|
<span class="sd"> monthly_indexes (bool): Use monthly indexes instead of daily</span>
|
|
<span class="sd"> indexes</span>
|
|
<span class="sd"> number_of_shards (int): The number of shards to use in the index</span>
|
|
<span class="sd"> number_of_replicas (int): The number of replicas to use in the</span>
|
|
<span class="sd"> index</span>
|
|
|
|
<span class="sd"> Raises:</span>
|
|
<span class="sd"> AlreadySaved</span>
|
|
|
|
<span class="sd"> """</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Saving forensic report to Elasticsearch"</span><span class="p">)</span>
|
|
<span class="n">forensic_report</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
|
<span class="n">sample_date</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="k">if</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"date"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">sample_date</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"date"</span><span class="p">]</span>
|
|
<span class="n">sample_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">sample_date</span><span class="p">)</span>
|
|
<span class="n">original_headers</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"headers"</span><span class="p">]</span>
|
|
<span class="n">headers</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">()</span>
|
|
<span class="k">for</span> <span class="n">original_header</span> <span class="ow">in</span> <span class="n">original_headers</span><span class="p">:</span>
|
|
<span class="n">headers</span><span class="p">[</span><span class="n">original_header</span><span class="o">.</span><span class="n">lower</span><span class="p">()]</span> <span class="o">=</span> <span class="n">original_headers</span><span class="p">[</span><span class="n">original_header</span><span class="p">]</span>
|
|
|
|
<span class="n">arrival_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"arrival_date_utc"</span><span class="p">])</span>
|
|
<span class="n">arrival_date_epoch_milliseconds</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">arrival_date</span><span class="o">.</span><span class="n">timestamp</span><span class="p">()</span> <span class="o">*</span> <span class="mi">1000</span><span class="p">)</span>
|
|
|
|
<span class="k">if</span> <span class="n">index_suffix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"dmarc_forensic_</span><span class="si">{0}</span><span class="s2">*"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_suffix</span><span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"dmarc_forensic*"</span>
|
|
<span class="k">if</span> <span class="n">index_prefix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">search_index</span><span class="p">)</span>
|
|
<span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="n">search_index</span><span class="p">)</span>
|
|
<span class="n">q</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">arrival_date</span><span class="o">=</span><span class="n">arrival_date_epoch_milliseconds</span><span class="p">)))</span>
|
|
|
|
<span class="n">from_</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="n">to_</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="n">subject</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="k">if</span> <span class="s2">"from"</span> <span class="ow">in</span> <span class="n">headers</span><span class="p">:</span>
|
|
<span class="c1"># We convert the FROM header from a string list to a flat string.</span>
|
|
<span class="n">headers</span><span class="p">[</span><span class="s2">"from"</span><span class="p">]</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"from"</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span>
|
|
<span class="k">if</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"from"</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="s2">""</span><span class="p">:</span>
|
|
<span class="n">headers</span><span class="p">[</span><span class="s2">"from"</span><span class="p">]</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"from"</span><span class="p">][</span><span class="mi">1</span><span class="p">]</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">headers</span><span class="p">[</span><span class="s2">"from"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">" <"</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">headers</span><span class="p">[</span><span class="s2">"from"</span><span class="p">])</span> <span class="o">+</span> <span class="s2">">"</span>
|
|
|
|
<span class="n">from_</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
|
|
<span class="n">from_</span><span class="p">[</span><span class="s2">"sample.headers.from"</span><span class="p">]</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"from"</span><span class="p">]</span>
|
|
<span class="n">from_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match_phrase</span><span class="o">=</span><span class="n">from_</span><span class="p">))</span>
|
|
<span class="n">q</span> <span class="o">=</span> <span class="n">q</span> <span class="o">&</span> <span class="n">from_query</span>
|
|
<span class="k">if</span> <span class="s2">"to"</span> <span class="ow">in</span> <span class="n">headers</span><span class="p">:</span>
|
|
<span class="c1"># We convert the TO header from a string list to a flat string.</span>
|
|
<span class="n">headers</span><span class="p">[</span><span class="s2">"to"</span><span class="p">]</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"to"</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span>
|
|
<span class="k">if</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"to"</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="s2">""</span><span class="p">:</span>
|
|
<span class="n">headers</span><span class="p">[</span><span class="s2">"to"</span><span class="p">]</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"to"</span><span class="p">][</span><span class="mi">1</span><span class="p">]</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">headers</span><span class="p">[</span><span class="s2">"to"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">" <"</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">headers</span><span class="p">[</span><span class="s2">"to"</span><span class="p">])</span> <span class="o">+</span> <span class="s2">">"</span>
|
|
|
|
<span class="n">to_</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
|
|
<span class="n">to_</span><span class="p">[</span><span class="s2">"sample.headers.to"</span><span class="p">]</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"to"</span><span class="p">]</span>
|
|
<span class="n">to_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match_phrase</span><span class="o">=</span><span class="n">to_</span><span class="p">))</span>
|
|
<span class="n">q</span> <span class="o">=</span> <span class="n">q</span> <span class="o">&</span> <span class="n">to_query</span>
|
|
<span class="k">if</span> <span class="s2">"subject"</span> <span class="ow">in</span> <span class="n">headers</span><span class="p">:</span>
|
|
<span class="n">subject</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"subject"</span><span class="p">]</span>
|
|
<span class="n">subject_query</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"match_phrase"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"sample.headers.subject"</span><span class="p">:</span> <span class="n">subject</span><span class="p">}}</span>
|
|
<span class="n">q</span> <span class="o">=</span> <span class="n">q</span> <span class="o">&</span> <span class="n">Q</span><span class="p">(</span><span class="n">subject_query</span><span class="p">)</span>
|
|
|
|
<span class="n">search</span><span class="o">.</span><span class="n">query</span> <span class="o">=</span> <span class="n">q</span>
|
|
<span class="n">existing</span> <span class="o">=</span> <span class="n">search</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
|
|
|
|
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">existing</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">AlreadySaved</span><span class="p">(</span>
|
|
<span class="s2">"A forensic sample to </span><span class="si">{0}</span><span class="s2"> from </span><span class="si">{1}</span><span class="s2"> "</span>
|
|
<span class="s2">"with a subject of </span><span class="si">{2}</span><span class="s2"> and arrival date of </span><span class="si">{3}</span><span class="s2"> "</span>
|
|
<span class="s2">"already exists in "</span>
|
|
<span class="s2">"Elasticsearch"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
|
<span class="n">to_</span><span class="p">,</span> <span class="n">from_</span><span class="p">,</span> <span class="n">subject</span><span class="p">,</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"arrival_date_utc"</span><span class="p">]</span>
|
|
<span class="p">)</span>
|
|
<span class="p">)</span>
|
|
|
|
<span class="n">parsed_sample</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">]</span>
|
|
<span class="n">sample</span> <span class="o">=</span> <span class="n">_ForensicSampleDoc</span><span class="p">(</span>
|
|
<span class="n">raw</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"sample"</span><span class="p">],</span>
|
|
<span class="n">headers</span><span class="o">=</span><span class="n">headers</span><span class="p">,</span>
|
|
<span class="n">headers_only</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"sample_headers_only"</span><span class="p">],</span>
|
|
<span class="n">date</span><span class="o">=</span><span class="n">sample_date</span><span class="p">,</span>
|
|
<span class="n">subject</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"subject"</span><span class="p">],</span>
|
|
<span class="n">filename_safe_subject</span><span class="o">=</span><span class="n">parsed_sample</span><span class="p">[</span><span class="s2">"filename_safe_subject"</span><span class="p">],</span>
|
|
<span class="n">body</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"body"</span><span class="p">],</span>
|
|
<span class="p">)</span>
|
|
|
|
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"to"</span><span class="p">]:</span>
|
|
<span class="n">sample</span><span class="o">.</span><span class="n">add_to</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">])</span>
|
|
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"reply_to"</span><span class="p">]:</span>
|
|
<span class="n">sample</span><span class="o">.</span><span class="n">add_reply_to</span><span class="p">(</span>
|
|
<span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">]</span>
|
|
<span class="p">)</span>
|
|
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"cc"</span><span class="p">]:</span>
|
|
<span class="n">sample</span><span class="o">.</span><span class="n">add_cc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">])</span>
|
|
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"bcc"</span><span class="p">]:</span>
|
|
<span class="n">sample</span><span class="o">.</span><span class="n">add_bcc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">])</span>
|
|
<span class="k">for</span> <span class="n">attachment</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"attachments"</span><span class="p">]:</span>
|
|
<span class="n">sample</span><span class="o">.</span><span class="n">add_attachment</span><span class="p">(</span>
|
|
<span class="n">filename</span><span class="o">=</span><span class="n">attachment</span><span class="p">[</span><span class="s2">"filename"</span><span class="p">],</span>
|
|
<span class="n">content_type</span><span class="o">=</span><span class="n">attachment</span><span class="p">[</span><span class="s2">"mail_content_type"</span><span class="p">],</span>
|
|
<span class="n">sha256</span><span class="o">=</span><span class="n">attachment</span><span class="p">[</span><span class="s2">"sha256"</span><span class="p">],</span>
|
|
<span class="p">)</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">forensic_doc</span> <span class="o">=</span> <span class="n">_ForensicReportDoc</span><span class="p">(</span>
|
|
<span class="n">feedback_type</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"feedback_type"</span><span class="p">],</span>
|
|
<span class="n">user_agent</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"user_agent"</span><span class="p">],</span>
|
|
<span class="n">version</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"version"</span><span class="p">],</span>
|
|
<span class="n">original_mail_from</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"original_mail_from"</span><span class="p">],</span>
|
|
<span class="n">arrival_date</span><span class="o">=</span><span class="n">arrival_date_epoch_milliseconds</span><span class="p">,</span>
|
|
<span class="n">domain</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"reported_domain"</span><span class="p">],</span>
|
|
<span class="n">original_envelope_id</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"original_envelope_id"</span><span class="p">],</span>
|
|
<span class="n">authentication_results</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"authentication_results"</span><span class="p">],</span>
|
|
<span class="n">delivery_results</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"delivery_result"</span><span class="p">],</span>
|
|
<span class="n">source_ip_address</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"ip_address"</span><span class="p">],</span>
|
|
<span class="n">source_country</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"country"</span><span class="p">],</span>
|
|
<span class="n">source_reverse_dns</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"reverse_dns"</span><span class="p">],</span>
|
|
<span class="n">source_base_domain</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"base_domain"</span><span class="p">],</span>
|
|
<span class="n">authentication_mechanisms</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"authentication_mechanisms"</span><span class="p">],</span>
|
|
<span class="n">auth_failure</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"auth_failure"</span><span class="p">],</span>
|
|
<span class="n">dkim_domain</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"dkim_domain"</span><span class="p">],</span>
|
|
<span class="n">original_rcpt_to</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"original_rcpt_to"</span><span class="p">],</span>
|
|
<span class="n">sample</span><span class="o">=</span><span class="n">sample</span><span class="p">,</span>
|
|
<span class="p">)</span>
|
|
|
|
<span class="n">index</span> <span class="o">=</span> <span class="s2">"dmarc_forensic"</span>
|
|
<span class="k">if</span> <span class="n">index_suffix</span><span class="p">:</span>
|
|
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">_</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_suffix</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">index_prefix</span><span class="p">:</span>
|
|
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">index</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">monthly_indexes</span><span class="p">:</span>
|
|
<span class="n">index_date</span> <span class="o">=</span> <span class="n">arrival_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m"</span><span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">index_date</span> <span class="o">=</span> <span class="n">arrival_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2">"</span><span class="p">)</span>
|
|
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">-</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_date</span><span class="p">)</span>
|
|
<span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
|
|
<span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span> <span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span>
|
|
<span class="p">)</span>
|
|
<span class="n">create_indexes</span><span class="p">([</span><span class="n">index</span><span class="p">],</span> <span class="n">index_settings</span><span class="p">)</span>
|
|
<span class="n">forensic_doc</span><span class="o">.</span><span class="n">meta</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">index</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">forensic_doc</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
|
|
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="s2">"Elasticsearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
|
<span class="k">except</span> <span class="ne">KeyError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">InvalidForensicReport</span><span class="p">(</span>
|
|
<span class="s2">"Forensic report missing required field: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
|
<span class="p">)</span></div>
|
|
|
|
|
|
|
|
<div class="viewcode-block" id="save_smtp_tls_report_to_elasticsearch">
|
|
<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.save_smtp_tls_report_to_elasticsearch">[docs]</a>
|
|
<span class="k">def</span><span class="w"> </span><span class="nf">save_smtp_tls_report_to_elasticsearch</span><span class="p">(</span>
|
|
<span class="n">report</span><span class="p">:</span> <span class="n">OrderedDict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">],</span>
|
|
<span class="n">index_suffix</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">index_prefix</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
|
<span class="n">monthly_indexes</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
|
<span class="n">number_of_shards</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
|
|
<span class="n">number_of_replicas</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="mi">0</span><span class="p">,</span>
|
|
<span class="p">):</span>
|
|
<span class="w"> </span><span class="sd">"""</span>
|
|
<span class="sd"> Saves a parsed SMTP TLS report to Elasticsearch</span>
|
|
|
|
<span class="sd"> Args:</span>
|
|
<span class="sd"> report (OrderedDict): A parsed SMTP TLS report</span>
|
|
<span class="sd"> index_suffix (str): The suffix of the name of the index to save to</span>
|
|
<span class="sd"> index_prefix (str): The prefix of the name of the index to save to</span>
|
|
<span class="sd"> monthly_indexes (bool): Use monthly indexes instead of daily indexes</span>
|
|
<span class="sd"> number_of_shards (int): The number of shards to use in the index</span>
|
|
<span class="sd"> number_of_replicas (int): The number of replicas to use in the index</span>
|
|
|
|
<span class="sd"> Raises:</span>
|
|
<span class="sd"> AlreadySaved</span>
|
|
<span class="sd"> """</span>
|
|
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Saving smtp tls report to Elasticsearch"</span><span class="p">)</span>
|
|
<span class="n">org_name</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"organization_name"</span><span class="p">]</span>
|
|
<span class="n">report_id</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_id"</span><span class="p">]</span>
|
|
<span class="n">begin_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
|
<span class="n">end_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
|
<span class="n">begin_date_human</span> <span class="o">=</span> <span class="n">begin_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%SZ"</span><span class="p">)</span>
|
|
<span class="n">end_date_human</span> <span class="o">=</span> <span class="n">end_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%SZ"</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">monthly_indexes</span><span class="p">:</span>
|
|
<span class="n">index_date</span> <span class="o">=</span> <span class="n">begin_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m"</span><span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">index_date</span> <span class="o">=</span> <span class="n">begin_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2">"</span><span class="p">)</span>
|
|
<span class="n">report</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">]</span> <span class="o">=</span> <span class="n">begin_date</span>
|
|
<span class="n">report</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">]</span> <span class="o">=</span> <span class="n">end_date</span>
|
|
|
|
<span class="n">org_name_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match_phrase</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">org_name</span><span class="o">=</span><span class="n">org_name</span><span class="p">)))</span>
|
|
<span class="n">report_id_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match_phrase</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">report_id</span><span class="o">=</span><span class="n">report_id</span><span class="p">)))</span>
|
|
<span class="n">begin_date_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">date_begin</span><span class="o">=</span><span class="n">begin_date</span><span class="p">)))</span>
|
|
<span class="n">end_date_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">date_end</span><span class="o">=</span><span class="n">end_date</span><span class="p">)))</span>
|
|
|
|
<span class="k">if</span> <span class="n">index_suffix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"smtp_tls_</span><span class="si">{0}</span><span class="s2">*"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_suffix</span><span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"smtp_tls*"</span>
|
|
<span class="k">if</span> <span class="n">index_prefix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
|
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">search_index</span><span class="p">)</span>
|
|
<span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="n">search_index</span><span class="p">)</span>
|
|
<span class="n">query</span> <span class="o">=</span> <span class="n">org_name_query</span> <span class="o">&</span> <span class="n">report_id_query</span>
|
|
<span class="n">query</span> <span class="o">=</span> <span class="n">query</span> <span class="o">&</span> <span class="n">begin_date_query</span> <span class="o">&</span> <span class="n">end_date_query</span>
|
|
<span class="n">search</span><span class="o">.</span><span class="n">query</span> <span class="o">=</span> <span class="n">query</span>
|
|
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">existing</span> <span class="o">=</span> <span class="n">search</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
|
|
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error_</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span>
|
|
<span class="s2">"Elasticsearch's search for existing report </span><span class="se">\</span>
|
|
<span class="s2"> error: </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error_</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
|
<span class="p">)</span>
|
|
|
|
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">existing</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">AlreadySaved</span><span class="p">(</span>
|
|
<span class="sa">f</span><span class="s2">"An SMTP TLS report ID </span><span class="si">{</span><span class="n">report_id</span><span class="si">}</span><span class="s2"> from "</span>
|
|
<span class="sa">f</span><span class="s2">" </span><span class="si">{</span><span class="n">org_name</span><span class="si">}</span><span class="s2"> with a date range of "</span>
|
|
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">begin_date_human</span><span class="si">}</span><span class="s2"> UTC to "</span>
|
|
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">end_date_human</span><span class="si">}</span><span class="s2"> UTC already "</span>
|
|
<span class="s2">"exists in Elasticsearch"</span>
|
|
<span class="p">)</span>
|
|
|
|
<span class="n">index</span> <span class="o">=</span> <span class="s2">"smtp_tls"</span>
|
|
<span class="k">if</span> <span class="n">index_suffix</span><span class="p">:</span>
|
|
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">_</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_suffix</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">index_prefix</span><span class="p">:</span>
|
|
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">index</span><span class="p">)</span>
|
|
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">-</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_date</span><span class="p">)</span>
|
|
<span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
|
|
<span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span> <span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span>
|
|
<span class="p">)</span>
|
|
|
|
<span class="n">smtp_tls_doc</span> <span class="o">=</span> <span class="n">_SMTPTLSReportDoc</span><span class="p">(</span>
|
|
<span class="n">org_name</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">"organization_name"</span><span class="p">],</span>
|
|
<span class="n">date_range</span><span class="o">=</span><span class="p">[</span><span class="n">report</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">],</span> <span class="n">report</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">]],</span>
|
|
<span class="n">date_begin</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">],</span>
|
|
<span class="n">date_end</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">],</span>
|
|
<span class="n">contact_info</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">"contact_info"</span><span class="p">],</span>
|
|
<span class="n">report_id</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">"report_id"</span><span class="p">],</span>
|
|
<span class="p">)</span>
|
|
|
|
<span class="k">for</span> <span class="n">policy</span> <span class="ow">in</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policies"</span><span class="p">]:</span>
|
|
<span class="n">policy_strings</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="n">mx_host_patterns</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="k">if</span> <span class="s2">"policy_strings"</span> <span class="ow">in</span> <span class="n">policy</span><span class="p">:</span>
|
|
<span class="n">policy_strings</span> <span class="o">=</span> <span class="n">policy</span><span class="p">[</span><span class="s2">"policy_strings"</span><span class="p">]</span>
|
|
<span class="k">if</span> <span class="s2">"mx_host_patterns"</span> <span class="ow">in</span> <span class="n">policy</span><span class="p">:</span>
|
|
<span class="n">mx_host_patterns</span> <span class="o">=</span> <span class="n">policy</span><span class="p">[</span><span class="s2">"mx_host_patterns"</span><span class="p">]</span>
|
|
<span class="n">policy_doc</span> <span class="o">=</span> <span class="n">_SMTPTLSPolicyDoc</span><span class="p">(</span>
|
|
<span class="n">policy_domain</span><span class="o">=</span><span class="n">policy</span><span class="p">[</span><span class="s2">"policy_domain"</span><span class="p">],</span>
|
|
<span class="n">policy_type</span><span class="o">=</span><span class="n">policy</span><span class="p">[</span><span class="s2">"policy_type"</span><span class="p">],</span>
|
|
<span class="n">successful_session_count</span><span class="o">=</span><span class="n">policy</span><span class="p">[</span><span class="s2">"successful_session_count"</span><span class="p">],</span>
|
|
<span class="n">failed_session_count</span><span class="o">=</span><span class="n">policy</span><span class="p">[</span><span class="s2">"failed_session_count"</span><span class="p">],</span>
|
|
<span class="n">policy_string</span><span class="o">=</span><span class="n">policy_strings</span><span class="p">,</span>
|
|
<span class="n">mx_host_patterns</span><span class="o">=</span><span class="n">mx_host_patterns</span><span class="p">,</span>
|
|
<span class="p">)</span>
|
|
<span class="k">if</span> <span class="s2">"failure_details"</span> <span class="ow">in</span> <span class="n">policy</span><span class="p">:</span>
|
|
<span class="k">for</span> <span class="n">failure_detail</span> <span class="ow">in</span> <span class="n">policy</span><span class="p">[</span><span class="s2">"failure_details"</span><span class="p">]:</span>
|
|
<span class="n">receiving_mx_hostname</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="n">additional_information_uri</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="n">failure_reason_code</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="n">ip_address</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="n">receiving_ip</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="n">receiving_mx_helo</span> <span class="o">=</span> <span class="kc">None</span>
|
|
<span class="n">sending_mta_ip</span> <span class="o">=</span> <span class="kc">None</span>
|
|
|
|
<span class="k">if</span> <span class="s2">"receiving_mx_hostname"</span> <span class="ow">in</span> <span class="n">failure_detail</span><span class="p">:</span>
|
|
<span class="n">receiving_mx_hostname</span> <span class="o">=</span> <span class="n">failure_detail</span><span class="p">[</span><span class="s2">"receiving_mx_hostname"</span><span class="p">]</span>
|
|
<span class="k">if</span> <span class="s2">"additional_information_uri"</span> <span class="ow">in</span> <span class="n">failure_detail</span><span class="p">:</span>
|
|
<span class="n">additional_information_uri</span> <span class="o">=</span> <span class="n">failure_detail</span><span class="p">[</span>
|
|
<span class="s2">"additional_information_uri"</span>
|
|
<span class="p">]</span>
|
|
<span class="k">if</span> <span class="s2">"failure_reason_code"</span> <span class="ow">in</span> <span class="n">failure_detail</span><span class="p">:</span>
|
|
<span class="n">failure_reason_code</span> <span class="o">=</span> <span class="n">failure_detail</span><span class="p">[</span><span class="s2">"failure_reason_code"</span><span class="p">]</span>
|
|
<span class="k">if</span> <span class="s2">"ip_address"</span> <span class="ow">in</span> <span class="n">failure_detail</span><span class="p">:</span>
|
|
<span class="n">ip_address</span> <span class="o">=</span> <span class="n">failure_detail</span><span class="p">[</span><span class="s2">"ip_address"</span><span class="p">]</span>
|
|
<span class="k">if</span> <span class="s2">"receiving_ip"</span> <span class="ow">in</span> <span class="n">failure_detail</span><span class="p">:</span>
|
|
<span class="n">receiving_ip</span> <span class="o">=</span> <span class="n">failure_detail</span><span class="p">[</span><span class="s2">"receiving_ip"</span><span class="p">]</span>
|
|
<span class="k">if</span> <span class="s2">"receiving_mx_helo"</span> <span class="ow">in</span> <span class="n">failure_detail</span><span class="p">:</span>
|
|
<span class="n">receiving_mx_helo</span> <span class="o">=</span> <span class="n">failure_detail</span><span class="p">[</span><span class="s2">"receiving_mx_helo"</span><span class="p">]</span>
|
|
<span class="k">if</span> <span class="s2">"sending_mta_ip"</span> <span class="ow">in</span> <span class="n">failure_detail</span><span class="p">:</span>
|
|
<span class="n">sending_mta_ip</span> <span class="o">=</span> <span class="n">failure_detail</span><span class="p">[</span><span class="s2">"sending_mta_ip"</span><span class="p">]</span>
|
|
<span class="n">policy_doc</span><span class="o">.</span><span class="n">add_failure_details</span><span class="p">(</span>
|
|
<span class="n">result_type</span><span class="o">=</span><span class="n">failure_detail</span><span class="p">[</span><span class="s2">"result_type"</span><span class="p">],</span>
|
|
<span class="n">ip_address</span><span class="o">=</span><span class="n">ip_address</span><span class="p">,</span>
|
|
<span class="n">receiving_ip</span><span class="o">=</span><span class="n">receiving_ip</span><span class="p">,</span>
|
|
<span class="n">receiving_mx_helo</span><span class="o">=</span><span class="n">receiving_mx_helo</span><span class="p">,</span>
|
|
<span class="n">failed_session_count</span><span class="o">=</span><span class="n">failure_detail</span><span class="p">[</span><span class="s2">"failed_session_count"</span><span class="p">],</span>
|
|
<span class="n">sending_mta_ip</span><span class="o">=</span><span class="n">sending_mta_ip</span><span class="p">,</span>
|
|
<span class="n">receiving_mx_hostname</span><span class="o">=</span><span class="n">receiving_mx_hostname</span><span class="p">,</span>
|
|
<span class="n">additional_information_uri</span><span class="o">=</span><span class="n">additional_information_uri</span><span class="p">,</span>
|
|
<span class="n">failure_reason_code</span><span class="o">=</span><span class="n">failure_reason_code</span><span class="p">,</span>
|
|
<span class="p">)</span>
|
|
<span class="n">smtp_tls_doc</span><span class="o">.</span><span class="n">policies</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">policy_doc</span><span class="p">)</span>
|
|
|
|
<span class="n">create_indexes</span><span class="p">([</span><span class="n">index</span><span class="p">],</span> <span class="n">index_settings</span><span class="p">)</span>
|
|
<span class="n">smtp_tls_doc</span><span class="o">.</span><span class="n">meta</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">index</span>
|
|
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">smtp_tls_doc</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
|
|
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="s2">"Elasticsearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
|
|
|
</pre></div>
|
|
|
|
</div>
|
|
</div>
|
|
<footer>
|
|
|
|
<hr/>
|
|
|
|
<div role="contentinfo">
|
|
<p>© Copyright 2018 - 2025, Sean Whalen and contributors.</p>
|
|
</div>
|
|
|
|
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
|
|
<a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
|
|
provided by <a href="https://readthedocs.org">Read the Docs</a>.
|
|
|
|
|
|
</footer>
|
|
</div>
|
|
</div>
|
|
</section>
|
|
</div>
|
|
<script>
|
|
jQuery(function () {
|
|
SphinxRtdTheme.Navigation.enable(true);
|
|
});
|
|
</script>
|
|
|
|
</body>
|
|
</html> |