mirror of
https://github.com/domainaware/parsedmarc.git
synced 2026-06-03 09:29:43 +00:00
Update docs
This commit is contained in:
Sean Whalen
+11
-17
@@ -1,23 +1,20 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="../">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Overview: module code — parsedmarc 8.15.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../_static/css/theme.css" />
|
||||
<title>Overview: module code — parsedmarc 8.15.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=80d5e7a1" />
|
||||
<link rel="stylesheet" type="text/css" href="../_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="../_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js"></script>
|
||||
<script src="../_static/jquery.js"></script>
|
||||
<script src="../_static/underscore.js"></script>
|
||||
<script src="../_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="../_static/doctools.js"></script>
|
||||
<script src="../_static/sphinx_highlight.js"></script>
|
||||
<script src="../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../_static/documentation_options.js?v=6add7570"></script>
|
||||
<script src="../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="../_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="../genindex.html" />
|
||||
<link rel="search" title="Search" href="../search.html" />
|
||||
@@ -34,9 +31,6 @@
|
||||
<a href="../index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.15.0
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
|
||||
+630
-399
File diff suppressed because it is too large
Load Diff
+253
-209
@@ -1,23 +1,20 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="../../">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc.elastic — parsedmarc 8.15.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css" />
|
||||
<title>parsedmarc.elastic — parsedmarc 8.15.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=80d5e7a1" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="../../_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="../../" id="documentation_options" src="../../_static/documentation_options.js"></script>
|
||||
<script src="../../_static/jquery.js"></script>
|
||||
<script src="../../_static/underscore.js"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="../../_static/doctools.js"></script>
|
||||
<script src="../../_static/sphinx_highlight.js"></script>
|
||||
<script src="../../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../../_static/documentation_options.js?v=6add7570"></script>
|
||||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="../../_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="../../genindex.html" />
|
||||
<link rel="search" title="Search" href="../../search.html" />
|
||||
@@ -34,9 +31,6 @@
|
||||
<a href="../../index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.15.0
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
@@ -91,8 +85,20 @@
|
||||
<span class="kn">from</span> <span class="nn">collections</span> <span class="kn">import</span> <span class="n">OrderedDict</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">elasticsearch_dsl.search</span> <span class="kn">import</span> <span class="n">Q</span>
|
||||
<span class="kn">from</span> <span class="nn">elasticsearch_dsl</span> <span class="kn">import</span> <span class="n">connections</span><span class="p">,</span> <span class="n">Object</span><span class="p">,</span> <span class="n">Document</span><span class="p">,</span> <span class="n">Index</span><span class="p">,</span> <span class="n">Nested</span><span class="p">,</span> \
|
||||
<span class="n">InnerDoc</span><span class="p">,</span> <span class="n">Integer</span><span class="p">,</span> <span class="n">Text</span><span class="p">,</span> <span class="n">Boolean</span><span class="p">,</span> <span class="n">Ip</span><span class="p">,</span> <span class="n">Date</span><span class="p">,</span> <span class="n">Search</span>
|
||||
<span class="kn">from</span> <span class="nn">elasticsearch_dsl</span> <span class="kn">import</span> <span class="p">(</span>
|
||||
<span class="n">connections</span><span class="p">,</span>
|
||||
<span class="n">Object</span><span class="p">,</span>
|
||||
<span class="n">Document</span><span class="p">,</span>
|
||||
<span class="n">Index</span><span class="p">,</span>
|
||||
<span class="n">Nested</span><span class="p">,</span>
|
||||
<span class="n">InnerDoc</span><span class="p">,</span>
|
||||
<span class="n">Integer</span><span class="p">,</span>
|
||||
<span class="n">Text</span><span class="p">,</span>
|
||||
<span class="n">Boolean</span><span class="p">,</span>
|
||||
<span class="n">Ip</span><span class="p">,</span>
|
||||
<span class="n">Date</span><span class="p">,</span>
|
||||
<span class="n">Search</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="kn">from</span> <span class="nn">elasticsearch.helpers</span> <span class="kn">import</span> <span class="n">reindex</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">parsedmarc.log</span> <span class="kn">import</span> <span class="n">logger</span>
|
||||
@@ -100,10 +106,13 @@
|
||||
<span class="kn">from</span> <span class="nn">parsedmarc</span> <span class="kn">import</span> <span class="n">InvalidForensicReport</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="ElasticsearchError"><a class="viewcode-back" href="../../api.html#parsedmarc.elastic.ElasticsearchError">[docs]</a><span class="k">class</span> <span class="nc">ElasticsearchError</span><span class="p">(</span><span class="ne">Exception</span><span class="p">):</span>
|
||||
<div class="viewcode-block" id="ElasticsearchError">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.ElasticsearchError">[docs]</a>
|
||||
<span class="k">class</span> <span class="nc">ElasticsearchError</span><span class="p">(</span><span class="ne">Exception</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Raised when an Elasticsearch error occurs"""</span></div>
|
||||
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_PolicyOverride</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="nb">type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">comment</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
@@ -164,24 +173,21 @@
|
||||
<span class="n">spf_results</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SPFResult</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_policy_override</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">type_</span><span class="p">,</span> <span class="n">comment</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">policy_overrides</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_PolicyOverride</span><span class="p">(</span><span class="nb">type</span><span class="o">=</span><span class="n">type_</span><span class="p">,</span>
|
||||
<span class="n">comment</span><span class="o">=</span><span class="n">comment</span><span class="p">))</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">policy_overrides</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_PolicyOverride</span><span class="p">(</span><span class="nb">type</span><span class="o">=</span><span class="n">type_</span><span class="p">,</span> <span class="n">comment</span><span class="o">=</span><span class="n">comment</span><span class="p">))</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_dkim_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">domain</span><span class="p">,</span> <span class="n">selector</span><span class="p">,</span> <span class="n">result</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">dkim_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_DKIMResult</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span>
|
||||
<span class="n">selector</span><span class="o">=</span><span class="n">selector</span><span class="p">,</span>
|
||||
<span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">))</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">dkim_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="n">_DKIMResult</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span> <span class="n">selector</span><span class="o">=</span><span class="n">selector</span><span class="p">,</span> <span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_spf_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">domain</span><span class="p">,</span> <span class="n">scope</span><span class="p">,</span> <span class="n">result</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">spf_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_SPFResult</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span>
|
||||
<span class="n">scope</span><span class="o">=</span><span class="n">scope</span><span class="p">,</span>
|
||||
<span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">))</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">spf_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_SPFResult</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span> <span class="n">scope</span><span class="o">=</span><span class="n">scope</span><span class="p">,</span> <span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">))</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">save</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">**</span> <span class="n">kwargs</span><span class="p">):</span>
|
||||
<span class="k">def</span> <span class="nf">save</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">passed_dmarc</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">passed_dmarc</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">spf_aligned</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">dkim_aligned</span>
|
||||
|
||||
<span class="k">return</span> <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="o">**</span> <span class="n">kwargs</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_EmailAddressDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
@@ -211,24 +217,25 @@
|
||||
<span class="n">attachments</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_EmailAttachmentDoc</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_to</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">to</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span>
|
||||
<span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">to</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_reply_to</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">reply_to</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span>
|
||||
<span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">reply_to</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_cc</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">cc</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span>
|
||||
<span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">cc</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_bcc</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">bcc</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span>
|
||||
<span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">bcc</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_attachment</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">filename</span><span class="p">,</span> <span class="n">content_type</span><span class="p">,</span> <span class="n">sha256</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">attachments</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAttachmentDoc</span><span class="p">(</span><span class="n">filename</span><span class="o">=</span><span class="n">filename</span><span class="p">,</span>
|
||||
<span class="n">content_type</span><span class="o">=</span><span class="n">content_type</span><span class="p">,</span> <span class="n">sha256</span><span class="o">=</span><span class="n">sha256</span><span class="p">))</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">attachments</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="n">_EmailAttachmentDoc</span><span class="p">(</span>
|
||||
<span class="n">filename</span><span class="o">=</span><span class="n">filename</span><span class="p">,</span> <span class="n">content_type</span><span class="o">=</span><span class="n">content_type</span><span class="p">,</span> <span class="n">sha256</span><span class="o">=</span><span class="n">sha256</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_ForensicReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
@@ -273,14 +280,18 @@
|
||||
<span class="n">failed_session_count</span> <span class="o">=</span> <span class="n">Integer</span><span class="p">()</span>
|
||||
<span class="n">failure_details</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SMTPTLSFailureDetailsDoc</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_failure_details</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">result_type</span><span class="p">,</span> <span class="n">ip_address</span><span class="p">,</span>
|
||||
<span class="n">receiving_ip</span><span class="p">,</span>
|
||||
<span class="n">receiving_mx_helo</span><span class="p">,</span>
|
||||
<span class="n">failed_session_count</span><span class="p">,</span>
|
||||
<span class="n">sending_mta_ip</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">receiving_mx_hostname</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">additional_information_uri</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">failure_reason_code</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
<span class="k">def</span> <span class="nf">add_failure_details</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="p">,</span>
|
||||
<span class="n">result_type</span><span class="p">,</span>
|
||||
<span class="n">ip_address</span><span class="p">,</span>
|
||||
<span class="n">receiving_ip</span><span class="p">,</span>
|
||||
<span class="n">receiving_mx_helo</span><span class="p">,</span>
|
||||
<span class="n">failed_session_count</span><span class="p">,</span>
|
||||
<span class="n">sending_mta_ip</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">receiving_mx_hostname</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">additional_information_uri</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">failure_reason_code</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="n">_details</span> <span class="o">=</span> <span class="n">_SMTPTLSFailureDetailsDoc</span><span class="p">(</span>
|
||||
<span class="n">result_type</span><span class="o">=</span><span class="n">result_type</span><span class="p">,</span>
|
||||
<span class="n">ip_address</span><span class="o">=</span><span class="n">ip_address</span><span class="p">,</span>
|
||||
@@ -290,13 +301,12 @@
|
||||
<span class="n">receiving_ip</span><span class="o">=</span><span class="n">receiving_ip</span><span class="p">,</span>
|
||||
<span class="n">failed_session_count</span><span class="o">=</span><span class="n">failed_session_count</span><span class="p">,</span>
|
||||
<span class="n">additional_information</span><span class="o">=</span><span class="n">additional_information_uri</span><span class="p">,</span>
|
||||
<span class="n">failure_reason_code</span><span class="o">=</span><span class="n">failure_reason_code</span>
|
||||
<span class="n">failure_reason_code</span><span class="o">=</span><span class="n">failure_reason_code</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">failure_details</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_details</span><span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_SMTPTLSReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
|
||||
<span class="k">class</span> <span class="nc">Index</span><span class="p">:</span>
|
||||
<span class="n">name</span> <span class="o">=</span> <span class="s2">"smtp_tls"</span>
|
||||
|
||||
@@ -308,27 +318,45 @@
|
||||
<span class="n">report_id</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">policies</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SMTPTLSPolicyDoc</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_policy</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">policy_type</span><span class="p">,</span> <span class="n">policy_domain</span><span class="p">,</span>
|
||||
<span class="n">successful_session_count</span><span class="p">,</span>
|
||||
<span class="n">failed_session_count</span><span class="p">,</span>
|
||||
<span class="n">policy_string</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">mx_host_patterns</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">failure_details</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">policies</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">policy_type</span><span class="o">=</span><span class="n">policy_type</span><span class="p">,</span>
|
||||
<span class="n">policy_domain</span><span class="o">=</span><span class="n">policy_domain</span><span class="p">,</span>
|
||||
<span class="n">successful_session_count</span><span class="o">=</span><span class="n">successful_session_count</span><span class="p">,</span>
|
||||
<span class="n">failed_session_count</span><span class="o">=</span><span class="n">failed_session_count</span><span class="p">,</span>
|
||||
<span class="n">policy_string</span><span class="o">=</span><span class="n">policy_string</span><span class="p">,</span>
|
||||
<span class="n">mx_host_patterns</span><span class="o">=</span><span class="n">mx_host_patterns</span><span class="p">,</span>
|
||||
<span class="n">failure_details</span><span class="o">=</span><span class="n">failure_details</span><span class="p">)</span>
|
||||
<span class="k">def</span> <span class="nf">add_policy</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="p">,</span>
|
||||
<span class="n">policy_type</span><span class="p">,</span>
|
||||
<span class="n">policy_domain</span><span class="p">,</span>
|
||||
<span class="n">successful_session_count</span><span class="p">,</span>
|
||||
<span class="n">failed_session_count</span><span class="p">,</span>
|
||||
<span class="n">policy_string</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">mx_host_patterns</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">failure_details</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">policies</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="n">policy_type</span><span class="o">=</span><span class="n">policy_type</span><span class="p">,</span>
|
||||
<span class="n">policy_domain</span><span class="o">=</span><span class="n">policy_domain</span><span class="p">,</span>
|
||||
<span class="n">successful_session_count</span><span class="o">=</span><span class="n">successful_session_count</span><span class="p">,</span>
|
||||
<span class="n">failed_session_count</span><span class="o">=</span><span class="n">failed_session_count</span><span class="p">,</span>
|
||||
<span class="n">policy_string</span><span class="o">=</span><span class="n">policy_string</span><span class="p">,</span>
|
||||
<span class="n">mx_host_patterns</span><span class="o">=</span><span class="n">mx_host_patterns</span><span class="p">,</span>
|
||||
<span class="n">failure_details</span><span class="o">=</span><span class="n">failure_details</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="AlreadySaved"><a class="viewcode-back" href="../../api.html#parsedmarc.elastic.AlreadySaved">[docs]</a><span class="k">class</span> <span class="nc">AlreadySaved</span><span class="p">(</span><span class="ne">ValueError</span><span class="p">):</span>
|
||||
<div class="viewcode-block" id="AlreadySaved">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.AlreadySaved">[docs]</a>
|
||||
<span class="k">class</span> <span class="nc">AlreadySaved</span><span class="p">(</span><span class="ne">ValueError</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Raised when a report to be saved matches an existing report"""</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="set_hosts"><a class="viewcode-back" href="../../api.html#parsedmarc.elastic.set_hosts">[docs]</a><span class="k">def</span> <span class="nf">set_hosts</span><span class="p">(</span><span class="n">hosts</span><span class="p">,</span> <span class="n">use_ssl</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">ssl_cert_path</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">username</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">password</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">apiKey</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mf">60.0</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="set_hosts">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.set_hosts">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">set_hosts</span><span class="p">(</span>
|
||||
<span class="n">hosts</span><span class="p">,</span>
|
||||
<span class="n">use_ssl</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">ssl_cert_path</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">username</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">password</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">apiKey</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">timeout</span><span class="o">=</span><span class="mf">60.0</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Sets the Elasticsearch hosts to use</span>
|
||||
|
||||
@@ -343,25 +371,25 @@
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">hosts</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
|
||||
<span class="n">hosts</span> <span class="o">=</span> <span class="p">[</span><span class="n">hosts</span><span class="p">]</span>
|
||||
<span class="n">conn_params</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s2">"hosts"</span><span class="p">:</span> <span class="n">hosts</span><span class="p">,</span>
|
||||
<span class="s2">"timeout"</span><span class="p">:</span> <span class="n">timeout</span>
|
||||
<span class="p">}</span>
|
||||
<span class="n">conn_params</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"hosts"</span><span class="p">:</span> <span class="n">hosts</span><span class="p">,</span> <span class="s2">"timeout"</span><span class="p">:</span> <span class="n">timeout</span><span class="p">}</span>
|
||||
<span class="k">if</span> <span class="n">use_ssl</span><span class="p">:</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s1">'use_ssl'</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s2">"use_ssl"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="k">if</span> <span class="n">ssl_cert_path</span><span class="p">:</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s1">'verify_certs'</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s1">'ca_certs'</span><span class="p">]</span> <span class="o">=</span> <span class="n">ssl_cert_path</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s2">"verify_certs"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s2">"ca_certs"</span><span class="p">]</span> <span class="o">=</span> <span class="n">ssl_cert_path</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s1">'verify_certs'</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s2">"verify_certs"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
<span class="k">if</span> <span class="n">username</span><span class="p">:</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s1">'http_auth'</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span><span class="n">username</span><span class="o">+</span><span class="s2">":"</span><span class="o">+</span><span class="n">password</span><span class="p">)</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s2">"http_auth"</span><span class="p">]</span> <span class="o">=</span> <span class="n">username</span> <span class="o">+</span> <span class="s2">":"</span> <span class="o">+</span> <span class="n">password</span>
|
||||
<span class="k">if</span> <span class="n">apiKey</span><span class="p">:</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s1">'api_key'</span><span class="p">]</span> <span class="o">=</span> <span class="n">apiKey</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s2">"api_key"</span><span class="p">]</span> <span class="o">=</span> <span class="n">apiKey</span>
|
||||
<span class="n">connections</span><span class="o">.</span><span class="n">create_connection</span><span class="p">(</span><span class="o">**</span><span class="n">conn_params</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="create_indexes"><a class="viewcode-back" href="../../api.html#parsedmarc.elastic.create_indexes">[docs]</a><span class="k">def</span> <span class="nf">create_indexes</span><span class="p">(</span><span class="n">names</span><span class="p">,</span> <span class="n">settings</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="create_indexes">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.create_indexes">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">create_indexes</span><span class="p">(</span><span class="n">names</span><span class="p">,</span> <span class="n">settings</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Create Elasticsearch indexes</span>
|
||||
|
||||
@@ -376,17 +404,18 @@
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">index</span><span class="o">.</span><span class="n">exists</span><span class="p">():</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Creating Elasticsearch index: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">name</span><span class="p">))</span>
|
||||
<span class="k">if</span> <span class="n">settings</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">index</span><span class="o">.</span><span class="n">settings</span><span class="p">(</span><span class="n">number_of_shards</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
|
||||
<span class="n">index</span><span class="o">.</span><span class="n">settings</span><span class="p">(</span><span class="n">number_of_shards</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">number_of_replicas</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">index</span><span class="o">.</span><span class="n">settings</span><span class="p">(</span><span class="o">**</span><span class="n">settings</span><span class="p">)</span>
|
||||
<span class="n">index</span><span class="o">.</span><span class="n">create</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span>
|
||||
<span class="s2">"Elasticsearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="s2">"Elasticsearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="migrate_indexes"><a class="viewcode-back" href="../../api.html#parsedmarc.elastic.migrate_indexes">[docs]</a><span class="k">def</span> <span class="nf">migrate_indexes</span><span class="p">(</span><span class="n">aggregate_indexes</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">forensic_indexes</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="migrate_indexes">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.migrate_indexes">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">migrate_indexes</span><span class="p">(</span><span class="n">aggregate_indexes</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">forensic_indexes</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Updates index mappings</span>
|
||||
|
||||
@@ -415,33 +444,34 @@
|
||||
<span class="n">fo_type</span> <span class="o">=</span> <span class="n">fo_mapping</span><span class="p">[</span><span class="s2">"type"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="n">fo_type</span> <span class="o">==</span> <span class="s2">"long"</span><span class="p">:</span>
|
||||
<span class="n">new_index_name</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">-v</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">aggregate_index_name</span><span class="p">,</span> <span class="n">version</span><span class="p">)</span>
|
||||
<span class="n">body</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"properties"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"published_policy.fo"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"type"</span><span class="p">:</span> <span class="s2">"text"</span><span class="p">,</span>
|
||||
<span class="s2">"fields"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"keyword"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"type"</span><span class="p">:</span> <span class="s2">"keyword"</span><span class="p">,</span>
|
||||
<span class="s2">"ignore_above"</span><span class="p">:</span> <span class="mi">256</span>
|
||||
<span class="n">body</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s2">"properties"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"published_policy.fo"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"type"</span><span class="p">:</span> <span class="s2">"text"</span><span class="p">,</span>
|
||||
<span class="s2">"fields"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"keyword"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"type"</span><span class="p">:</span> <span class="s2">"keyword"</span><span class="p">,</span> <span class="s2">"ignore_above"</span><span class="p">:</span> <span class="mi">256</span><span class="p">}},</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">}</span>
|
||||
<span class="n">Index</span><span class="p">(</span><span class="n">new_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">create</span><span class="p">()</span>
|
||||
<span class="n">Index</span><span class="p">(</span><span class="n">new_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">put_mapping</span><span class="p">(</span><span class="n">doc_type</span><span class="o">=</span><span class="n">doc</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="n">body</span><span class="p">)</span>
|
||||
<span class="n">reindex</span><span class="p">(</span><span class="n">connections</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(),</span> <span class="n">aggregate_index_name</span><span class="p">,</span>
|
||||
<span class="n">new_index_name</span><span class="p">)</span>
|
||||
<span class="n">reindex</span><span class="p">(</span><span class="n">connections</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(),</span> <span class="n">aggregate_index_name</span><span class="p">,</span> <span class="n">new_index_name</span><span class="p">)</span>
|
||||
<span class="n">Index</span><span class="p">(</span><span class="n">aggregate_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">delete</span><span class="p">()</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">forensic_index</span> <span class="ow">in</span> <span class="n">forensic_indexes</span><span class="p">:</span>
|
||||
<span class="k">pass</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="save_aggregate_report_to_elasticsearch"><a class="viewcode-back" href="../../api.html#parsedmarc.elastic.save_aggregate_report_to_elasticsearch">[docs]</a><span class="k">def</span> <span class="nf">save_aggregate_report_to_elasticsearch</span><span class="p">(</span><span class="n">aggregate_report</span><span class="p">,</span>
|
||||
<span class="n">index_suffix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">index_prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">monthly_indexes</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="mi">0</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="save_aggregate_report_to_elasticsearch">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.save_aggregate_report_to_elasticsearch">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">save_aggregate_report_to_elasticsearch</span><span class="p">(</span>
|
||||
<span class="n">aggregate_report</span><span class="p">,</span>
|
||||
<span class="n">index_suffix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">index_prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">monthly_indexes</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Saves a parsed DMARC aggregate report to Elasticsearch</span>
|
||||
|
||||
@@ -462,10 +492,8 @@
|
||||
<span class="n">org_name</span> <span class="o">=</span> <span class="n">metadata</span><span class="p">[</span><span class="s2">"org_name"</span><span class="p">]</span>
|
||||
<span class="n">report_id</span> <span class="o">=</span> <span class="n">metadata</span><span class="p">[</span><span class="s2">"report_id"</span><span class="p">]</span>
|
||||
<span class="n">domain</span> <span class="o">=</span> <span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"domain"</span><span class="p">]</span>
|
||||
<span class="n">begin_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">metadata</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">],</span>
|
||||
<span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="n">end_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">metadata</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">],</span>
|
||||
<span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="n">begin_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">metadata</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="n">end_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">metadata</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="n">begin_date_human</span> <span class="o">=</span> <span class="n">begin_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%SZ"</span><span class="p">)</span>
|
||||
<span class="n">end_date_human</span> <span class="o">=</span> <span class="n">end_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%SZ"</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">monthly_indexes</span><span class="p">:</span>
|
||||
@@ -474,8 +502,7 @@
|
||||
<span class="n">index_date</span> <span class="o">=</span> <span class="n">begin_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">]</span> <span class="o">=</span> <span class="n">begin_date</span>
|
||||
<span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">]</span> <span class="o">=</span> <span class="n">end_date</span>
|
||||
<span class="n">date_range</span> <span class="o">=</span> <span class="p">[</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">],</span>
|
||||
<span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">]]</span>
|
||||
<span class="n">date_range</span> <span class="o">=</span> <span class="p">[</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">],</span> <span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">]]</span>
|
||||
|
||||
<span class="n">org_name_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match_phrase</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">org_name</span><span class="o">=</span><span class="n">org_name</span><span class="p">)))</span>
|
||||
<span class="n">report_id_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match_phrase</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">report_id</span><span class="o">=</span><span class="n">report_id</span><span class="p">)))</span>
|
||||
@@ -497,18 +524,20 @@
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">existing</span> <span class="o">=</span> <span class="n">search</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error_</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="s2">"Elasticsearch's search for existing report </span><span class="se">\</span>
|
||||
<span class="s2"> error: </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error_</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span>
|
||||
<span class="s2">"Elasticsearch's search for existing report </span><span class="se">\</span>
|
||||
<span class="s2"> error: </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error_</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">existing</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">AlreadySaved</span><span class="p">(</span><span class="s2">"An aggregate report ID </span><span class="si">{0}</span><span class="s2"> from </span><span class="si">{1}</span><span class="s2"> about </span><span class="si">{2}</span><span class="s2"> "</span>
|
||||
<span class="s2">"with a date range of </span><span class="si">{3}</span><span class="s2"> UTC to </span><span class="si">{4}</span><span class="s2"> UTC already "</span>
|
||||
<span class="s2">"exists in "</span>
|
||||
<span class="s2">"Elasticsearch"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">report_id</span><span class="p">,</span>
|
||||
<span class="n">org_name</span><span class="p">,</span>
|
||||
<span class="n">domain</span><span class="p">,</span>
|
||||
<span class="n">begin_date_human</span><span class="p">,</span>
|
||||
<span class="n">end_date_human</span><span class="p">))</span>
|
||||
<span class="k">raise</span> <span class="n">AlreadySaved</span><span class="p">(</span>
|
||||
<span class="s2">"An aggregate report ID </span><span class="si">{0}</span><span class="s2"> from </span><span class="si">{1}</span><span class="s2"> about </span><span class="si">{2}</span><span class="s2"> "</span>
|
||||
<span class="s2">"with a date range of </span><span class="si">{3}</span><span class="s2"> UTC to </span><span class="si">{4}</span><span class="s2"> UTC already "</span>
|
||||
<span class="s2">"exists in "</span>
|
||||
<span class="s2">"Elasticsearch"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">report_id</span><span class="p">,</span> <span class="n">org_name</span><span class="p">,</span> <span class="n">domain</span><span class="p">,</span> <span class="n">begin_date_human</span><span class="p">,</span> <span class="n">end_date_human</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">published_policy</span> <span class="o">=</span> <span class="n">_PublishedPolicy</span><span class="p">(</span>
|
||||
<span class="n">domain</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"domain"</span><span class="p">],</span>
|
||||
<span class="n">adkim</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"adkim"</span><span class="p">],</span>
|
||||
@@ -516,7 +545,7 @@
|
||||
<span class="n">p</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"p"</span><span class="p">],</span>
|
||||
<span class="n">sp</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"sp"</span><span class="p">],</span>
|
||||
<span class="n">pct</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"pct"</span><span class="p">],</span>
|
||||
<span class="n">fo</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"fo"</span><span class="p">]</span>
|
||||
<span class="n">fo</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"fo"</span><span class="p">],</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">record</span> <span class="ow">in</span> <span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"records"</span><span class="p">]:</span>
|
||||
@@ -539,28 +568,33 @@
|
||||
<span class="n">source_name</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"name"</span><span class="p">],</span>
|
||||
<span class="n">message_count</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"count"</span><span class="p">],</span>
|
||||
<span class="n">disposition</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"disposition"</span><span class="p">],</span>
|
||||
<span class="n">dkim_aligned</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"dkim"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span>
|
||||
<span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"dkim"</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s2">"pass"</span><span class="p">,</span>
|
||||
<span class="n">spf_aligned</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span>
|
||||
<span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s2">"pass"</span><span class="p">,</span>
|
||||
<span class="n">dkim_aligned</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"dkim"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
|
||||
<span class="ow">and</span> <span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"dkim"</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s2">"pass"</span><span class="p">,</span>
|
||||
<span class="n">spf_aligned</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
|
||||
<span class="ow">and</span> <span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s2">"pass"</span><span class="p">,</span>
|
||||
<span class="n">header_from</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"header_from"</span><span class="p">],</span>
|
||||
<span class="n">envelope_from</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"envelope_from"</span><span class="p">],</span>
|
||||
<span class="n">envelope_to</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"envelope_to"</span><span class="p">]</span>
|
||||
<span class="n">envelope_to</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"envelope_to"</span><span class="p">],</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">override</span> <span class="ow">in</span> <span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"policy_override_reasons"</span><span class="p">]:</span>
|
||||
<span class="n">agg_doc</span><span class="o">.</span><span class="n">add_policy_override</span><span class="p">(</span><span class="n">type_</span><span class="o">=</span><span class="n">override</span><span class="p">[</span><span class="s2">"type"</span><span class="p">],</span>
|
||||
<span class="n">comment</span><span class="o">=</span><span class="n">override</span><span class="p">[</span><span class="s2">"comment"</span><span class="p">])</span>
|
||||
<span class="n">agg_doc</span><span class="o">.</span><span class="n">add_policy_override</span><span class="p">(</span>
|
||||
<span class="n">type_</span><span class="o">=</span><span class="n">override</span><span class="p">[</span><span class="s2">"type"</span><span class="p">],</span> <span class="n">comment</span><span class="o">=</span><span class="n">override</span><span class="p">[</span><span class="s2">"comment"</span><span class="p">]</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">dkim_result</span> <span class="ow">in</span> <span class="n">record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">][</span><span class="s2">"dkim"</span><span class="p">]:</span>
|
||||
<span class="n">agg_doc</span><span class="o">.</span><span class="n">add_dkim_result</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">],</span>
|
||||
<span class="n">selector</span><span class="o">=</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">"selector"</span><span class="p">],</span>
|
||||
<span class="n">result</span><span class="o">=</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">])</span>
|
||||
<span class="n">agg_doc</span><span class="o">.</span><span class="n">add_dkim_result</span><span class="p">(</span>
|
||||
<span class="n">domain</span><span class="o">=</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">],</span>
|
||||
<span class="n">selector</span><span class="o">=</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">"selector"</span><span class="p">],</span>
|
||||
<span class="n">result</span><span class="o">=</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">],</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">spf_result</span> <span class="ow">in</span> <span class="n">record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">]:</span>
|
||||
<span class="n">agg_doc</span><span class="o">.</span><span class="n">add_spf_result</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">],</span>
|
||||
<span class="n">scope</span><span class="o">=</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">"scope"</span><span class="p">],</span>
|
||||
<span class="n">result</span><span class="o">=</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">])</span>
|
||||
<span class="n">agg_doc</span><span class="o">.</span><span class="n">add_spf_result</span><span class="p">(</span>
|
||||
<span class="n">domain</span><span class="o">=</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">],</span>
|
||||
<span class="n">scope</span><span class="o">=</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">"scope"</span><span class="p">],</span>
|
||||
<span class="n">result</span><span class="o">=</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">],</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"dmarc_aggregate"</span>
|
||||
<span class="k">if</span> <span class="n">index_suffix</span><span class="p">:</span>
|
||||
@@ -569,41 +603,46 @@
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">index</span><span class="p">)</span>
|
||||
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">-</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_date</span><span class="p">)</span>
|
||||
<span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span><span class="p">)</span>
|
||||
<span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span> <span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">create_indexes</span><span class="p">([</span><span class="n">index</span><span class="p">],</span> <span class="n">index_settings</span><span class="p">)</span>
|
||||
<span class="n">agg_doc</span><span class="o">.</span><span class="n">meta</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">index</span>
|
||||
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">agg_doc</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span>
|
||||
<span class="s2">"Elasticsearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="s2">"Elasticsearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="save_forensic_report_to_elasticsearch"><a class="viewcode-back" href="../../api.html#parsedmarc.elastic.save_forensic_report_to_elasticsearch">[docs]</a><span class="k">def</span> <span class="nf">save_forensic_report_to_elasticsearch</span><span class="p">(</span><span class="n">forensic_report</span><span class="p">,</span>
|
||||
<span class="n">index_suffix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">index_prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">monthly_indexes</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="mi">0</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="save_forensic_report_to_elasticsearch">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.save_forensic_report_to_elasticsearch">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">save_forensic_report_to_elasticsearch</span><span class="p">(</span>
|
||||
<span class="n">forensic_report</span><span class="p">,</span>
|
||||
<span class="n">index_suffix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">index_prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">monthly_indexes</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Saves a parsed DMARC forensic report to Elasticsearch</span>
|
||||
<span class="sd"> Saves a parsed DMARC forensic report to Elasticsearch</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> forensic_report (OrderedDict): A parsed forensic report</span>
|
||||
<span class="sd"> index_suffix (str): The suffix of the name of the index to save to</span>
|
||||
<span class="sd"> index_prefix (str): The prefix of the name of the index to save to</span>
|
||||
<span class="sd"> monthly_indexes (bool): Use monthly indexes instead of daily</span>
|
||||
<span class="sd"> indexes</span>
|
||||
<span class="sd"> number_of_shards (int): The number of shards to use in the index</span>
|
||||
<span class="sd"> number_of_replicas (int): The number of replicas to use in the</span>
|
||||
<span class="sd"> index</span>
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> forensic_report (OrderedDict): A parsed forensic report</span>
|
||||
<span class="sd"> index_suffix (str): The suffix of the name of the index to save to</span>
|
||||
<span class="sd"> index_prefix (str): The prefix of the name of the index to save to</span>
|
||||
<span class="sd"> monthly_indexes (bool): Use monthly indexes instead of daily</span>
|
||||
<span class="sd"> indexes</span>
|
||||
<span class="sd"> number_of_shards (int): The number of shards to use in the index</span>
|
||||
<span class="sd"> number_of_replicas (int): The number of replicas to use in the</span>
|
||||
<span class="sd"> index</span>
|
||||
|
||||
<span class="sd"> Raises:</span>
|
||||
<span class="sd"> AlreadySaved</span>
|
||||
<span class="sd"> Raises:</span>
|
||||
<span class="sd"> AlreadySaved</span>
|
||||
|
||||
<span class="sd"> """</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Saving forensic report to Elasticsearch"</span><span class="p">)</span>
|
||||
<span class="n">forensic_report</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
<span class="n">sample_date</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
@@ -648,14 +687,12 @@
|
||||
<span class="n">existing</span> <span class="o">=</span> <span class="n">search</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
|
||||
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">existing</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">AlreadySaved</span><span class="p">(</span><span class="s2">"A forensic sample to </span><span class="si">{0}</span><span class="s2"> from </span><span class="si">{1}</span><span class="s2"> "</span>
|
||||
<span class="s2">"with a subject of </span><span class="si">{2}</span><span class="s2"> and arrival date of </span><span class="si">{3}</span><span class="s2"> "</span>
|
||||
<span class="s2">"already exists in "</span>
|
||||
<span class="s2">"Elasticsearch"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">to_</span><span class="p">,</span>
|
||||
<span class="n">from_</span><span class="p">,</span>
|
||||
<span class="n">subject</span><span class="p">,</span>
|
||||
<span class="n">arrival_date_human</span>
|
||||
<span class="p">))</span>
|
||||
<span class="k">raise</span> <span class="n">AlreadySaved</span><span class="p">(</span>
|
||||
<span class="s2">"A forensic sample to </span><span class="si">{0}</span><span class="s2"> from </span><span class="si">{1}</span><span class="s2"> "</span>
|
||||
<span class="s2">"with a subject of </span><span class="si">{2}</span><span class="s2"> and arrival date of </span><span class="si">{3}</span><span class="s2"> "</span>
|
||||
<span class="s2">"already exists in "</span>
|
||||
<span class="s2">"Elasticsearch"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">to_</span><span class="p">,</span> <span class="n">from_</span><span class="p">,</span> <span class="n">subject</span><span class="p">,</span> <span class="n">arrival_date_human</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">parsed_sample</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">]</span>
|
||||
<span class="n">sample</span> <span class="o">=</span> <span class="n">_ForensicSampleDoc</span><span class="p">(</span>
|
||||
@@ -665,25 +702,25 @@
|
||||
<span class="n">date</span><span class="o">=</span><span class="n">sample_date</span><span class="p">,</span>
|
||||
<span class="n">subject</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"subject"</span><span class="p">],</span>
|
||||
<span class="n">filename_safe_subject</span><span class="o">=</span><span class="n">parsed_sample</span><span class="p">[</span><span class="s2">"filename_safe_subject"</span><span class="p">],</span>
|
||||
<span class="n">body</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"body"</span><span class="p">]</span>
|
||||
<span class="n">body</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"body"</span><span class="p">],</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"to"</span><span class="p">]:</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_to</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span>
|
||||
<span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">])</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_to</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">])</span>
|
||||
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"reply_to"</span><span class="p">]:</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_reply_to</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span>
|
||||
<span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">])</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_reply_to</span><span class="p">(</span>
|
||||
<span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">]</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"cc"</span><span class="p">]:</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_cc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span>
|
||||
<span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">])</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_cc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">])</span>
|
||||
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"bcc"</span><span class="p">]:</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_bcc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span>
|
||||
<span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">])</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_bcc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">])</span>
|
||||
<span class="k">for</span> <span class="n">attachment</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"attachments"</span><span class="p">]:</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_attachment</span><span class="p">(</span><span class="n">filename</span><span class="o">=</span><span class="n">attachment</span><span class="p">[</span><span class="s2">"filename"</span><span class="p">],</span>
|
||||
<span class="n">content_type</span><span class="o">=</span><span class="n">attachment</span><span class="p">[</span><span class="s2">"mail_content_type"</span><span class="p">],</span>
|
||||
<span class="n">sha256</span><span class="o">=</span><span class="n">attachment</span><span class="p">[</span><span class="s2">"sha256"</span><span class="p">])</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_attachment</span><span class="p">(</span>
|
||||
<span class="n">filename</span><span class="o">=</span><span class="n">attachment</span><span class="p">[</span><span class="s2">"filename"</span><span class="p">],</span>
|
||||
<span class="n">content_type</span><span class="o">=</span><span class="n">attachment</span><span class="p">[</span><span class="s2">"mail_content_type"</span><span class="p">],</span>
|
||||
<span class="n">sha256</span><span class="o">=</span><span class="n">attachment</span><span class="p">[</span><span class="s2">"sha256"</span><span class="p">],</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">forensic_doc</span> <span class="o">=</span> <span class="n">_ForensicReportDoc</span><span class="p">(</span>
|
||||
<span class="n">feedback_type</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"feedback_type"</span><span class="p">],</span>
|
||||
@@ -699,12 +736,11 @@
|
||||
<span class="n">source_country</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"country"</span><span class="p">],</span>
|
||||
<span class="n">source_reverse_dns</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"reverse_dns"</span><span class="p">],</span>
|
||||
<span class="n">source_base_domain</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"base_domain"</span><span class="p">],</span>
|
||||
<span class="n">authentication_mechanisms</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span>
|
||||
<span class="s2">"authentication_mechanisms"</span><span class="p">],</span>
|
||||
<span class="n">authentication_mechanisms</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"authentication_mechanisms"</span><span class="p">],</span>
|
||||
<span class="n">auth_failure</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"auth_failure"</span><span class="p">],</span>
|
||||
<span class="n">dkim_domain</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"dkim_domain"</span><span class="p">],</span>
|
||||
<span class="n">original_rcpt_to</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"original_rcpt_to"</span><span class="p">],</span>
|
||||
<span class="n">sample</span><span class="o">=</span><span class="n">sample</span>
|
||||
<span class="n">sample</span><span class="o">=</span><span class="n">sample</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"dmarc_forensic"</span>
|
||||
@@ -717,26 +753,32 @@
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">index_date</span> <span class="o">=</span> <span class="n">arrival_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">-</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_date</span><span class="p">)</span>
|
||||
<span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span><span class="p">)</span>
|
||||
<span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span> <span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">create_indexes</span><span class="p">([</span><span class="n">index</span><span class="p">],</span> <span class="n">index_settings</span><span class="p">)</span>
|
||||
<span class="n">forensic_doc</span><span class="o">.</span><span class="n">meta</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">index</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">forensic_doc</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span>
|
||||
<span class="s2">"Elasticsearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="s2">"Elasticsearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
||||
<span class="k">except</span> <span class="ne">KeyError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidForensicReport</span><span class="p">(</span>
|
||||
<span class="s2">"Forensic report missing required field: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
<span class="s2">"Forensic report missing required field: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="save_smtp_tls_report_to_elasticsearch"><a class="viewcode-back" href="../../api.html#parsedmarc.elastic.save_smtp_tls_report_to_elasticsearch">[docs]</a><span class="k">def</span> <span class="nf">save_smtp_tls_report_to_elasticsearch</span><span class="p">(</span><span class="n">report</span><span class="p">,</span>
|
||||
<span class="n">index_suffix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">index_prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">monthly_indexes</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="mi">0</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="save_smtp_tls_report_to_elasticsearch">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.save_smtp_tls_report_to_elasticsearch">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">save_smtp_tls_report_to_elasticsearch</span><span class="p">(</span>
|
||||
<span class="n">report</span><span class="p">,</span>
|
||||
<span class="n">index_suffix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">index_prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">monthly_indexes</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Saves a parsed SMTP TLS report to Elasticsearch</span>
|
||||
|
||||
@@ -754,10 +796,8 @@
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Saving smtp tls report to Elasticsearch"</span><span class="p">)</span>
|
||||
<span class="n">org_name</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"organization_name"</span><span class="p">]</span>
|
||||
<span class="n">report_id</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_id"</span><span class="p">]</span>
|
||||
<span class="n">begin_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">],</span>
|
||||
<span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="n">end_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">],</span>
|
||||
<span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="n">begin_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="n">end_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="n">begin_date_human</span> <span class="o">=</span> <span class="n">begin_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%SZ"</span><span class="p">)</span>
|
||||
<span class="n">end_date_human</span> <span class="o">=</span> <span class="n">end_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%SZ"</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">monthly_indexes</span><span class="p">:</span>
|
||||
@@ -786,15 +826,19 @@
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">existing</span> <span class="o">=</span> <span class="n">search</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error_</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="s2">"Elasticsearch's search for existing report </span><span class="se">\</span>
|
||||
<span class="s2"> error: </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error_</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span>
|
||||
<span class="s2">"Elasticsearch's search for existing report </span><span class="se">\</span>
|
||||
<span class="s2"> error: </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error_</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">existing</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">AlreadySaved</span><span class="p">(</span><span class="sa">f</span><span class="s2">"An SMTP TLS report ID </span><span class="si">{</span><span class="n">report_id</span><span class="si">}</span><span class="s2"> from "</span>
|
||||
<span class="sa">f</span><span class="s2">" </span><span class="si">{</span><span class="n">org_name</span><span class="si">}</span><span class="s2"> with a date range of "</span>
|
||||
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">begin_date_human</span><span class="si">}</span><span class="s2"> UTC to "</span>
|
||||
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">end_date_human</span><span class="si">}</span><span class="s2"> UTC already "</span>
|
||||
<span class="s2">"exists in Elasticsearch"</span><span class="p">)</span>
|
||||
<span class="k">raise</span> <span class="n">AlreadySaved</span><span class="p">(</span>
|
||||
<span class="sa">f</span><span class="s2">"An SMTP TLS report ID </span><span class="si">{</span><span class="n">report_id</span><span class="si">}</span><span class="s2"> from "</span>
|
||||
<span class="sa">f</span><span class="s2">" </span><span class="si">{</span><span class="n">org_name</span><span class="si">}</span><span class="s2"> with a date range of "</span>
|
||||
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">begin_date_human</span><span class="si">}</span><span class="s2"> UTC to "</span>
|
||||
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">end_date_human</span><span class="si">}</span><span class="s2"> UTC already "</span>
|
||||
<span class="s2">"exists in Elasticsearch"</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"smtp_tls"</span>
|
||||
<span class="k">if</span> <span class="n">index_suffix</span><span class="p">:</span>
|
||||
@@ -802,8 +846,9 @@
|
||||
<span class="k">if</span> <span class="n">index_prefix</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">index</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">-</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_date</span><span class="p">)</span>
|
||||
<span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span><span class="p">)</span>
|
||||
<span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span> <span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">smtp_tls_doc</span> <span class="o">=</span> <span class="n">_SMTPTLSReportDoc</span><span class="p">(</span>
|
||||
<span class="n">org_name</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">"organization_name"</span><span class="p">],</span>
|
||||
@@ -811,10 +856,10 @@
|
||||
<span class="n">date_begin</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">],</span>
|
||||
<span class="n">date_end</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">],</span>
|
||||
<span class="n">contact_info</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">"contact_info"</span><span class="p">],</span>
|
||||
<span class="n">report_id</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">"report_id"</span><span class="p">]</span>
|
||||
<span class="n">report_id</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">"report_id"</span><span class="p">],</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">policy</span> <span class="ow">in</span> <span class="n">report</span><span class="p">[</span><span class="s1">'policies'</span><span class="p">]:</span>
|
||||
<span class="k">for</span> <span class="n">policy</span> <span class="ow">in</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policies"</span><span class="p">]:</span>
|
||||
<span class="n">policy_strings</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="n">mx_host_patterns</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="k">if</span> <span class="s2">"policy_strings"</span> <span class="ow">in</span> <span class="n">policy</span><span class="p">:</span>
|
||||
@@ -827,7 +872,7 @@
|
||||
<span class="n">succesful_session_count</span><span class="o">=</span><span class="n">policy</span><span class="p">[</span><span class="s2">"successful_session_count"</span><span class="p">],</span>
|
||||
<span class="n">failed_session_count</span><span class="o">=</span><span class="n">policy</span><span class="p">[</span><span class="s2">"failed_session_count"</span><span class="p">],</span>
|
||||
<span class="n">policy_string</span><span class="o">=</span><span class="n">policy_strings</span><span class="p">,</span>
|
||||
<span class="n">mx_host_patterns</span><span class="o">=</span><span class="n">mx_host_patterns</span>
|
||||
<span class="n">mx_host_patterns</span><span class="o">=</span><span class="n">mx_host_patterns</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">if</span> <span class="s2">"failure_details"</span> <span class="ow">in</span> <span class="n">policy</span><span class="p">:</span>
|
||||
<span class="k">for</span> <span class="n">failure_detail</span> <span class="ow">in</span> <span class="n">policy</span><span class="p">[</span><span class="s2">"failure_details"</span><span class="p">]:</span>
|
||||
@@ -840,11 +885,11 @@
|
||||
<span class="n">sending_mta_ip</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
|
||||
<span class="k">if</span> <span class="s2">"receiving_mx_hostname"</span> <span class="ow">in</span> <span class="n">failure_detail</span><span class="p">:</span>
|
||||
<span class="n">receiving_mx_hostname</span> <span class="o">=</span> <span class="n">failure_detail</span><span class="p">[</span>
|
||||
<span class="s2">"receiving_mx_hostname"</span><span class="p">]</span>
|
||||
<span class="n">receiving_mx_hostname</span> <span class="o">=</span> <span class="n">failure_detail</span><span class="p">[</span><span class="s2">"receiving_mx_hostname"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="s2">"additional_information_uri"</span> <span class="ow">in</span> <span class="n">failure_detail</span><span class="p">:</span>
|
||||
<span class="n">additional_information_uri</span> <span class="o">=</span> <span class="n">failure_detail</span><span class="p">[</span>
|
||||
<span class="s2">"additional_information_uri"</span><span class="p">]</span>
|
||||
<span class="s2">"additional_information_uri"</span>
|
||||
<span class="p">]</span>
|
||||
<span class="k">if</span> <span class="s2">"failure_reason_code"</span> <span class="ow">in</span> <span class="n">failure_detail</span><span class="p">:</span>
|
||||
<span class="n">failure_reason_code</span> <span class="o">=</span> <span class="n">failure_detail</span><span class="p">[</span><span class="s2">"failure_reason_code"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="s2">"ip_address"</span> <span class="ow">in</span> <span class="n">failure_detail</span><span class="p">:</span>
|
||||
@@ -860,12 +905,11 @@
|
||||
<span class="n">ip_address</span><span class="o">=</span><span class="n">ip_address</span><span class="p">,</span>
|
||||
<span class="n">receiving_ip</span><span class="o">=</span><span class="n">receiving_ip</span><span class="p">,</span>
|
||||
<span class="n">receiving_mx_helo</span><span class="o">=</span><span class="n">receiving_mx_helo</span><span class="p">,</span>
|
||||
<span class="n">failed_session_count</span><span class="o">=</span><span class="n">failure_detail</span><span class="p">[</span>
|
||||
<span class="s2">"failed_session_count"</span><span class="p">],</span>
|
||||
<span class="n">failed_session_count</span><span class="o">=</span><span class="n">failure_detail</span><span class="p">[</span><span class="s2">"failed_session_count"</span><span class="p">],</span>
|
||||
<span class="n">sending_mta_ip</span><span class="o">=</span><span class="n">sending_mta_ip</span><span class="p">,</span>
|
||||
<span class="n">receiving_mx_hostname</span><span class="o">=</span><span class="n">receiving_mx_hostname</span><span class="p">,</span>
|
||||
<span class="n">additional_information_uri</span><span class="o">=</span><span class="n">additional_information_uri</span><span class="p">,</span>
|
||||
<span class="n">failure_reason_code</span><span class="o">=</span><span class="n">failure_reason_code</span>
|
||||
<span class="n">failure_reason_code</span><span class="o">=</span><span class="n">failure_reason_code</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">smtp_tls_doc</span><span class="o">.</span><span class="n">policies</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">policy_doc</span><span class="p">)</span>
|
||||
|
||||
@@ -875,8 +919,8 @@
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">smtp_tls_doc</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span>
|
||||
<span class="s2">"Elasticsearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="s2">"Elasticsearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
|
||||
</pre></div>
|
||||
|
||||
</div>
|
||||
|
||||
+252
-204
@@ -1,23 +1,20 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="../../">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc.opensearch — parsedmarc 8.15.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css" />
|
||||
<title>parsedmarc.opensearch — parsedmarc 8.15.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=80d5e7a1" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="../../_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="../../" id="documentation_options" src="../../_static/documentation_options.js"></script>
|
||||
<script src="../../_static/jquery.js"></script>
|
||||
<script src="../../_static/underscore.js"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="../../_static/doctools.js"></script>
|
||||
<script src="../../_static/sphinx_highlight.js"></script>
|
||||
<script src="../../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../../_static/documentation_options.js?v=6add7570"></script>
|
||||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="../../_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="../../genindex.html" />
|
||||
<link rel="search" title="Search" href="../../search.html" />
|
||||
@@ -34,9 +31,6 @@
|
||||
<a href="../../index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.15.0
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
@@ -90,8 +84,21 @@
|
||||
|
||||
<span class="kn">from</span> <span class="nn">collections</span> <span class="kn">import</span> <span class="n">OrderedDict</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">opensearchpy</span> <span class="kn">import</span> <span class="n">Q</span><span class="p">,</span> <span class="n">connections</span><span class="p">,</span> <span class="n">Object</span><span class="p">,</span> <span class="n">Document</span><span class="p">,</span> <span class="n">Index</span><span class="p">,</span> <span class="n">Nested</span><span class="p">,</span> \
|
||||
<span class="n">InnerDoc</span><span class="p">,</span> <span class="n">Integer</span><span class="p">,</span> <span class="n">Text</span><span class="p">,</span> <span class="n">Boolean</span><span class="p">,</span> <span class="n">Ip</span><span class="p">,</span> <span class="n">Date</span><span class="p">,</span> <span class="n">Search</span>
|
||||
<span class="kn">from</span> <span class="nn">opensearchpy</span> <span class="kn">import</span> <span class="p">(</span>
|
||||
<span class="n">Q</span><span class="p">,</span>
|
||||
<span class="n">connections</span><span class="p">,</span>
|
||||
<span class="n">Object</span><span class="p">,</span>
|
||||
<span class="n">Document</span><span class="p">,</span>
|
||||
<span class="n">Index</span><span class="p">,</span>
|
||||
<span class="n">Nested</span><span class="p">,</span>
|
||||
<span class="n">InnerDoc</span><span class="p">,</span>
|
||||
<span class="n">Integer</span><span class="p">,</span>
|
||||
<span class="n">Text</span><span class="p">,</span>
|
||||
<span class="n">Boolean</span><span class="p">,</span>
|
||||
<span class="n">Ip</span><span class="p">,</span>
|
||||
<span class="n">Date</span><span class="p">,</span>
|
||||
<span class="n">Search</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="kn">from</span> <span class="nn">opensearchpy.helpers</span> <span class="kn">import</span> <span class="n">reindex</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">parsedmarc.log</span> <span class="kn">import</span> <span class="n">logger</span>
|
||||
@@ -99,10 +106,13 @@
|
||||
<span class="kn">from</span> <span class="nn">parsedmarc</span> <span class="kn">import</span> <span class="n">InvalidForensicReport</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="OpenSearchError"><a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.OpenSearchError">[docs]</a><span class="k">class</span> <span class="nc">OpenSearchError</span><span class="p">(</span><span class="ne">Exception</span><span class="p">):</span>
|
||||
<div class="viewcode-block" id="OpenSearchError">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.OpenSearchError">[docs]</a>
|
||||
<span class="k">class</span> <span class="nc">OpenSearchError</span><span class="p">(</span><span class="ne">Exception</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Raised when an OpenSearch error occurs"""</span></div>
|
||||
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_PolicyOverride</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="nb">type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">comment</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
@@ -163,24 +173,21 @@
|
||||
<span class="n">spf_results</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SPFResult</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_policy_override</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">type_</span><span class="p">,</span> <span class="n">comment</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">policy_overrides</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_PolicyOverride</span><span class="p">(</span><span class="nb">type</span><span class="o">=</span><span class="n">type_</span><span class="p">,</span>
|
||||
<span class="n">comment</span><span class="o">=</span><span class="n">comment</span><span class="p">))</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">policy_overrides</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_PolicyOverride</span><span class="p">(</span><span class="nb">type</span><span class="o">=</span><span class="n">type_</span><span class="p">,</span> <span class="n">comment</span><span class="o">=</span><span class="n">comment</span><span class="p">))</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_dkim_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">domain</span><span class="p">,</span> <span class="n">selector</span><span class="p">,</span> <span class="n">result</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">dkim_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_DKIMResult</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span>
|
||||
<span class="n">selector</span><span class="o">=</span><span class="n">selector</span><span class="p">,</span>
|
||||
<span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">))</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">dkim_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="n">_DKIMResult</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span> <span class="n">selector</span><span class="o">=</span><span class="n">selector</span><span class="p">,</span> <span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_spf_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">domain</span><span class="p">,</span> <span class="n">scope</span><span class="p">,</span> <span class="n">result</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">spf_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_SPFResult</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span>
|
||||
<span class="n">scope</span><span class="o">=</span><span class="n">scope</span><span class="p">,</span>
|
||||
<span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">))</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">spf_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_SPFResult</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span> <span class="n">scope</span><span class="o">=</span><span class="n">scope</span><span class="p">,</span> <span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">))</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">save</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">**</span> <span class="n">kwargs</span><span class="p">):</span>
|
||||
<span class="k">def</span> <span class="nf">save</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">passed_dmarc</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">passed_dmarc</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">spf_aligned</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">dkim_aligned</span>
|
||||
|
||||
<span class="k">return</span> <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="o">**</span> <span class="n">kwargs</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_EmailAddressDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
@@ -210,24 +217,25 @@
|
||||
<span class="n">attachments</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_EmailAttachmentDoc</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_to</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">to</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span>
|
||||
<span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">to</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_reply_to</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">reply_to</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span>
|
||||
<span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">reply_to</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_cc</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">cc</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span>
|
||||
<span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">cc</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_bcc</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">bcc</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span>
|
||||
<span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">bcc</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_attachment</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">filename</span><span class="p">,</span> <span class="n">content_type</span><span class="p">,</span> <span class="n">sha256</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">attachments</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAttachmentDoc</span><span class="p">(</span><span class="n">filename</span><span class="o">=</span><span class="n">filename</span><span class="p">,</span>
|
||||
<span class="n">content_type</span><span class="o">=</span><span class="n">content_type</span><span class="p">,</span> <span class="n">sha256</span><span class="o">=</span><span class="n">sha256</span><span class="p">))</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">attachments</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="n">_EmailAttachmentDoc</span><span class="p">(</span>
|
||||
<span class="n">filename</span><span class="o">=</span><span class="n">filename</span><span class="p">,</span> <span class="n">content_type</span><span class="o">=</span><span class="n">content_type</span><span class="p">,</span> <span class="n">sha256</span><span class="o">=</span><span class="n">sha256</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_ForensicReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
@@ -272,13 +280,17 @@
|
||||
<span class="n">failed_session_count</span> <span class="o">=</span> <span class="n">Integer</span><span class="p">()</span>
|
||||
<span class="n">failure_details</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SMTPTLSFailureDetailsDoc</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_failure_details</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">result_type</span><span class="p">,</span> <span class="n">ip_address</span><span class="p">,</span>
|
||||
<span class="n">receiving_ip</span><span class="p">,</span>
|
||||
<span class="n">receiving_mx_helo</span><span class="p">,</span>
|
||||
<span class="n">failed_session_count</span><span class="p">,</span>
|
||||
<span class="n">receiving_mx_hostname</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">additional_information_uri</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">failure_reason_code</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
<span class="k">def</span> <span class="nf">add_failure_details</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="p">,</span>
|
||||
<span class="n">result_type</span><span class="p">,</span>
|
||||
<span class="n">ip_address</span><span class="p">,</span>
|
||||
<span class="n">receiving_ip</span><span class="p">,</span>
|
||||
<span class="n">receiving_mx_helo</span><span class="p">,</span>
|
||||
<span class="n">failed_session_count</span><span class="p">,</span>
|
||||
<span class="n">receiving_mx_hostname</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">additional_information_uri</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">failure_reason_code</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">failure_details</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="n">result_type</span><span class="o">=</span><span class="n">result_type</span><span class="p">,</span>
|
||||
<span class="n">ip_address</span><span class="o">=</span><span class="n">ip_address</span><span class="p">,</span>
|
||||
@@ -287,12 +299,11 @@
|
||||
<span class="n">receiving_ip</span><span class="o">=</span><span class="n">receiving_ip</span><span class="p">,</span>
|
||||
<span class="n">failed_session_count</span><span class="o">=</span><span class="n">failed_session_count</span><span class="p">,</span>
|
||||
<span class="n">additional_information</span><span class="o">=</span><span class="n">additional_information_uri</span><span class="p">,</span>
|
||||
<span class="n">failure_reason_code</span><span class="o">=</span><span class="n">failure_reason_code</span>
|
||||
<span class="n">failure_reason_code</span><span class="o">=</span><span class="n">failure_reason_code</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_SMTPTLSFailureReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
|
||||
<span class="k">class</span> <span class="nc">Index</span><span class="p">:</span>
|
||||
<span class="n">name</span> <span class="o">=</span> <span class="s2">"smtp_tls"</span>
|
||||
|
||||
@@ -304,27 +315,45 @@
|
||||
<span class="n">report_id</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">policies</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SMTPTLSPolicyDoc</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_policy</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">policy_type</span><span class="p">,</span> <span class="n">policy_domain</span><span class="p">,</span>
|
||||
<span class="n">successful_session_count</span><span class="p">,</span>
|
||||
<span class="n">failed_session_count</span><span class="p">,</span>
|
||||
<span class="n">policy_string</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">mx_host_patterns</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">failure_details</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">policies</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">policy_type</span><span class="o">=</span><span class="n">policy_type</span><span class="p">,</span>
|
||||
<span class="n">policy_domain</span><span class="o">=</span><span class="n">policy_domain</span><span class="p">,</span>
|
||||
<span class="n">successful_session_count</span><span class="o">=</span><span class="n">successful_session_count</span><span class="p">,</span>
|
||||
<span class="n">failed_session_count</span><span class="o">=</span><span class="n">failed_session_count</span><span class="p">,</span>
|
||||
<span class="n">policy_string</span><span class="o">=</span><span class="n">policy_string</span><span class="p">,</span>
|
||||
<span class="n">mx_host_patterns</span><span class="o">=</span><span class="n">mx_host_patterns</span><span class="p">,</span>
|
||||
<span class="n">failure_details</span><span class="o">=</span><span class="n">failure_details</span><span class="p">)</span>
|
||||
<span class="k">def</span> <span class="nf">add_policy</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="p">,</span>
|
||||
<span class="n">policy_type</span><span class="p">,</span>
|
||||
<span class="n">policy_domain</span><span class="p">,</span>
|
||||
<span class="n">successful_session_count</span><span class="p">,</span>
|
||||
<span class="n">failed_session_count</span><span class="p">,</span>
|
||||
<span class="n">policy_string</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">mx_host_patterns</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">failure_details</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">policies</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="n">policy_type</span><span class="o">=</span><span class="n">policy_type</span><span class="p">,</span>
|
||||
<span class="n">policy_domain</span><span class="o">=</span><span class="n">policy_domain</span><span class="p">,</span>
|
||||
<span class="n">successful_session_count</span><span class="o">=</span><span class="n">successful_session_count</span><span class="p">,</span>
|
||||
<span class="n">failed_session_count</span><span class="o">=</span><span class="n">failed_session_count</span><span class="p">,</span>
|
||||
<span class="n">policy_string</span><span class="o">=</span><span class="n">policy_string</span><span class="p">,</span>
|
||||
<span class="n">mx_host_patterns</span><span class="o">=</span><span class="n">mx_host_patterns</span><span class="p">,</span>
|
||||
<span class="n">failure_details</span><span class="o">=</span><span class="n">failure_details</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="AlreadySaved"><a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.AlreadySaved">[docs]</a><span class="k">class</span> <span class="nc">AlreadySaved</span><span class="p">(</span><span class="ne">ValueError</span><span class="p">):</span>
|
||||
<div class="viewcode-block" id="AlreadySaved">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.AlreadySaved">[docs]</a>
|
||||
<span class="k">class</span> <span class="nc">AlreadySaved</span><span class="p">(</span><span class="ne">ValueError</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Raised when a report to be saved matches an existing report"""</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="set_hosts"><a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.set_hosts">[docs]</a><span class="k">def</span> <span class="nf">set_hosts</span><span class="p">(</span><span class="n">hosts</span><span class="p">,</span> <span class="n">use_ssl</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">ssl_cert_path</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">username</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">password</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">apiKey</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mf">60.0</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="set_hosts">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.set_hosts">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">set_hosts</span><span class="p">(</span>
|
||||
<span class="n">hosts</span><span class="p">,</span>
|
||||
<span class="n">use_ssl</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">ssl_cert_path</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">username</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">password</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">apiKey</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">timeout</span><span class="o">=</span><span class="mf">60.0</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Sets the OpenSearch hosts to use</span>
|
||||
|
||||
@@ -339,25 +368,25 @@
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">hosts</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
|
||||
<span class="n">hosts</span> <span class="o">=</span> <span class="p">[</span><span class="n">hosts</span><span class="p">]</span>
|
||||
<span class="n">conn_params</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s2">"hosts"</span><span class="p">:</span> <span class="n">hosts</span><span class="p">,</span>
|
||||
<span class="s2">"timeout"</span><span class="p">:</span> <span class="n">timeout</span>
|
||||
<span class="p">}</span>
|
||||
<span class="n">conn_params</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"hosts"</span><span class="p">:</span> <span class="n">hosts</span><span class="p">,</span> <span class="s2">"timeout"</span><span class="p">:</span> <span class="n">timeout</span><span class="p">}</span>
|
||||
<span class="k">if</span> <span class="n">use_ssl</span><span class="p">:</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s1">'use_ssl'</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s2">"use_ssl"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="k">if</span> <span class="n">ssl_cert_path</span><span class="p">:</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s1">'verify_certs'</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s1">'ca_certs'</span><span class="p">]</span> <span class="o">=</span> <span class="n">ssl_cert_path</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s2">"verify_certs"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s2">"ca_certs"</span><span class="p">]</span> <span class="o">=</span> <span class="n">ssl_cert_path</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s1">'verify_certs'</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s2">"verify_certs"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
<span class="k">if</span> <span class="n">username</span><span class="p">:</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s1">'http_auth'</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span><span class="n">username</span><span class="o">+</span><span class="s2">":"</span><span class="o">+</span><span class="n">password</span><span class="p">)</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s2">"http_auth"</span><span class="p">]</span> <span class="o">=</span> <span class="n">username</span> <span class="o">+</span> <span class="s2">":"</span> <span class="o">+</span> <span class="n">password</span>
|
||||
<span class="k">if</span> <span class="n">apiKey</span><span class="p">:</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s1">'api_key'</span><span class="p">]</span> <span class="o">=</span> <span class="n">apiKey</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s2">"api_key"</span><span class="p">]</span> <span class="o">=</span> <span class="n">apiKey</span>
|
||||
<span class="n">connections</span><span class="o">.</span><span class="n">create_connection</span><span class="p">(</span><span class="o">**</span><span class="n">conn_params</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="create_indexes"><a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.create_indexes">[docs]</a><span class="k">def</span> <span class="nf">create_indexes</span><span class="p">(</span><span class="n">names</span><span class="p">,</span> <span class="n">settings</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="create_indexes">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.create_indexes">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">create_indexes</span><span class="p">(</span><span class="n">names</span><span class="p">,</span> <span class="n">settings</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Create OpenSearch indexes</span>
|
||||
|
||||
@@ -372,17 +401,18 @@
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">index</span><span class="o">.</span><span class="n">exists</span><span class="p">():</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Creating OpenSearch index: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">name</span><span class="p">))</span>
|
||||
<span class="k">if</span> <span class="n">settings</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">index</span><span class="o">.</span><span class="n">settings</span><span class="p">(</span><span class="n">number_of_shards</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
|
||||
<span class="n">index</span><span class="o">.</span><span class="n">settings</span><span class="p">(</span><span class="n">number_of_shards</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">number_of_replicas</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">index</span><span class="o">.</span><span class="n">settings</span><span class="p">(</span><span class="o">**</span><span class="n">settings</span><span class="p">)</span>
|
||||
<span class="n">index</span><span class="o">.</span><span class="n">create</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span>
|
||||
<span class="s2">"OpenSearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span><span class="s2">"OpenSearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="migrate_indexes"><a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.migrate_indexes">[docs]</a><span class="k">def</span> <span class="nf">migrate_indexes</span><span class="p">(</span><span class="n">aggregate_indexes</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">forensic_indexes</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="migrate_indexes">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.migrate_indexes">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">migrate_indexes</span><span class="p">(</span><span class="n">aggregate_indexes</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">forensic_indexes</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Updates index mappings</span>
|
||||
|
||||
@@ -411,33 +441,34 @@
|
||||
<span class="n">fo_type</span> <span class="o">=</span> <span class="n">fo_mapping</span><span class="p">[</span><span class="s2">"type"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="n">fo_type</span> <span class="o">==</span> <span class="s2">"long"</span><span class="p">:</span>
|
||||
<span class="n">new_index_name</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">-v</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">aggregate_index_name</span><span class="p">,</span> <span class="n">version</span><span class="p">)</span>
|
||||
<span class="n">body</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"properties"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"published_policy.fo"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"type"</span><span class="p">:</span> <span class="s2">"text"</span><span class="p">,</span>
|
||||
<span class="s2">"fields"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"keyword"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"type"</span><span class="p">:</span> <span class="s2">"keyword"</span><span class="p">,</span>
|
||||
<span class="s2">"ignore_above"</span><span class="p">:</span> <span class="mi">256</span>
|
||||
<span class="n">body</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s2">"properties"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"published_policy.fo"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"type"</span><span class="p">:</span> <span class="s2">"text"</span><span class="p">,</span>
|
||||
<span class="s2">"fields"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"keyword"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"type"</span><span class="p">:</span> <span class="s2">"keyword"</span><span class="p">,</span> <span class="s2">"ignore_above"</span><span class="p">:</span> <span class="mi">256</span><span class="p">}},</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">}</span>
|
||||
<span class="n">Index</span><span class="p">(</span><span class="n">new_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">create</span><span class="p">()</span>
|
||||
<span class="n">Index</span><span class="p">(</span><span class="n">new_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">put_mapping</span><span class="p">(</span><span class="n">doc_type</span><span class="o">=</span><span class="n">doc</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="n">body</span><span class="p">)</span>
|
||||
<span class="n">reindex</span><span class="p">(</span><span class="n">connections</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(),</span> <span class="n">aggregate_index_name</span><span class="p">,</span>
|
||||
<span class="n">new_index_name</span><span class="p">)</span>
|
||||
<span class="n">reindex</span><span class="p">(</span><span class="n">connections</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(),</span> <span class="n">aggregate_index_name</span><span class="p">,</span> <span class="n">new_index_name</span><span class="p">)</span>
|
||||
<span class="n">Index</span><span class="p">(</span><span class="n">aggregate_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">delete</span><span class="p">()</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">forensic_index</span> <span class="ow">in</span> <span class="n">forensic_indexes</span><span class="p">:</span>
|
||||
<span class="k">pass</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="save_aggregate_report_to_opensearch"><a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.save_aggregate_report_to_opensearch">[docs]</a><span class="k">def</span> <span class="nf">save_aggregate_report_to_opensearch</span><span class="p">(</span><span class="n">aggregate_report</span><span class="p">,</span>
|
||||
<span class="n">index_suffix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">index_prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">monthly_indexes</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="mi">0</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="save_aggregate_report_to_opensearch">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.save_aggregate_report_to_opensearch">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">save_aggregate_report_to_opensearch</span><span class="p">(</span>
|
||||
<span class="n">aggregate_report</span><span class="p">,</span>
|
||||
<span class="n">index_suffix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">index_prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">monthly_indexes</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Saves a parsed DMARC aggregate report to OpenSearch</span>
|
||||
|
||||
@@ -458,10 +489,8 @@
|
||||
<span class="n">org_name</span> <span class="o">=</span> <span class="n">metadata</span><span class="p">[</span><span class="s2">"org_name"</span><span class="p">]</span>
|
||||
<span class="n">report_id</span> <span class="o">=</span> <span class="n">metadata</span><span class="p">[</span><span class="s2">"report_id"</span><span class="p">]</span>
|
||||
<span class="n">domain</span> <span class="o">=</span> <span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"domain"</span><span class="p">]</span>
|
||||
<span class="n">begin_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">metadata</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">],</span>
|
||||
<span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="n">end_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">metadata</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">],</span>
|
||||
<span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="n">begin_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">metadata</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="n">end_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">metadata</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="n">begin_date_human</span> <span class="o">=</span> <span class="n">begin_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%SZ"</span><span class="p">)</span>
|
||||
<span class="n">end_date_human</span> <span class="o">=</span> <span class="n">end_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%SZ"</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">monthly_indexes</span><span class="p">:</span>
|
||||
@@ -470,8 +499,7 @@
|
||||
<span class="n">index_date</span> <span class="o">=</span> <span class="n">begin_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">]</span> <span class="o">=</span> <span class="n">begin_date</span>
|
||||
<span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">]</span> <span class="o">=</span> <span class="n">end_date</span>
|
||||
<span class="n">date_range</span> <span class="o">=</span> <span class="p">[</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">],</span>
|
||||
<span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">]]</span>
|
||||
<span class="n">date_range</span> <span class="o">=</span> <span class="p">[</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">],</span> <span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">]]</span>
|
||||
|
||||
<span class="n">org_name_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match_phrase</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">org_name</span><span class="o">=</span><span class="n">org_name</span><span class="p">)))</span>
|
||||
<span class="n">report_id_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match_phrase</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">report_id</span><span class="o">=</span><span class="n">report_id</span><span class="p">)))</span>
|
||||
@@ -493,18 +521,20 @@
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">existing</span> <span class="o">=</span> <span class="n">search</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error_</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span><span class="s2">"OpenSearch's search for existing report </span><span class="se">\</span>
|
||||
<span class="s2"> error: </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error_</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span>
|
||||
<span class="s2">"OpenSearch's search for existing report </span><span class="se">\</span>
|
||||
<span class="s2"> error: </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error_</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">existing</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">AlreadySaved</span><span class="p">(</span><span class="s2">"An aggregate report ID </span><span class="si">{0}</span><span class="s2"> from </span><span class="si">{1}</span><span class="s2"> about </span><span class="si">{2}</span><span class="s2"> "</span>
|
||||
<span class="s2">"with a date range of </span><span class="si">{3}</span><span class="s2"> UTC to </span><span class="si">{4}</span><span class="s2"> UTC already "</span>
|
||||
<span class="s2">"exists in "</span>
|
||||
<span class="s2">"OpenSearch"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">report_id</span><span class="p">,</span>
|
||||
<span class="n">org_name</span><span class="p">,</span>
|
||||
<span class="n">domain</span><span class="p">,</span>
|
||||
<span class="n">begin_date_human</span><span class="p">,</span>
|
||||
<span class="n">end_date_human</span><span class="p">))</span>
|
||||
<span class="k">raise</span> <span class="n">AlreadySaved</span><span class="p">(</span>
|
||||
<span class="s2">"An aggregate report ID </span><span class="si">{0}</span><span class="s2"> from </span><span class="si">{1}</span><span class="s2"> about </span><span class="si">{2}</span><span class="s2"> "</span>
|
||||
<span class="s2">"with a date range of </span><span class="si">{3}</span><span class="s2"> UTC to </span><span class="si">{4}</span><span class="s2"> UTC already "</span>
|
||||
<span class="s2">"exists in "</span>
|
||||
<span class="s2">"OpenSearch"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">report_id</span><span class="p">,</span> <span class="n">org_name</span><span class="p">,</span> <span class="n">domain</span><span class="p">,</span> <span class="n">begin_date_human</span><span class="p">,</span> <span class="n">end_date_human</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">published_policy</span> <span class="o">=</span> <span class="n">_PublishedPolicy</span><span class="p">(</span>
|
||||
<span class="n">domain</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"domain"</span><span class="p">],</span>
|
||||
<span class="n">adkim</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"adkim"</span><span class="p">],</span>
|
||||
@@ -512,7 +542,7 @@
|
||||
<span class="n">p</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"p"</span><span class="p">],</span>
|
||||
<span class="n">sp</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"sp"</span><span class="p">],</span>
|
||||
<span class="n">pct</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"pct"</span><span class="p">],</span>
|
||||
<span class="n">fo</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"fo"</span><span class="p">]</span>
|
||||
<span class="n">fo</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"fo"</span><span class="p">],</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">record</span> <span class="ow">in</span> <span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"records"</span><span class="p">]:</span>
|
||||
@@ -535,28 +565,33 @@
|
||||
<span class="n">source_name</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"name"</span><span class="p">],</span>
|
||||
<span class="n">message_count</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"count"</span><span class="p">],</span>
|
||||
<span class="n">disposition</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"disposition"</span><span class="p">],</span>
|
||||
<span class="n">dkim_aligned</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"dkim"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span>
|
||||
<span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"dkim"</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s2">"pass"</span><span class="p">,</span>
|
||||
<span class="n">spf_aligned</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span>
|
||||
<span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s2">"pass"</span><span class="p">,</span>
|
||||
<span class="n">dkim_aligned</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"dkim"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
|
||||
<span class="ow">and</span> <span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"dkim"</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s2">"pass"</span><span class="p">,</span>
|
||||
<span class="n">spf_aligned</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
|
||||
<span class="ow">and</span> <span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s2">"pass"</span><span class="p">,</span>
|
||||
<span class="n">header_from</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"header_from"</span><span class="p">],</span>
|
||||
<span class="n">envelope_from</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"envelope_from"</span><span class="p">],</span>
|
||||
<span class="n">envelope_to</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"envelope_to"</span><span class="p">]</span>
|
||||
<span class="n">envelope_to</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"envelope_to"</span><span class="p">],</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">override</span> <span class="ow">in</span> <span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"policy_override_reasons"</span><span class="p">]:</span>
|
||||
<span class="n">agg_doc</span><span class="o">.</span><span class="n">add_policy_override</span><span class="p">(</span><span class="n">type_</span><span class="o">=</span><span class="n">override</span><span class="p">[</span><span class="s2">"type"</span><span class="p">],</span>
|
||||
<span class="n">comment</span><span class="o">=</span><span class="n">override</span><span class="p">[</span><span class="s2">"comment"</span><span class="p">])</span>
|
||||
<span class="n">agg_doc</span><span class="o">.</span><span class="n">add_policy_override</span><span class="p">(</span>
|
||||
<span class="n">type_</span><span class="o">=</span><span class="n">override</span><span class="p">[</span><span class="s2">"type"</span><span class="p">],</span> <span class="n">comment</span><span class="o">=</span><span class="n">override</span><span class="p">[</span><span class="s2">"comment"</span><span class="p">]</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">dkim_result</span> <span class="ow">in</span> <span class="n">record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">][</span><span class="s2">"dkim"</span><span class="p">]:</span>
|
||||
<span class="n">agg_doc</span><span class="o">.</span><span class="n">add_dkim_result</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">],</span>
|
||||
<span class="n">selector</span><span class="o">=</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">"selector"</span><span class="p">],</span>
|
||||
<span class="n">result</span><span class="o">=</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">])</span>
|
||||
<span class="n">agg_doc</span><span class="o">.</span><span class="n">add_dkim_result</span><span class="p">(</span>
|
||||
<span class="n">domain</span><span class="o">=</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">],</span>
|
||||
<span class="n">selector</span><span class="o">=</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">"selector"</span><span class="p">],</span>
|
||||
<span class="n">result</span><span class="o">=</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">],</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">spf_result</span> <span class="ow">in</span> <span class="n">record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">]:</span>
|
||||
<span class="n">agg_doc</span><span class="o">.</span><span class="n">add_spf_result</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">],</span>
|
||||
<span class="n">scope</span><span class="o">=</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">"scope"</span><span class="p">],</span>
|
||||
<span class="n">result</span><span class="o">=</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">])</span>
|
||||
<span class="n">agg_doc</span><span class="o">.</span><span class="n">add_spf_result</span><span class="p">(</span>
|
||||
<span class="n">domain</span><span class="o">=</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">],</span>
|
||||
<span class="n">scope</span><span class="o">=</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">"scope"</span><span class="p">],</span>
|
||||
<span class="n">result</span><span class="o">=</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">],</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"dmarc_aggregate"</span>
|
||||
<span class="k">if</span> <span class="n">index_suffix</span><span class="p">:</span>
|
||||
@@ -564,41 +599,46 @@
|
||||
<span class="k">if</span> <span class="n">index_prefix</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">index</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">-</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_date</span><span class="p">)</span>
|
||||
<span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span><span class="p">)</span>
|
||||
<span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span> <span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">create_indexes</span><span class="p">([</span><span class="n">index</span><span class="p">],</span> <span class="n">index_settings</span><span class="p">)</span>
|
||||
<span class="n">agg_doc</span><span class="o">.</span><span class="n">meta</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">index</span>
|
||||
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">agg_doc</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span>
|
||||
<span class="s2">"OpenSearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span><span class="s2">"OpenSearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="save_forensic_report_to_opensearch"><a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.save_forensic_report_to_opensearch">[docs]</a><span class="k">def</span> <span class="nf">save_forensic_report_to_opensearch</span><span class="p">(</span><span class="n">forensic_report</span><span class="p">,</span>
|
||||
<span class="n">index_suffix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">index_prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">monthly_indexes</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="mi">0</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="save_forensic_report_to_opensearch">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.save_forensic_report_to_opensearch">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">save_forensic_report_to_opensearch</span><span class="p">(</span>
|
||||
<span class="n">forensic_report</span><span class="p">,</span>
|
||||
<span class="n">index_suffix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">index_prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">monthly_indexes</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Saves a parsed DMARC forensic report to OpenSearch</span>
|
||||
<span class="sd"> Saves a parsed DMARC forensic report to OpenSearch</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> forensic_report (OrderedDict): A parsed forensic report</span>
|
||||
<span class="sd"> index_suffix (str): The suffix of the name of the index to save to</span>
|
||||
<span class="sd"> index_prefix (str): The prefix of the name of the index to save to</span>
|
||||
<span class="sd"> monthly_indexes (bool): Use monthly indexes instead of daily</span>
|
||||
<span class="sd"> indexes</span>
|
||||
<span class="sd"> number_of_shards (int): The number of shards to use in the index</span>
|
||||
<span class="sd"> number_of_replicas (int): The number of replicas to use in the</span>
|
||||
<span class="sd"> index</span>
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> forensic_report (OrderedDict): A parsed forensic report</span>
|
||||
<span class="sd"> index_suffix (str): The suffix of the name of the index to save to</span>
|
||||
<span class="sd"> index_prefix (str): The prefix of the name of the index to save to</span>
|
||||
<span class="sd"> monthly_indexes (bool): Use monthly indexes instead of daily</span>
|
||||
<span class="sd"> indexes</span>
|
||||
<span class="sd"> number_of_shards (int): The number of shards to use in the index</span>
|
||||
<span class="sd"> number_of_replicas (int): The number of replicas to use in the</span>
|
||||
<span class="sd"> index</span>
|
||||
|
||||
<span class="sd"> Raises:</span>
|
||||
<span class="sd"> AlreadySaved</span>
|
||||
<span class="sd"> Raises:</span>
|
||||
<span class="sd"> AlreadySaved</span>
|
||||
|
||||
<span class="sd"> """</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Saving forensic report to OpenSearch"</span><span class="p">)</span>
|
||||
<span class="n">forensic_report</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
<span class="n">sample_date</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
@@ -643,12 +683,12 @@
|
||||
<span class="n">existing</span> <span class="o">=</span> <span class="n">search</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
|
||||
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">existing</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">AlreadySaved</span><span class="p">(</span><span class="s2">"A forensic sample to </span><span class="si">{0}</span><span class="s2"> from </span><span class="si">{1}</span><span class="s2"> "</span>
|
||||
<span class="s2">"with a subject of </span><span class="si">{2}</span><span class="s2"> and arrival date of </span><span class="si">{3}</span><span class="s2"> "</span>
|
||||
<span class="s2">"already exists in "</span>
|
||||
<span class="s2">"OpenSearch"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">to_</span><span class="p">,</span> <span class="n">from_</span><span class="p">,</span> <span class="n">subject</span><span class="p">,</span> <span class="n">arrival_date_human</span>
|
||||
<span class="p">))</span>
|
||||
<span class="k">raise</span> <span class="n">AlreadySaved</span><span class="p">(</span>
|
||||
<span class="s2">"A forensic sample to </span><span class="si">{0}</span><span class="s2"> from </span><span class="si">{1}</span><span class="s2"> "</span>
|
||||
<span class="s2">"with a subject of </span><span class="si">{2}</span><span class="s2"> and arrival date of </span><span class="si">{3}</span><span class="s2"> "</span>
|
||||
<span class="s2">"already exists in "</span>
|
||||
<span class="s2">"OpenSearch"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">to_</span><span class="p">,</span> <span class="n">from_</span><span class="p">,</span> <span class="n">subject</span><span class="p">,</span> <span class="n">arrival_date_human</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">parsed_sample</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">]</span>
|
||||
<span class="n">sample</span> <span class="o">=</span> <span class="n">_ForensicSampleDoc</span><span class="p">(</span>
|
||||
@@ -658,25 +698,25 @@
|
||||
<span class="n">date</span><span class="o">=</span><span class="n">sample_date</span><span class="p">,</span>
|
||||
<span class="n">subject</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"subject"</span><span class="p">],</span>
|
||||
<span class="n">filename_safe_subject</span><span class="o">=</span><span class="n">parsed_sample</span><span class="p">[</span><span class="s2">"filename_safe_subject"</span><span class="p">],</span>
|
||||
<span class="n">body</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"body"</span><span class="p">]</span>
|
||||
<span class="n">body</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"body"</span><span class="p">],</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"to"</span><span class="p">]:</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_to</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span>
|
||||
<span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">])</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_to</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">])</span>
|
||||
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"reply_to"</span><span class="p">]:</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_reply_to</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span>
|
||||
<span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">])</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_reply_to</span><span class="p">(</span>
|
||||
<span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">]</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"cc"</span><span class="p">]:</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_cc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span>
|
||||
<span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">])</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_cc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">])</span>
|
||||
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"bcc"</span><span class="p">]:</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_bcc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span>
|
||||
<span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">])</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_bcc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">])</span>
|
||||
<span class="k">for</span> <span class="n">attachment</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"attachments"</span><span class="p">]:</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_attachment</span><span class="p">(</span><span class="n">filename</span><span class="o">=</span><span class="n">attachment</span><span class="p">[</span><span class="s2">"filename"</span><span class="p">],</span>
|
||||
<span class="n">content_type</span><span class="o">=</span><span class="n">attachment</span><span class="p">[</span><span class="s2">"mail_content_type"</span><span class="p">],</span>
|
||||
<span class="n">sha256</span><span class="o">=</span><span class="n">attachment</span><span class="p">[</span><span class="s2">"sha256"</span><span class="p">])</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_attachment</span><span class="p">(</span>
|
||||
<span class="n">filename</span><span class="o">=</span><span class="n">attachment</span><span class="p">[</span><span class="s2">"filename"</span><span class="p">],</span>
|
||||
<span class="n">content_type</span><span class="o">=</span><span class="n">attachment</span><span class="p">[</span><span class="s2">"mail_content_type"</span><span class="p">],</span>
|
||||
<span class="n">sha256</span><span class="o">=</span><span class="n">attachment</span><span class="p">[</span><span class="s2">"sha256"</span><span class="p">],</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">forensic_doc</span> <span class="o">=</span> <span class="n">_ForensicReportDoc</span><span class="p">(</span>
|
||||
<span class="n">feedback_type</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"feedback_type"</span><span class="p">],</span>
|
||||
@@ -692,12 +732,11 @@
|
||||
<span class="n">source_country</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"country"</span><span class="p">],</span>
|
||||
<span class="n">source_reverse_dns</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"reverse_dns"</span><span class="p">],</span>
|
||||
<span class="n">source_base_domain</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"base_domain"</span><span class="p">],</span>
|
||||
<span class="n">authentication_mechanisms</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span>
|
||||
<span class="s2">"authentication_mechanisms"</span><span class="p">],</span>
|
||||
<span class="n">authentication_mechanisms</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"authentication_mechanisms"</span><span class="p">],</span>
|
||||
<span class="n">auth_failure</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"auth_failure"</span><span class="p">],</span>
|
||||
<span class="n">dkim_domain</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"dkim_domain"</span><span class="p">],</span>
|
||||
<span class="n">original_rcpt_to</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"original_rcpt_to"</span><span class="p">],</span>
|
||||
<span class="n">sample</span><span class="o">=</span><span class="n">sample</span>
|
||||
<span class="n">sample</span><span class="o">=</span><span class="n">sample</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"dmarc_forensic"</span>
|
||||
@@ -710,26 +749,32 @@
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">index_date</span> <span class="o">=</span> <span class="n">arrival_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">-</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_date</span><span class="p">)</span>
|
||||
<span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span><span class="p">)</span>
|
||||
<span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span> <span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">create_indexes</span><span class="p">([</span><span class="n">index</span><span class="p">],</span> <span class="n">index_settings</span><span class="p">)</span>
|
||||
<span class="n">forensic_doc</span><span class="o">.</span><span class="n">meta</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">index</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">forensic_doc</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span>
|
||||
<span class="s2">"OpenSearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span><span class="s2">"OpenSearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
||||
<span class="k">except</span> <span class="ne">KeyError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidForensicReport</span><span class="p">(</span>
|
||||
<span class="s2">"Forensic report missing required field: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
<span class="s2">"Forensic report missing required field: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="save_smtp_tls_report_to_opensearch"><a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.save_smtp_tls_report_to_opensearch">[docs]</a><span class="k">def</span> <span class="nf">save_smtp_tls_report_to_opensearch</span><span class="p">(</span><span class="n">report</span><span class="p">,</span>
|
||||
<span class="n">index_suffix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">index_prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">monthly_indexes</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="mi">0</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="save_smtp_tls_report_to_opensearch">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.save_smtp_tls_report_to_opensearch">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">save_smtp_tls_report_to_opensearch</span><span class="p">(</span>
|
||||
<span class="n">report</span><span class="p">,</span>
|
||||
<span class="n">index_suffix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">index_prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">monthly_indexes</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Saves a parsed SMTP TLS report to OpenSearch</span>
|
||||
|
||||
@@ -747,10 +792,8 @@
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Saving aggregate report to OpenSearch"</span><span class="p">)</span>
|
||||
<span class="n">org_name</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"org_name"</span><span class="p">]</span>
|
||||
<span class="n">report_id</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_id"</span><span class="p">]</span>
|
||||
<span class="n">begin_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">],</span>
|
||||
<span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="n">end_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">],</span>
|
||||
<span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="n">begin_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="n">end_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="n">begin_date_human</span> <span class="o">=</span> <span class="n">begin_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%SZ"</span><span class="p">)</span>
|
||||
<span class="n">end_date_human</span> <span class="o">=</span> <span class="n">end_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%SZ"</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">monthly_indexes</span><span class="p">:</span>
|
||||
@@ -779,15 +822,19 @@
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">existing</span> <span class="o">=</span> <span class="n">search</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error_</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span><span class="s2">"OpenSearch's search for existing report </span><span class="se">\</span>
|
||||
<span class="s2"> error: </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error_</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span>
|
||||
<span class="s2">"OpenSearch's search for existing report </span><span class="se">\</span>
|
||||
<span class="s2"> error: </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error_</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">existing</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">AlreadySaved</span><span class="p">(</span><span class="sa">f</span><span class="s2">"An SMTP TLS report ID </span><span class="si">{</span><span class="n">report_id</span><span class="si">}</span><span class="s2"> from "</span>
|
||||
<span class="sa">f</span><span class="s2">" </span><span class="si">{</span><span class="n">org_name</span><span class="si">}</span><span class="s2"> with a date range of "</span>
|
||||
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">begin_date_human</span><span class="si">}</span><span class="s2"> UTC to "</span>
|
||||
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">end_date_human</span><span class="si">}</span><span class="s2"> UTC already "</span>
|
||||
<span class="s2">"exists in OpenSearch"</span><span class="p">)</span>
|
||||
<span class="k">raise</span> <span class="n">AlreadySaved</span><span class="p">(</span>
|
||||
<span class="sa">f</span><span class="s2">"An SMTP TLS report ID </span><span class="si">{</span><span class="n">report_id</span><span class="si">}</span><span class="s2"> from "</span>
|
||||
<span class="sa">f</span><span class="s2">" </span><span class="si">{</span><span class="n">org_name</span><span class="si">}</span><span class="s2"> with a date range of "</span>
|
||||
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">begin_date_human</span><span class="si">}</span><span class="s2"> UTC to "</span>
|
||||
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">end_date_human</span><span class="si">}</span><span class="s2"> UTC already "</span>
|
||||
<span class="s2">"exists in OpenSearch"</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"smtp_tls"</span>
|
||||
<span class="k">if</span> <span class="n">index_suffix</span><span class="p">:</span>
|
||||
@@ -795,8 +842,9 @@
|
||||
<span class="k">if</span> <span class="n">index_prefix</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">index</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">-</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_date</span><span class="p">)</span>
|
||||
<span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span><span class="p">)</span>
|
||||
<span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span> <span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">smtp_tls_doc</span> <span class="o">=</span> <span class="n">_SMTPTLSFailureReportDoc</span><span class="p">(</span>
|
||||
<span class="n">organization_name</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">"organization_name"</span><span class="p">],</span>
|
||||
@@ -804,10 +852,10 @@
|
||||
<span class="n">date_begin</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">"date_begin"</span><span class="p">],</span>
|
||||
<span class="n">date_end</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">"date_end"</span><span class="p">],</span>
|
||||
<span class="n">contact_info</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">"contact_info"</span><span class="p">],</span>
|
||||
<span class="n">report_id</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">"report_id"</span><span class="p">]</span>
|
||||
<span class="n">report_id</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">"report_id"</span><span class="p">],</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">policy</span> <span class="ow">in</span> <span class="n">report</span><span class="p">[</span><span class="s1">'policies'</span><span class="p">]:</span>
|
||||
<span class="k">for</span> <span class="n">policy</span> <span class="ow">in</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policies"</span><span class="p">]:</span>
|
||||
<span class="n">policy_strings</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="n">mx_host_patterns</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="k">if</span> <span class="s2">"policy_strings"</span> <span class="ow">in</span> <span class="n">policy</span><span class="p">:</span>
|
||||
@@ -818,7 +866,7 @@
|
||||
<span class="n">policy_domain</span><span class="o">=</span><span class="n">policy</span><span class="p">[</span><span class="s2">"policy_domain"</span><span class="p">],</span>
|
||||
<span class="n">policy_type</span><span class="o">=</span><span class="n">policy</span><span class="p">[</span><span class="s2">"policy_type"</span><span class="p">],</span>
|
||||
<span class="n">policy_string</span><span class="o">=</span><span class="n">policy_strings</span><span class="p">,</span>
|
||||
<span class="n">mx_host_patterns</span><span class="o">=</span><span class="n">mx_host_patterns</span>
|
||||
<span class="n">mx_host_patterns</span><span class="o">=</span><span class="n">mx_host_patterns</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">if</span> <span class="s2">"failure_details"</span> <span class="ow">in</span> <span class="n">policy</span><span class="p">:</span>
|
||||
<span class="n">failure_details</span> <span class="o">=</span> <span class="n">policy</span><span class="p">[</span><span class="s2">"failure_details"</span><span class="p">]</span>
|
||||
@@ -826,11 +874,11 @@
|
||||
<span class="n">additional_information_uri</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="n">failure_reason_code</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="k">if</span> <span class="s2">"receiving_mx_hostname"</span> <span class="ow">in</span> <span class="n">failure_details</span><span class="p">:</span>
|
||||
<span class="n">receiving_mx_hostname</span> <span class="o">=</span> <span class="n">failure_details</span><span class="p">[</span>
|
||||
<span class="s2">"receiving_mx_hostname"</span><span class="p">]</span>
|
||||
<span class="n">receiving_mx_hostname</span> <span class="o">=</span> <span class="n">failure_details</span><span class="p">[</span><span class="s2">"receiving_mx_hostname"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="s2">"additional_information_uri"</span> <span class="ow">in</span> <span class="n">failure_details</span><span class="p">:</span>
|
||||
<span class="n">additional_information_uri</span> <span class="o">=</span> <span class="n">failure_details</span><span class="p">[</span>
|
||||
<span class="s2">"additional_information_uri"</span><span class="p">]</span>
|
||||
<span class="s2">"additional_information_uri"</span>
|
||||
<span class="p">]</span>
|
||||
<span class="k">if</span> <span class="s2">"failure_reason_code"</span> <span class="ow">in</span> <span class="n">failure_details</span><span class="p">:</span>
|
||||
<span class="n">failure_reason_code</span> <span class="o">=</span> <span class="n">failure_details</span><span class="p">[</span><span class="s2">"failure_reason_code"</span><span class="p">]</span>
|
||||
<span class="n">policy_doc</span><span class="o">.</span><span class="n">add_failure_details</span><span class="p">(</span>
|
||||
@@ -841,7 +889,7 @@
|
||||
<span class="n">failed_session_count</span><span class="o">=</span><span class="n">failure_details</span><span class="p">[</span><span class="s2">"failed_session_count"</span><span class="p">],</span>
|
||||
<span class="n">receiving_mx_hostname</span><span class="o">=</span><span class="n">receiving_mx_hostname</span><span class="p">,</span>
|
||||
<span class="n">additional_information_uri</span><span class="o">=</span><span class="n">additional_information_uri</span><span class="p">,</span>
|
||||
<span class="n">failure_reason_code</span><span class="o">=</span><span class="n">failure_reason_code</span>
|
||||
<span class="n">failure_reason_code</span><span class="o">=</span><span class="n">failure_reason_code</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">smtp_tls_doc</span><span class="o">.</span><span class="n">policies</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">policy_doc</span><span class="p">)</span>
|
||||
|
||||
@@ -851,8 +899,8 @@
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">smtp_tls_doc</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span>
|
||||
<span class="s2">"OpenSearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span><span class="s2">"OpenSearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
|
||||
</pre></div>
|
||||
|
||||
</div>
|
||||
|
||||
@@ -1,23 +1,20 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="../../">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc.splunk — parsedmarc 8.15.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css" />
|
||||
<title>parsedmarc.splunk — parsedmarc 8.15.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=80d5e7a1" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="../../_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="../../" id="documentation_options" src="../../_static/documentation_options.js"></script>
|
||||
<script src="../../_static/jquery.js"></script>
|
||||
<script src="../../_static/underscore.js"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="../../_static/doctools.js"></script>
|
||||
<script src="../../_static/sphinx_highlight.js"></script>
|
||||
<script src="../../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../../_static/documentation_options.js?v=6add7570"></script>
|
||||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="../../_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="../../genindex.html" />
|
||||
<link rel="search" title="Search" href="../../search.html" />
|
||||
@@ -34,9 +31,6 @@
|
||||
<a href="../../index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.15.0
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
@@ -100,18 +94,24 @@
|
||||
<span class="n">urllib3</span><span class="o">.</span><span class="n">disable_warnings</span><span class="p">(</span><span class="n">urllib3</span><span class="o">.</span><span class="n">exceptions</span><span class="o">.</span><span class="n">InsecureRequestWarning</span><span class="p">)</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="SplunkError"><a class="viewcode-back" href="../../api.html#parsedmarc.splunk.SplunkError">[docs]</a><span class="k">class</span> <span class="nc">SplunkError</span><span class="p">(</span><span class="ne">RuntimeError</span><span class="p">):</span>
|
||||
<div class="viewcode-block" id="SplunkError">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.splunk.SplunkError">[docs]</a>
|
||||
<span class="k">class</span> <span class="nc">SplunkError</span><span class="p">(</span><span class="ne">RuntimeError</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Raised when a Splunk API error occurs"""</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="HECClient"><a class="viewcode-back" href="../../api.html#parsedmarc.splunk.HECClient">[docs]</a><span class="k">class</span> <span class="nc">HECClient</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="HECClient">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.splunk.HECClient">[docs]</a>
|
||||
<span class="k">class</span> <span class="nc">HECClient</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""A client for a Splunk HTTP Events Collector (HEC)"""</span>
|
||||
|
||||
<span class="c1"># http://docs.splunk.com/Documentation/Splunk/latest/Data/AboutHEC</span>
|
||||
<span class="c1"># http://docs.splunk.com/Documentation/Splunk/latest/RESTREF/RESTinput#services.2Fcollector</span>
|
||||
|
||||
<span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">url</span><span class="p">,</span> <span class="n">access_token</span><span class="p">,</span> <span class="n">index</span><span class="p">,</span>
|
||||
<span class="n">source</span><span class="o">=</span><span class="s2">"parsedmarc"</span><span class="p">,</span> <span class="n">verify</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mi">60</span><span class="p">):</span>
|
||||
<span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="p">,</span> <span class="n">url</span><span class="p">,</span> <span class="n">access_token</span><span class="p">,</span> <span class="n">index</span><span class="p">,</span> <span class="n">source</span><span class="o">=</span><span class="s2">"parsedmarc"</span><span class="p">,</span> <span class="n">verify</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mi">60</span>
|
||||
<span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Initializes the HECClient</span>
|
||||
|
||||
@@ -125,8 +125,9 @@
|
||||
<span class="sd"> data before giving up</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">url</span> <span class="o">=</span> <span class="n">urlparse</span><span class="p">(</span><span class="n">url</span><span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">url</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">://</span><span class="si">{1}</span><span class="s2">/services/collector/event/1.0"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">url</span><span class="o">.</span><span class="n">scheme</span><span class="p">,</span>
|
||||
<span class="n">url</span><span class="o">.</span><span class="n">netloc</span><span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">url</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">://</span><span class="si">{1}</span><span class="s2">/services/collector/event/1.0"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">url</span><span class="o">.</span><span class="n">scheme</span><span class="p">,</span> <span class="n">url</span><span class="o">.</span><span class="n">netloc</span>
|
||||
<span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">access_token</span> <span class="o">=</span> <span class="n">access_token</span><span class="o">.</span><span class="n">lstrip</span><span class="p">(</span><span class="s2">"Splunk "</span><span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">index</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">host</span> <span class="o">=</span> <span class="n">socket</span><span class="o">.</span><span class="n">getfqdn</span><span class="p">()</span>
|
||||
@@ -134,15 +135,16 @@
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">session</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">Session</span><span class="p">()</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">timeout</span> <span class="o">=</span> <span class="n">timeout</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">verify</span> <span class="o">=</span> <span class="n">verify</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">_common_data</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="n">host</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">host</span><span class="p">,</span> <span class="n">source</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">source</span><span class="p">,</span>
|
||||
<span class="n">index</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">index</span><span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">_common_data</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="n">host</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">host</span><span class="p">,</span> <span class="n">source</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">source</span><span class="p">,</span> <span class="n">index</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">index</span><span class="p">)</span>
|
||||
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">headers</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s2">"User-Agent"</span><span class="p">:</span> <span class="s2">"parsedmarc/</span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">__version__</span><span class="p">),</span>
|
||||
<span class="s2">"Authorization"</span><span class="p">:</span> <span class="s2">"Splunk </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">access_token</span><span class="p">)</span>
|
||||
<span class="s2">"Authorization"</span><span class="p">:</span> <span class="s2">"Splunk </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">access_token</span><span class="p">),</span>
|
||||
<span class="p">}</span>
|
||||
|
||||
<div class="viewcode-block" id="HECClient.save_aggregate_reports_to_splunk"><a class="viewcode-back" href="../../api.html#parsedmarc.splunk.HECClient.save_aggregate_reports_to_splunk">[docs]</a> <span class="k">def</span> <span class="nf">save_aggregate_reports_to_splunk</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">aggregate_reports</span><span class="p">):</span>
|
||||
<div class="viewcode-block" id="HECClient.save_aggregate_reports_to_splunk">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.splunk.HECClient.save_aggregate_reports_to_splunk">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">save_aggregate_reports_to_splunk</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">aggregate_reports</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Saves aggregate DMARC reports to Splunk</span>
|
||||
|
||||
@@ -166,36 +168,26 @@
|
||||
<span class="k">for</span> <span class="n">metadata</span> <span class="ow">in</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">]:</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="n">metadata</span><span class="p">]</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">][</span><span class="n">metadata</span><span class="p">]</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"published_policy"</span><span class="p">]</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">]</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"source_ip_address"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span>
|
||||
<span class="s2">"ip_address"</span><span class="p">]</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"source_ip_address"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"ip_address"</span><span class="p">]</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"source_country"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"country"</span><span class="p">]</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"source_reverse_dns"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span>
|
||||
<span class="s2">"reverse_dns"</span><span class="p">]</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"source_base_domain"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span>
|
||||
<span class="s2">"base_domain"</span><span class="p">]</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"source_reverse_dns"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"reverse_dns"</span><span class="p">]</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"source_base_domain"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"base_domain"</span><span class="p">]</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"source_type"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"type"</span><span class="p">]</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"source_name"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"name"</span><span class="p">]</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"message_count"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"count"</span><span class="p">]</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"disposition"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span>
|
||||
<span class="s2">"disposition"</span>
|
||||
<span class="p">]</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"disposition"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"disposition"</span><span class="p">]</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"spf_aligned"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"alignment"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">]</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"dkim_aligned"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"alignment"</span><span class="p">][</span><span class="s2">"dkim"</span><span class="p">]</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"passed_dmarc"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"alignment"</span><span class="p">][</span><span class="s2">"dmarc"</span><span class="p">]</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"header_from"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span>
|
||||
<span class="s2">"header_from"</span><span class="p">]</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"envelope_from"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span>
|
||||
<span class="s2">"envelope_from"</span><span class="p">]</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"header_from"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"header_from"</span><span class="p">]</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"envelope_from"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"envelope_from"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="s2">"dkim"</span> <span class="ow">in</span> <span class="n">record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">]:</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"dkim_results"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">][</span>
|
||||
<span class="s2">"dkim"</span><span class="p">]</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"dkim_results"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">][</span><span class="s2">"dkim"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="s2">"spf"</span> <span class="ow">in</span> <span class="n">record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">]:</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"spf_results"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">][</span>
|
||||
<span class="s2">"spf"</span><span class="p">]</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"spf_results"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">]</span>
|
||||
|
||||
<span class="n">data</span><span class="p">[</span><span class="s2">"sourcetype"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"dmarc:aggregate"</span>
|
||||
<span class="n">timestamp</span> <span class="o">=</span> <span class="n">human_timestamp_to_unix_timestamp</span><span class="p">(</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">])</span>
|
||||
<span class="n">timestamp</span> <span class="o">=</span> <span class="n">human_timestamp_to_unix_timestamp</span><span class="p">(</span><span class="n">new_report</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">])</span>
|
||||
<span class="n">data</span><span class="p">[</span><span class="s2">"time"</span><span class="p">]</span> <span class="o">=</span> <span class="n">timestamp</span>
|
||||
<span class="n">data</span><span class="p">[</span><span class="s2">"event"</span><span class="p">]</span> <span class="o">=</span> <span class="n">new_report</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
<span class="n">json_str</span> <span class="o">+=</span> <span class="s2">"</span><span class="si">{0}</span><span class="se">\n</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">data</span><span class="p">))</span>
|
||||
@@ -203,15 +195,17 @@
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">verify</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Skipping certificate verification for Splunk HEC"</span><span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">post</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">url</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="n">json_str</span><span class="p">,</span>
|
||||
<span class="n">timeout</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="p">)</span>
|
||||
<span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">post</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">url</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="n">json_str</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="p">)</span>
|
||||
<span class="n">response</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">SplunkError</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="k">if</span> <span class="n">response</span><span class="p">[</span><span class="s2">"code"</span><span class="p">]</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">SplunkError</span><span class="p">(</span><span class="n">response</span><span class="p">[</span><span class="s2">"text"</span><span class="p">])</span></div>
|
||||
|
||||
<div class="viewcode-block" id="HECClient.save_forensic_reports_to_splunk"><a class="viewcode-back" href="../../api.html#parsedmarc.splunk.HECClient.save_forensic_reports_to_splunk">[docs]</a> <span class="k">def</span> <span class="nf">save_forensic_reports_to_splunk</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">forensic_reports</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="HECClient.save_forensic_reports_to_splunk">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.splunk.HECClient.save_forensic_reports_to_splunk">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">save_forensic_reports_to_splunk</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">forensic_reports</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Saves forensic DMARC reports to Splunk</span>
|
||||
|
||||
@@ -230,8 +224,7 @@
|
||||
<span class="k">for</span> <span class="n">report</span> <span class="ow">in</span> <span class="n">forensic_reports</span><span class="p">:</span>
|
||||
<span class="n">data</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_common_data</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
<span class="n">data</span><span class="p">[</span><span class="s2">"sourcetype"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"dmarc:forensic"</span>
|
||||
<span class="n">timestamp</span> <span class="o">=</span> <span class="n">human_timestamp_to_unix_timestamp</span><span class="p">(</span>
|
||||
<span class="n">report</span><span class="p">[</span><span class="s2">"arrival_date_utc"</span><span class="p">])</span>
|
||||
<span class="n">timestamp</span> <span class="o">=</span> <span class="n">human_timestamp_to_unix_timestamp</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">"arrival_date_utc"</span><span class="p">])</span>
|
||||
<span class="n">data</span><span class="p">[</span><span class="s2">"time"</span><span class="p">]</span> <span class="o">=</span> <span class="n">timestamp</span>
|
||||
<span class="n">data</span><span class="p">[</span><span class="s2">"event"</span><span class="p">]</span> <span class="o">=</span> <span class="n">report</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
<span class="n">json_str</span> <span class="o">+=</span> <span class="s2">"</span><span class="si">{0}</span><span class="se">\n</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">data</span><span class="p">))</span>
|
||||
@@ -239,15 +232,17 @@
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">verify</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Skipping certificate verification for Splunk HEC"</span><span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">post</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">url</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="n">json_str</span><span class="p">,</span>
|
||||
<span class="n">timeout</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="p">)</span>
|
||||
<span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">post</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">url</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="n">json_str</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="p">)</span>
|
||||
<span class="n">response</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">SplunkError</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="k">if</span> <span class="n">response</span><span class="p">[</span><span class="s2">"code"</span><span class="p">]</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">SplunkError</span><span class="p">(</span><span class="n">response</span><span class="p">[</span><span class="s2">"text"</span><span class="p">])</span></div>
|
||||
|
||||
<div class="viewcode-block" id="HECClient.save_smtp_tls_reports_to_splunk"><a class="viewcode-back" href="../../api.html#parsedmarc.splunk.HECClient.save_smtp_tls_reports_to_splunk">[docs]</a> <span class="k">def</span> <span class="nf">save_smtp_tls_reports_to_splunk</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">reports</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="HECClient.save_smtp_tls_reports_to_splunk">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.splunk.HECClient.save_smtp_tls_reports_to_splunk">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">save_smtp_tls_reports_to_splunk</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">reports</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Saves aggregate DMARC reports to Splunk</span>
|
||||
|
||||
@@ -267,8 +262,7 @@
|
||||
<span class="n">json_str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||
<span class="k">for</span> <span class="n">report</span> <span class="ow">in</span> <span class="n">reports</span><span class="p">:</span>
|
||||
<span class="n">data</span><span class="p">[</span><span class="s2">"sourcetype"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"smtp:tls"</span>
|
||||
<span class="n">timestamp</span> <span class="o">=</span> <span class="n">human_timestamp_to_unix_timestamp</span><span class="p">(</span>
|
||||
<span class="n">report</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">])</span>
|
||||
<span class="n">timestamp</span> <span class="o">=</span> <span class="n">human_timestamp_to_unix_timestamp</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">])</span>
|
||||
<span class="n">data</span><span class="p">[</span><span class="s2">"time"</span><span class="p">]</span> <span class="o">=</span> <span class="n">timestamp</span>
|
||||
<span class="n">data</span><span class="p">[</span><span class="s2">"event"</span><span class="p">]</span> <span class="o">=</span> <span class="n">report</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
<span class="n">json_str</span> <span class="o">+=</span> <span class="s2">"</span><span class="si">{0}</span><span class="se">\n</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">data</span><span class="p">))</span>
|
||||
@@ -276,13 +270,14 @@
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">verify</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Skipping certificate verification for Splunk HEC"</span><span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">post</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">url</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="n">json_str</span><span class="p">,</span>
|
||||
<span class="n">timeout</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="p">)</span>
|
||||
<span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">post</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">url</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="n">json_str</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="p">)</span>
|
||||
<span class="n">response</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">SplunkError</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="k">if</span> <span class="n">response</span><span class="p">[</span><span class="s2">"code"</span><span class="p">]</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">SplunkError</span><span class="p">(</span><span class="n">response</span><span class="p">[</span><span class="s2">"text"</span><span class="p">])</span></div></div>
|
||||
<span class="k">raise</span> <span class="n">SplunkError</span><span class="p">(</span><span class="n">response</span><span class="p">[</span><span class="s2">"text"</span><span class="p">])</span></div>
|
||||
</div>
|
||||
|
||||
</pre></div>
|
||||
|
||||
</div>
|
||||
|
||||
+186
-112
@@ -1,23 +1,20 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="../../">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc.utils — parsedmarc 8.15.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css" />
|
||||
<title>parsedmarc.utils — parsedmarc 8.15.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=80d5e7a1" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="../../_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="../../" id="documentation_options" src="../../_static/documentation_options.js"></script>
|
||||
<script src="../../_static/jquery.js"></script>
|
||||
<script src="../../_static/underscore.js"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="../../_static/doctools.js"></script>
|
||||
<script src="../../_static/sphinx_highlight.js"></script>
|
||||
<script src="../../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../../_static/documentation_options.js?v=6add7570"></script>
|
||||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="../../_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="../../genindex.html" />
|
||||
<link rel="search" title="Search" href="../../search.html" />
|
||||
@@ -34,9 +31,6 @@
|
||||
<a href="../../index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.15.0
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
@@ -126,22 +120,30 @@
|
||||
<span class="kn">import</span> <span class="nn">parsedmarc.resources.maps</span>
|
||||
|
||||
|
||||
<span class="n">parenthesis_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s1">'\s*\(.*\)\s*'</span><span class="p">)</span>
|
||||
<span class="n">parenthesis_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">"\s*\(.*\)\s*"</span><span class="p">)</span>
|
||||
|
||||
<span class="n">null_file</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">devnull</span><span class="p">,</span> <span class="s2">"w"</span><span class="p">)</span>
|
||||
<span class="n">mailparser_logger</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s2">"mailparser"</span><span class="p">)</span>
|
||||
<span class="n">mailparser_logger</span><span class="o">.</span><span class="n">setLevel</span><span class="p">(</span><span class="n">logging</span><span class="o">.</span><span class="n">CRITICAL</span><span class="p">)</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="EmailParserError"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.EmailParserError">[docs]</a><span class="k">class</span> <span class="nc">EmailParserError</span><span class="p">(</span><span class="ne">RuntimeError</span><span class="p">):</span>
|
||||
<div class="viewcode-block" id="EmailParserError">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.EmailParserError">[docs]</a>
|
||||
<span class="k">class</span> <span class="nc">EmailParserError</span><span class="p">(</span><span class="ne">RuntimeError</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Raised when an error parsing the email occurs"""</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="DownloadError"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.DownloadError">[docs]</a><span class="k">class</span> <span class="nc">DownloadError</span><span class="p">(</span><span class="ne">RuntimeError</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="DownloadError">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.DownloadError">[docs]</a>
|
||||
<span class="k">class</span> <span class="nc">DownloadError</span><span class="p">(</span><span class="ne">RuntimeError</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Raised when an error occurs when downloading a file"""</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="decode_base64"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.decode_base64">[docs]</a><span class="k">def</span> <span class="nf">decode_base64</span><span class="p">(</span><span class="n">data</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="decode_base64">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.decode_base64">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">decode_base64</span><span class="p">(</span><span class="n">data</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Decodes a base64 string, with padding being optional</span>
|
||||
|
||||
@@ -155,11 +157,14 @@
|
||||
<span class="n">data</span> <span class="o">=</span> <span class="nb">bytes</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s2">"ascii"</span><span class="p">)</span>
|
||||
<span class="n">missing_padding</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">data</span><span class="p">)</span> <span class="o">%</span> <span class="mi">4</span>
|
||||
<span class="k">if</span> <span class="n">missing_padding</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="n">data</span> <span class="o">+=</span> <span class="sa">b</span><span class="s1">'='</span> <span class="o">*</span> <span class="p">(</span><span class="mi">4</span> <span class="o">-</span> <span class="n">missing_padding</span><span class="p">)</span>
|
||||
<span class="n">data</span> <span class="o">+=</span> <span class="sa">b</span><span class="s2">"="</span> <span class="o">*</span> <span class="p">(</span><span class="mi">4</span> <span class="o">-</span> <span class="n">missing_padding</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="n">base64</span><span class="o">.</span><span class="n">b64decode</span><span class="p">(</span><span class="n">data</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="get_base_domain"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_base_domain">[docs]</a><span class="k">def</span> <span class="nf">get_base_domain</span><span class="p">(</span><span class="n">domain</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="get_base_domain">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_base_domain">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">get_base_domain</span><span class="p">(</span><span class="n">domain</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Gets the base domain name for the given domain</span>
|
||||
|
||||
@@ -178,7 +183,10 @@
|
||||
<span class="k">return</span> <span class="n">psl</span><span class="o">.</span><span class="n">privatesuffix</span><span class="p">(</span><span class="n">domain</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="query_dns"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.query_dns">[docs]</a><span class="k">def</span> <span class="nf">query_dns</span><span class="p">(</span><span class="n">domain</span><span class="p">,</span> <span class="n">record_type</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mf">2.0</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="query_dns">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.query_dns">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">query_dns</span><span class="p">(</span><span class="n">domain</span><span class="p">,</span> <span class="n">record_type</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mf">2.0</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Queries DNS</span>
|
||||
|
||||
@@ -204,31 +212,45 @@
|
||||
<span class="n">resolver</span> <span class="o">=</span> <span class="n">dns</span><span class="o">.</span><span class="n">resolver</span><span class="o">.</span><span class="n">Resolver</span><span class="p">()</span>
|
||||
<span class="n">timeout</span> <span class="o">=</span> <span class="nb">float</span><span class="p">(</span><span class="n">timeout</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">nameservers</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">nameservers</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"1.1.1.1"</span><span class="p">,</span> <span class="s2">"1.0.0.1"</span><span class="p">,</span>
|
||||
<span class="s2">"2606:4700:4700::1111"</span><span class="p">,</span> <span class="s2">"2606:4700:4700::1001"</span><span class="p">,</span>
|
||||
<span class="p">]</span>
|
||||
<span class="n">nameservers</span> <span class="o">=</span> <span class="p">[</span>
|
||||
<span class="s2">"1.1.1.1"</span><span class="p">,</span>
|
||||
<span class="s2">"1.0.0.1"</span><span class="p">,</span>
|
||||
<span class="s2">"2606:4700:4700::1111"</span><span class="p">,</span>
|
||||
<span class="s2">"2606:4700:4700::1001"</span><span class="p">,</span>
|
||||
<span class="p">]</span>
|
||||
<span class="n">resolver</span><span class="o">.</span><span class="n">nameservers</span> <span class="o">=</span> <span class="n">nameservers</span>
|
||||
<span class="n">resolver</span><span class="o">.</span><span class="n">timeout</span> <span class="o">=</span> <span class="n">timeout</span>
|
||||
<span class="n">resolver</span><span class="o">.</span><span class="n">lifetime</span> <span class="o">=</span> <span class="n">timeout</span>
|
||||
<span class="k">if</span> <span class="n">record_type</span> <span class="o">==</span> <span class="s2">"TXT"</span><span class="p">:</span>
|
||||
<span class="n">resource_records</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span>
|
||||
<span class="k">lambda</span> <span class="n">r</span><span class="p">:</span> <span class="n">r</span><span class="o">.</span><span class="n">strings</span><span class="p">,</span>
|
||||
<span class="n">resolver</span><span class="o">.</span><span class="n">resolve</span><span class="p">(</span><span class="n">domain</span><span class="p">,</span> <span class="n">record_type</span><span class="p">,</span> <span class="n">lifetime</span><span class="o">=</span><span class="n">timeout</span><span class="p">)))</span>
|
||||
<span class="n">resource_records</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span>
|
||||
<span class="nb">map</span><span class="p">(</span>
|
||||
<span class="k">lambda</span> <span class="n">r</span><span class="p">:</span> <span class="n">r</span><span class="o">.</span><span class="n">strings</span><span class="p">,</span>
|
||||
<span class="n">resolver</span><span class="o">.</span><span class="n">resolve</span><span class="p">(</span><span class="n">domain</span><span class="p">,</span> <span class="n">record_type</span><span class="p">,</span> <span class="n">lifetime</span><span class="o">=</span><span class="n">timeout</span><span class="p">),</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">_resource_record</span> <span class="o">=</span> <span class="p">[</span>
|
||||
<span class="n">resource_record</span><span class="p">[</span><span class="mi">0</span><span class="p">][:</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">resource_record</span><span class="p">)</span>
|
||||
<span class="k">for</span> <span class="n">resource_record</span> <span class="ow">in</span> <span class="n">resource_records</span> <span class="k">if</span> <span class="n">resource_record</span><span class="p">]</span>
|
||||
<span class="k">for</span> <span class="n">resource_record</span> <span class="ow">in</span> <span class="n">resource_records</span>
|
||||
<span class="k">if</span> <span class="n">resource_record</span>
|
||||
<span class="p">]</span>
|
||||
<span class="n">records</span> <span class="o">=</span> <span class="p">[</span><span class="n">r</span><span class="o">.</span><span class="n">decode</span><span class="p">()</span> <span class="k">for</span> <span class="n">r</span> <span class="ow">in</span> <span class="n">_resource_record</span><span class="p">]</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">records</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span>
|
||||
<span class="k">lambda</span> <span class="n">r</span><span class="p">:</span> <span class="n">r</span><span class="o">.</span><span class="n">to_text</span><span class="p">()</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'"'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s2">"."</span><span class="p">),</span>
|
||||
<span class="n">resolver</span><span class="o">.</span><span class="n">resolve</span><span class="p">(</span><span class="n">domain</span><span class="p">,</span> <span class="n">record_type</span><span class="p">,</span> <span class="n">lifetime</span><span class="o">=</span><span class="n">timeout</span><span class="p">)))</span>
|
||||
<span class="n">records</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span>
|
||||
<span class="nb">map</span><span class="p">(</span>
|
||||
<span class="k">lambda</span> <span class="n">r</span><span class="p">:</span> <span class="n">r</span><span class="o">.</span><span class="n">to_text</span><span class="p">()</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'"'</span><span class="p">,</span> <span class="s2">""</span><span class="p">)</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s2">"."</span><span class="p">),</span>
|
||||
<span class="n">resolver</span><span class="o">.</span><span class="n">resolve</span><span class="p">(</span><span class="n">domain</span><span class="p">,</span> <span class="n">record_type</span><span class="p">,</span> <span class="n">lifetime</span><span class="o">=</span><span class="n">timeout</span><span class="p">),</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">cache</span><span class="p">:</span>
|
||||
<span class="n">cache</span><span class="p">[</span><span class="n">cache_key</span><span class="p">]</span> <span class="o">=</span> <span class="n">records</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">records</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="get_reverse_dns"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_reverse_dns">[docs]</a><span class="k">def</span> <span class="nf">get_reverse_dns</span><span class="p">(</span><span class="n">ip_address</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mf">2.0</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="get_reverse_dns">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_reverse_dns">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">get_reverse_dns</span><span class="p">(</span><span class="n">ip_address</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mf">2.0</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Resolves an IP address to a hostname using a reverse DNS query</span>
|
||||
|
||||
@@ -245,9 +267,9 @@
|
||||
<span class="n">hostname</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">address</span> <span class="o">=</span> <span class="n">dns</span><span class="o">.</span><span class="n">reversename</span><span class="o">.</span><span class="n">from_address</span><span class="p">(</span><span class="n">ip_address</span><span class="p">)</span>
|
||||
<span class="n">hostname</span> <span class="o">=</span> <span class="n">query_dns</span><span class="p">(</span><span class="n">address</span><span class="p">,</span> <span class="s2">"PTR"</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="n">cache</span><span class="p">,</span>
|
||||
<span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
|
||||
<span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
|
||||
<span class="n">hostname</span> <span class="o">=</span> <span class="n">query_dns</span><span class="p">(</span>
|
||||
<span class="n">address</span><span class="p">,</span> <span class="s2">"PTR"</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="n">cache</span><span class="p">,</span> <span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span>
|
||||
<span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
|
||||
|
||||
<span class="k">except</span> <span class="n">dns</span><span class="o">.</span><span class="n">exception</span><span class="o">.</span><span class="n">DNSException</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">"get_reverse_dns(</span><span class="si">{</span><span class="n">ip_address</span><span class="si">}</span><span class="s2">) exception: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
@@ -256,7 +278,10 @@
|
||||
<span class="k">return</span> <span class="n">hostname</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="timestamp_to_datetime"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.timestamp_to_datetime">[docs]</a><span class="k">def</span> <span class="nf">timestamp_to_datetime</span><span class="p">(</span><span class="n">timestamp</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="timestamp_to_datetime">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.timestamp_to_datetime">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">timestamp_to_datetime</span><span class="p">(</span><span class="n">timestamp</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Converts a UNIX/DMARC timestamp to a Python ``datetime`` object</span>
|
||||
|
||||
@@ -269,7 +294,10 @@
|
||||
<span class="k">return</span> <span class="n">datetime</span><span class="o">.</span><span class="n">fromtimestamp</span><span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="n">timestamp</span><span class="p">))</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="timestamp_to_human"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.timestamp_to_human">[docs]</a><span class="k">def</span> <span class="nf">timestamp_to_human</span><span class="p">(</span><span class="n">timestamp</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="timestamp_to_human">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.timestamp_to_human">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">timestamp_to_human</span><span class="p">(</span><span class="n">timestamp</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Converts a UNIX/DMARC timestamp to a human-readable string</span>
|
||||
|
||||
@@ -282,7 +310,10 @@
|
||||
<span class="k">return</span> <span class="n">timestamp_to_datetime</span><span class="p">(</span><span class="n">timestamp</span><span class="p">)</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%S"</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="human_timestamp_to_datetime"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.human_timestamp_to_datetime">[docs]</a><span class="k">def</span> <span class="nf">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">human_timestamp</span><span class="p">,</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="human_timestamp_to_datetime">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.human_timestamp_to_datetime">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">human_timestamp</span><span class="p">,</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Converts a human-readable timestamp into a Python ``datetime`` object</span>
|
||||
|
||||
@@ -301,7 +332,10 @@
|
||||
<span class="k">return</span> <span class="n">dt</span><span class="o">.</span><span class="n">astimezone</span><span class="p">(</span><span class="n">timezone</span><span class="o">.</span><span class="n">utc</span><span class="p">)</span> <span class="k">if</span> <span class="n">to_utc</span> <span class="k">else</span> <span class="n">dt</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="human_timestamp_to_unix_timestamp"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.human_timestamp_to_unix_timestamp">[docs]</a><span class="k">def</span> <span class="nf">human_timestamp_to_unix_timestamp</span><span class="p">(</span><span class="n">human_timestamp</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="human_timestamp_to_unix_timestamp">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.human_timestamp_to_unix_timestamp">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">human_timestamp_to_unix_timestamp</span><span class="p">(</span><span class="n">human_timestamp</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Converts a human-readable timestamp into a UNIX timestamp</span>
|
||||
|
||||
@@ -315,7 +349,10 @@
|
||||
<span class="k">return</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">human_timestamp</span><span class="p">)</span><span class="o">.</span><span class="n">timestamp</span><span class="p">()</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="get_ip_address_country"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_ip_address_country">[docs]</a><span class="k">def</span> <span class="nf">get_ip_address_country</span><span class="p">(</span><span class="n">ip_address</span><span class="p">,</span> <span class="n">db_path</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="get_ip_address_country">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_ip_address_country">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">get_ip_address_country</span><span class="p">(</span><span class="n">ip_address</span><span class="p">,</span> <span class="n">db_path</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Returns the ISO code for the country associated</span>
|
||||
<span class="sd"> with the given IPv4 or IPv6 address</span>
|
||||
@@ -344,9 +381,11 @@
|
||||
<span class="k">if</span> <span class="n">db_path</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isfile</span><span class="p">(</span><span class="n">db_path</span><span class="p">)</span> <span class="ow">is</span> <span class="kc">False</span><span class="p">:</span>
|
||||
<span class="n">db_path</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">"No file exists at </span><span class="si">{</span><span class="n">db_path</span><span class="si">}</span><span class="s2">. Falling back to an "</span>
|
||||
<span class="s2">"included copy of the IPDB IP to Country "</span>
|
||||
<span class="s2">"Lite database."</span><span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
||||
<span class="sa">f</span><span class="s2">"No file exists at </span><span class="si">{</span><span class="n">db_path</span><span class="si">}</span><span class="s2">. Falling back to an "</span>
|
||||
<span class="s2">"included copy of the IPDB IP to Country "</span>
|
||||
<span class="s2">"Lite database."</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">db_path</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">for</span> <span class="n">system_path</span> <span class="ow">in</span> <span class="n">db_paths</span><span class="p">:</span>
|
||||
@@ -355,12 +394,12 @@
|
||||
<span class="k">break</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">db_path</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">with</span> <span class="n">pkg_resources</span><span class="o">.</span><span class="n">path</span><span class="p">(</span><span class="n">parsedmarc</span><span class="o">.</span><span class="n">resources</span><span class="o">.</span><span class="n">dbip</span><span class="p">,</span>
|
||||
<span class="s2">"dbip-country-lite.mmdb"</span><span class="p">)</span> <span class="k">as</span> <span class="n">path</span><span class="p">:</span>
|
||||
<span class="k">with</span> <span class="n">pkg_resources</span><span class="o">.</span><span class="n">path</span><span class="p">(</span>
|
||||
<span class="n">parsedmarc</span><span class="o">.</span><span class="n">resources</span><span class="o">.</span><span class="n">dbip</span><span class="p">,</span> <span class="s2">"dbip-country-lite.mmdb"</span>
|
||||
<span class="p">)</span> <span class="k">as</span> <span class="n">path</span><span class="p">:</span>
|
||||
<span class="n">db_path</span> <span class="o">=</span> <span class="n">path</span>
|
||||
|
||||
<span class="n">db_age</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span> <span class="o">-</span> <span class="n">datetime</span><span class="o">.</span><span class="n">fromtimestamp</span><span class="p">(</span>
|
||||
<span class="n">os</span><span class="o">.</span><span class="n">stat</span><span class="p">(</span><span class="n">db_path</span><span class="p">)</span><span class="o">.</span><span class="n">st_mtime</span><span class="p">)</span>
|
||||
<span class="n">db_age</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span> <span class="o">-</span> <span class="n">datetime</span><span class="o">.</span><span class="n">fromtimestamp</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">stat</span><span class="p">(</span><span class="n">db_path</span><span class="p">)</span><span class="o">.</span><span class="n">st_mtime</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">db_age</span> <span class="o">></span> <span class="n">timedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="mi">30</span><span class="p">):</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"IP database is more than a month old"</span><span class="p">)</span>
|
||||
|
||||
@@ -376,12 +415,17 @@
|
||||
<span class="k">return</span> <span class="n">country</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="get_service_from_reverse_dns_base_domain"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_service_from_reverse_dns_base_domain">[docs]</a><span class="k">def</span> <span class="nf">get_service_from_reverse_dns_base_domain</span><span class="p">(</span><span class="n">base_domain</span><span class="p">,</span>
|
||||
<span class="n">always_use_local_file</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">local_file_path</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">url</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">offline</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">reverse_dns_map</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="get_service_from_reverse_dns_base_domain">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_service_from_reverse_dns_base_domain">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">get_service_from_reverse_dns_base_domain</span><span class="p">(</span>
|
||||
<span class="n">base_domain</span><span class="p">,</span>
|
||||
<span class="n">always_use_local_file</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">local_file_path</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">url</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">offline</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">reverse_dns_map</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Returns the service name of a given base domain name from reverse DNS.</span>
|
||||
|
||||
@@ -397,28 +441,27 @@
|
||||
<span class="sd"> If the service is unknown, the name will be</span>
|
||||
<span class="sd"> the supplied reverse_dns_base_domain and the type will be None</span>
|
||||
<span class="sd"> """</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">load_csv</span><span class="p">(</span><span class="n">_csv_file</span><span class="p">):</span>
|
||||
<span class="n">reader</span> <span class="o">=</span> <span class="n">csv</span><span class="o">.</span><span class="n">DictReader</span><span class="p">(</span><span class="n">_csv_file</span><span class="p">)</span>
|
||||
<span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="n">reader</span><span class="p">:</span>
|
||||
<span class="n">key</span> <span class="o">=</span> <span class="n">row</span><span class="p">[</span><span class="s2">"base_reverse_dns"</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
|
||||
<span class="n">reverse_dns_map</span><span class="p">[</span><span class="n">key</span><span class="p">]</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
|
||||
<span class="n">name</span><span class="o">=</span><span class="n">row</span><span class="p">[</span><span class="s2">"name"</span><span class="p">],</span>
|
||||
<span class="nb">type</span><span class="o">=</span><span class="n">row</span><span class="p">[</span><span class="s2">"type"</span><span class="p">])</span>
|
||||
<span class="n">reverse_dns_map</span><span class="p">[</span><span class="n">key</span><span class="p">]</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="n">row</span><span class="p">[</span><span class="s2">"name"</span><span class="p">],</span> <span class="nb">type</span><span class="o">=</span><span class="n">row</span><span class="p">[</span><span class="s2">"type"</span><span class="p">])</span>
|
||||
|
||||
<span class="n">base_domain</span> <span class="o">=</span> <span class="n">base_domain</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
|
||||
<span class="k">if</span> <span class="n">url</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">url</span> <span class="o">=</span> <span class="p">(</span><span class="s2">"https://raw.githubusercontent.com/domainaware"</span>
|
||||
<span class="s2">"/parsedmarc/master/parsedmarc/"</span>
|
||||
<span class="s2">"resources/maps/base_reverse_dns_map.csv"</span><span class="p">)</span>
|
||||
<span class="n">url</span> <span class="o">=</span> <span class="p">(</span>
|
||||
<span class="s2">"https://raw.githubusercontent.com/domainaware"</span>
|
||||
<span class="s2">"/parsedmarc/master/parsedmarc/"</span>
|
||||
<span class="s2">"resources/maps/base_reverse_dns_map.csv"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">reverse_dns_map</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">reverse_dns_map</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
|
||||
<span class="n">csv_file</span> <span class="o">=</span> <span class="n">io</span><span class="o">.</span><span class="n">StringIO</span><span class="p">()</span>
|
||||
|
||||
<span class="k">if</span> <span class="p">(</span><span class="ow">not</span> <span class="p">(</span><span class="n">offline</span> <span class="ow">or</span> <span class="n">always_use_local_file</span><span class="p">)</span>
|
||||
<span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">reverse_dns_map</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">):</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="p">(</span><span class="n">offline</span> <span class="ow">or</span> <span class="n">always_use_local_file</span><span class="p">)</span> <span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">reverse_dns_map</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Trying to fetch "</span>
|
||||
<span class="sa">f</span><span class="s2">"reverse DNS map from </span><span class="si">{</span><span class="n">url</span><span class="si">}</span><span class="s2">..."</span><span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Trying to fetch "</span> <span class="sa">f</span><span class="s2">"reverse DNS map from </span><span class="si">{</span><span class="n">url</span><span class="si">}</span><span class="s2">..."</span><span class="p">)</span>
|
||||
<span class="n">csv_file</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">requests</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="p">)</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
|
||||
<span class="n">csv_file</span><span class="o">.</span><span class="n">seek</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
|
||||
<span class="n">load_csv</span><span class="p">(</span><span class="n">csv_file</span><span class="p">)</span>
|
||||
@@ -426,8 +469,9 @@
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Failed to fetch reverse DNS map: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">reverse_dns_map</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Loading included reverse DNS map..."</span><span class="p">)</span>
|
||||
<span class="k">with</span> <span class="n">pkg_resources</span><span class="o">.</span><span class="n">path</span><span class="p">(</span><span class="n">parsedmarc</span><span class="o">.</span><span class="n">resources</span><span class="o">.</span><span class="n">maps</span><span class="p">,</span>
|
||||
<span class="s2">"base_reverse_dns_map.csv"</span><span class="p">)</span> <span class="k">as</span> <span class="n">path</span><span class="p">:</span>
|
||||
<span class="k">with</span> <span class="n">pkg_resources</span><span class="o">.</span><span class="n">path</span><span class="p">(</span>
|
||||
<span class="n">parsedmarc</span><span class="o">.</span><span class="n">resources</span><span class="o">.</span><span class="n">maps</span><span class="p">,</span> <span class="s2">"base_reverse_dns_map.csv"</span>
|
||||
<span class="p">)</span> <span class="k">as</span> <span class="n">path</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">local_file_path</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">path</span> <span class="o">=</span> <span class="n">local_file_path</span>
|
||||
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">path</span><span class="p">)</span> <span class="k">as</span> <span class="n">csv_file</span><span class="p">:</span>
|
||||
@@ -440,15 +484,21 @@
|
||||
<span class="k">return</span> <span class="n">service</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="get_ip_address_info"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_ip_address_info">[docs]</a><span class="k">def</span> <span class="nf">get_ip_address_info</span><span class="p">(</span><span class="n">ip_address</span><span class="p">,</span>
|
||||
<span class="n">ip_db_path</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">reverse_dns_map_path</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">always_use_local_files</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">reverse_dns_map_url</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">cache</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">reverse_dns_map</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">offline</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mf">2.0</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="get_ip_address_info">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_ip_address_info">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">get_ip_address_info</span><span class="p">(</span>
|
||||
<span class="n">ip_address</span><span class="p">,</span>
|
||||
<span class="n">ip_db_path</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">reverse_dns_map_path</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">always_use_local_files</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">reverse_dns_map_url</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">cache</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">reverse_dns_map</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">offline</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">timeout</span><span class="o">=</span><span class="mf">2.0</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Returns reverse DNS and country information for the given IP address</span>
|
||||
|
||||
@@ -480,9 +530,9 @@
|
||||
<span class="k">if</span> <span class="n">offline</span><span class="p">:</span>
|
||||
<span class="n">reverse_dns</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">reverse_dns</span> <span class="o">=</span> <span class="n">get_reverse_dns</span><span class="p">(</span><span class="n">ip_address</span><span class="p">,</span>
|
||||
<span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
|
||||
<span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">)</span>
|
||||
<span class="n">reverse_dns</span> <span class="o">=</span> <span class="n">get_reverse_dns</span><span class="p">(</span>
|
||||
<span class="n">ip_address</span><span class="p">,</span> <span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">country</span> <span class="o">=</span> <span class="n">get_ip_address_country</span><span class="p">(</span><span class="n">ip_address</span><span class="p">,</span> <span class="n">db_path</span><span class="o">=</span><span class="n">ip_db_path</span><span class="p">)</span>
|
||||
<span class="n">info</span><span class="p">[</span><span class="s2">"country"</span><span class="p">]</span> <span class="o">=</span> <span class="n">country</span>
|
||||
<span class="n">info</span><span class="p">[</span><span class="s2">"reverse_dns"</span><span class="p">]</span> <span class="o">=</span> <span class="n">reverse_dns</span>
|
||||
@@ -498,7 +548,8 @@
|
||||
<span class="n">local_file_path</span><span class="o">=</span><span class="n">reverse_dns_map_path</span><span class="p">,</span>
|
||||
<span class="n">url</span><span class="o">=</span><span class="n">reverse_dns_map_url</span><span class="p">,</span>
|
||||
<span class="n">always_use_local_file</span><span class="o">=</span><span class="n">always_use_local_files</span><span class="p">,</span>
|
||||
<span class="n">reverse_dns_map</span><span class="o">=</span><span class="n">reverse_dns_map</span><span class="p">)</span>
|
||||
<span class="n">reverse_dns_map</span><span class="o">=</span><span class="n">reverse_dns_map</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">info</span><span class="p">[</span><span class="s2">"base_domain"</span><span class="p">]</span> <span class="o">=</span> <span class="n">base_domain</span>
|
||||
<span class="n">info</span><span class="p">[</span><span class="s2">"type"</span><span class="p">]</span> <span class="o">=</span> <span class="n">service</span><span class="p">[</span><span class="s2">"type"</span><span class="p">]</span>
|
||||
<span class="n">info</span><span class="p">[</span><span class="s2">"name"</span><span class="p">]</span> <span class="o">=</span> <span class="n">service</span><span class="p">[</span><span class="s2">"name"</span><span class="p">]</span>
|
||||
@@ -512,6 +563,7 @@
|
||||
<span class="k">return</span> <span class="n">info</span></div>
|
||||
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">parse_email_address</span><span class="p">(</span><span class="n">original_address</span><span class="p">):</span>
|
||||
<span class="k">if</span> <span class="n">original_address</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="s2">""</span><span class="p">:</span>
|
||||
<span class="n">display_name</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
@@ -525,13 +577,19 @@
|
||||
<span class="n">local</span> <span class="o">=</span> <span class="n">address_parts</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
|
||||
<span class="n">domain</span> <span class="o">=</span> <span class="n">address_parts</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">OrderedDict</span><span class="p">([(</span><span class="s2">"display_name"</span><span class="p">,</span> <span class="n">display_name</span><span class="p">),</span>
|
||||
<span class="p">(</span><span class="s2">"address"</span><span class="p">,</span> <span class="n">address</span><span class="p">),</span>
|
||||
<span class="p">(</span><span class="s2">"local"</span><span class="p">,</span> <span class="n">local</span><span class="p">),</span>
|
||||
<span class="p">(</span><span class="s2">"domain"</span><span class="p">,</span> <span class="n">domain</span><span class="p">)])</span>
|
||||
<span class="k">return</span> <span class="n">OrderedDict</span><span class="p">(</span>
|
||||
<span class="p">[</span>
|
||||
<span class="p">(</span><span class="s2">"display_name"</span><span class="p">,</span> <span class="n">display_name</span><span class="p">),</span>
|
||||
<span class="p">(</span><span class="s2">"address"</span><span class="p">,</span> <span class="n">address</span><span class="p">),</span>
|
||||
<span class="p">(</span><span class="s2">"local"</span><span class="p">,</span> <span class="n">local</span><span class="p">),</span>
|
||||
<span class="p">(</span><span class="s2">"domain"</span><span class="p">,</span> <span class="n">domain</span><span class="p">),</span>
|
||||
<span class="p">]</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="get_filename_safe_string"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_filename_safe_string">[docs]</a><span class="k">def</span> <span class="nf">get_filename_safe_string</span><span class="p">(</span><span class="n">string</span><span class="p">):</span>
|
||||
<div class="viewcode-block" id="get_filename_safe_string">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_filename_safe_string">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">get_filename_safe_string</span><span class="p">(</span><span class="n">string</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Converts a string to a string that is safe for a filename</span>
|
||||
|
||||
@@ -541,8 +599,7 @@
|
||||
<span class="sd"> Returns:</span>
|
||||
<span class="sd"> str: A string safe for a filename</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">invalid_filename_chars</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'</span><span class="se">\\</span><span class="s1">'</span><span class="p">,</span> <span class="s1">'/'</span><span class="p">,</span> <span class="s1">':'</span><span class="p">,</span> <span class="s1">'"'</span><span class="p">,</span> <span class="s1">'*'</span><span class="p">,</span> <span class="s1">'?'</span><span class="p">,</span> <span class="s1">'|'</span><span class="p">,</span> <span class="s1">'</span><span class="se">\n</span><span class="s1">'</span><span class="p">,</span>
|
||||
<span class="s1">'</span><span class="se">\r</span><span class="s1">'</span><span class="p">]</span>
|
||||
<span class="n">invalid_filename_chars</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"</span><span class="se">\\</span><span class="s2">"</span><span class="p">,</span> <span class="s2">"/"</span><span class="p">,</span> <span class="s2">":"</span><span class="p">,</span> <span class="s1">'"'</span><span class="p">,</span> <span class="s2">"*"</span><span class="p">,</span> <span class="s2">"?"</span><span class="p">,</span> <span class="s2">"|"</span><span class="p">,</span> <span class="s2">"</span><span class="se">\n</span><span class="s2">"</span><span class="p">,</span> <span class="s2">"</span><span class="se">\r</span><span class="s2">"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="n">string</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">string</span> <span class="o">=</span> <span class="s2">"None"</span>
|
||||
<span class="k">for</span> <span class="n">char</span> <span class="ow">in</span> <span class="n">invalid_filename_chars</span><span class="p">:</span>
|
||||
@@ -554,7 +611,10 @@
|
||||
<span class="k">return</span> <span class="n">string</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="is_mbox"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.is_mbox">[docs]</a><span class="k">def</span> <span class="nf">is_mbox</span><span class="p">(</span><span class="n">path</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="is_mbox">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.is_mbox">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">is_mbox</span><span class="p">(</span><span class="n">path</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Checks if the given content is an MBOX mailbox file</span>
|
||||
|
||||
@@ -575,7 +635,10 @@
|
||||
<span class="k">return</span> <span class="n">_is_mbox</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="is_outlook_msg"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.is_outlook_msg">[docs]</a><span class="k">def</span> <span class="nf">is_outlook_msg</span><span class="p">(</span><span class="n">content</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="is_outlook_msg">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.is_outlook_msg">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">is_outlook_msg</span><span class="p">(</span><span class="n">content</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Checks if the given content is an Outlook msg OLE/MSG file</span>
|
||||
|
||||
@@ -586,10 +649,14 @@
|
||||
<span class="sd"> bool: A flag that indicates if the file is an Outlook MSG file</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">return</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">content</span><span class="p">,</span> <span class="nb">bytes</span><span class="p">)</span> <span class="ow">and</span> <span class="n">content</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span>
|
||||
<span class="sa">b</span><span class="s2">"</span><span class="se">\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1</span><span class="s2">"</span><span class="p">)</span></div>
|
||||
<span class="sa">b</span><span class="s2">"</span><span class="se">\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1</span><span class="s2">"</span>
|
||||
<span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="convert_outlook_msg"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.convert_outlook_msg">[docs]</a><span class="k">def</span> <span class="nf">convert_outlook_msg</span><span class="p">(</span><span class="n">msg_bytes</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="convert_outlook_msg">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.convert_outlook_msg">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">convert_outlook_msg</span><span class="p">(</span><span class="n">msg_bytes</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Uses the ``msgconvert`` Perl utility to convert an Outlook MS file to</span>
|
||||
<span class="sd"> standard RFC 822 format</span>
|
||||
@@ -608,14 +675,16 @@
|
||||
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="s2">"sample.msg"</span><span class="p">,</span> <span class="s2">"wb"</span><span class="p">)</span> <span class="k">as</span> <span class="n">msg_file</span><span class="p">:</span>
|
||||
<span class="n">msg_file</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">msg_bytes</span><span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">subprocess</span><span class="o">.</span><span class="n">check_call</span><span class="p">([</span><span class="s2">"msgconvert"</span><span class="p">,</span> <span class="s2">"sample.msg"</span><span class="p">],</span>
|
||||
<span class="n">stdout</span><span class="o">=</span><span class="n">null_file</span><span class="p">,</span> <span class="n">stderr</span><span class="o">=</span><span class="n">null_file</span><span class="p">)</span>
|
||||
<span class="n">subprocess</span><span class="o">.</span><span class="n">check_call</span><span class="p">(</span>
|
||||
<span class="p">[</span><span class="s2">"msgconvert"</span><span class="p">,</span> <span class="s2">"sample.msg"</span><span class="p">],</span> <span class="n">stdout</span><span class="o">=</span><span class="n">null_file</span><span class="p">,</span> <span class="n">stderr</span><span class="o">=</span><span class="n">null_file</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">eml_path</span> <span class="o">=</span> <span class="s2">"sample.eml"</span>
|
||||
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">eml_path</span><span class="p">,</span> <span class="s2">"rb"</span><span class="p">)</span> <span class="k">as</span> <span class="n">eml_file</span><span class="p">:</span>
|
||||
<span class="n">rfc822</span> <span class="o">=</span> <span class="n">eml_file</span><span class="o">.</span><span class="n">read</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">FileNotFoundError</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">EmailParserError</span><span class="p">(</span>
|
||||
<span class="s2">"Failed to convert Outlook MSG: msgconvert utility not found"</span><span class="p">)</span>
|
||||
<span class="s2">"Failed to convert Outlook MSG: msgconvert utility not found"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">finally</span><span class="p">:</span>
|
||||
<span class="n">os</span><span class="o">.</span><span class="n">chdir</span><span class="p">(</span><span class="n">orig_dir</span><span class="p">)</span>
|
||||
<span class="n">shutil</span><span class="o">.</span><span class="n">rmtree</span><span class="p">(</span><span class="n">tmp_dir</span><span class="p">)</span>
|
||||
@@ -623,7 +692,10 @@
|
||||
<span class="k">return</span> <span class="n">rfc822</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="parse_email"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.parse_email">[docs]</a><span class="k">def</span> <span class="nf">parse_email</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">strip_attachment_payloads</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="parse_email">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.parse_email">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">parse_email</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">strip_attachment_payloads</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> A simplified email parser</span>
|
||||
|
||||
@@ -650,8 +722,7 @@
|
||||
<span class="k">if</span> <span class="n">received</span><span class="p">[</span><span class="s2">"date_utc"</span><span class="p">]</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">del</span> <span class="n">received</span><span class="p">[</span><span class="s2">"date_utc"</span><span class="p">]</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">received</span><span class="p">[</span><span class="s2">"date_utc"</span><span class="p">]</span> <span class="o">=</span> <span class="n">received</span><span class="p">[</span><span class="s2">"date_utc"</span><span class="p">]</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"T"</span><span class="p">,</span>
|
||||
<span class="s2">" "</span><span class="p">)</span>
|
||||
<span class="n">received</span><span class="p">[</span><span class="s2">"date_utc"</span><span class="p">]</span> <span class="o">=</span> <span class="n">received</span><span class="p">[</span><span class="s2">"date_utc"</span><span class="p">]</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"T"</span><span class="p">,</span> <span class="s2">" "</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="s2">"from"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="s2">"From"</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">"headers"</span><span class="p">]:</span>
|
||||
@@ -667,33 +738,36 @@
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"date"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="k">if</span> <span class="s2">"reply_to"</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">:</span>
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"reply_to"</span><span class="p">]</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">parse_email_address</span><span class="p">(</span><span class="n">x</span><span class="p">),</span>
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"reply_to"</span><span class="p">]))</span>
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"reply_to"</span><span class="p">]</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span>
|
||||
<span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">parse_email_address</span><span class="p">(</span><span class="n">x</span><span class="p">),</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">"reply_to"</span><span class="p">])</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"reply_to"</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
|
||||
<span class="k">if</span> <span class="s2">"to"</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">:</span>
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"to"</span><span class="p">]</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">parse_email_address</span><span class="p">(</span><span class="n">x</span><span class="p">),</span>
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"to"</span><span class="p">]))</span>
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"to"</span><span class="p">]</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span>
|
||||
<span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">parse_email_address</span><span class="p">(</span><span class="n">x</span><span class="p">),</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">"to"</span><span class="p">])</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"to"</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
|
||||
<span class="k">if</span> <span class="s2">"cc"</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">:</span>
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"cc"</span><span class="p">]</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">parse_email_address</span><span class="p">(</span><span class="n">x</span><span class="p">),</span>
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"cc"</span><span class="p">]))</span>
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"cc"</span><span class="p">]</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span>
|
||||
<span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">parse_email_address</span><span class="p">(</span><span class="n">x</span><span class="p">),</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">"cc"</span><span class="p">])</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"cc"</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
|
||||
<span class="k">if</span> <span class="s2">"bcc"</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">:</span>
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"bcc"</span><span class="p">]</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">parse_email_address</span><span class="p">(</span><span class="n">x</span><span class="p">),</span>
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"bcc"</span><span class="p">]))</span>
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"bcc"</span><span class="p">]</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span>
|
||||
<span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">parse_email_address</span><span class="p">(</span><span class="n">x</span><span class="p">),</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">"bcc"</span><span class="p">])</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"bcc"</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
|
||||
<span class="k">if</span> <span class="s2">"delivered_to"</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">:</span>
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"delivered_to"</span><span class="p">]</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span>
|
||||
<span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">parse_email_address</span><span class="p">(</span><span class="n">x</span><span class="p">),</span>
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"delivered_to"</span><span class="p">])</span>
|
||||
<span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">parse_email_address</span><span class="p">(</span><span class="n">x</span><span class="p">),</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">"delivered_to"</span><span class="p">])</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="s2">"attachments"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">:</span>
|
||||
@@ -710,9 +784,7 @@
|
||||
<span class="n">payload</span> <span class="o">=</span> <span class="nb">str</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="n">payload</span><span class="p">)</span>
|
||||
<span class="n">attachment</span><span class="p">[</span><span class="s2">"sha256"</span><span class="p">]</span> <span class="o">=</span> <span class="n">hashlib</span><span class="o">.</span><span class="n">sha256</span><span class="p">(</span><span class="n">payload</span><span class="p">)</span><span class="o">.</span><span class="n">hexdigest</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Unable to decode attachment: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span>
|
||||
<span class="p">))</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Unable to decode attachment: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
||||
<span class="k">if</span> <span class="n">strip_attachment_payloads</span><span class="p">:</span>
|
||||
<span class="k">for</span> <span class="n">attachment</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">"attachments"</span><span class="p">]:</span>
|
||||
<span class="k">if</span> <span class="s2">"payload"</span> <span class="ow">in</span> <span class="n">attachment</span><span class="p">:</span>
|
||||
@@ -722,12 +794,14 @@
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"subject"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"filename_safe_subject"</span><span class="p">]</span> <span class="o">=</span> <span class="n">get_filename_safe_string</span><span class="p">(</span>
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"subject"</span><span class="p">])</span>
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"subject"</span><span class="p">]</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="s2">"body"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">:</span>
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"body"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">parsed_email</span></div>
|
||||
|
||||
</pre></div>
|
||||
|
||||
</div>
|
||||
|
||||
Reference in New Issue
Block a user